Skip to content
Browse files

s3 interface is back

Change-Id: I9c87d906c501215e60521bf13175760b88dec900
  • Loading branch information...
1 parent eb0a1d3 commit dd7a8d282bccac6ec4bed68f71e7dc6265648e16 SonicWang committed
Showing with 6,015 additions and 1,127 deletions.
  1. +68 −16 README.md
  2. +53 −0 common/s3-auth.js
  3. +165 −0 common/s3/auth.js
  4. +71 −0 common/s3/utils.js
  5. +9 −0 config.json.sample
  6. +118 −70 doc/blob service driver api.md
  7. +269 −83 doc/blob service rest api.md
  8. +89 −38 drivers/fs/blob_fs.js
  9. +1 −1 drivers/fs/fs_ec.js
  10. +392 −0 drivers/s3/blob_s3.js
  11. +4 −0 drivers/s3/index.js
  12. +165 −0 drivers/s3/s3client/auth.js
  13. +189 −0 drivers/s3/s3client/client.js
  14. +71 −0 drivers/s3/s3client/utils.js
  15. +41 −0 node_modules/express/History.md
  16. +35 −62 node_modules/express/bin/express
  17. +1 −1 node_modules/express/lib/express.js
  18. +1 −2 node_modules/express/lib/http.js
  19. +9 −7 node_modules/express/lib/request.js
  20. +19 −16 node_modules/express/lib/view.js
  21. +1 −1 node_modules/express/node_modules/connect/lib/connect.js
  22. +2 −0 node_modules/express/node_modules/connect/lib/http.js
  23. +135 −31 node_modules/express/node_modules/connect/lib/middleware/bodyParser.js
  24. +1 −1 node_modules/express/node_modules/connect/lib/middleware/static.js
  25. +14 −0 node_modules/express/node_modules/connect/node_modules/formidable/Makefile
  26. +286 −0 node_modules/express/node_modules/connect/node_modules/formidable/Readme.md
  27. +1 −0 node_modules/express/node_modules/connect/node_modules/formidable/index.js
  28. +61 −0 node_modules/express/node_modules/connect/node_modules/formidable/lib/file.js
  29. +377 −0 node_modules/express/node_modules/connect/node_modules/formidable/lib/incoming_form.js
  30. +3 −0 node_modules/express/node_modules/connect/node_modules/formidable/lib/index.js
  31. +312 −0 node_modules/express/node_modules/connect/node_modules/formidable/lib/multipart_parser.js
  32. +25 −0 node_modules/express/node_modules/connect/node_modules/formidable/lib/querystring_parser.js
  33. +6 −0 node_modules/express/node_modules/connect/node_modules/formidable/lib/util.js
  34. +22 −0 node_modules/express/node_modules/connect/node_modules/formidable/package.json
  35. +47 −0 node_modules/express/node_modules/connect/node_modules/formidable/tool/record.js
  36. +4 −3 node_modules/express/node_modules/connect/package.json
  37. +49 −8 node_modules/express/node_modules/connect/test.js
  38. +35 −2 node_modules/express/node_modules/mkdirp/README.markdown
  39. +1 −1 node_modules/express/node_modules/mkdirp/examples/pow.js
  40. +72 −13 node_modules/express/node_modules/mkdirp/index.js
  41. +2 −2 node_modules/express/node_modules/mkdirp/package.json
  42. +21 −0 node_modules/express/node_modules/qs/History.md
  43. +1 −3 node_modules/express/node_modules/qs/Makefile
  44. +11 −6 node_modules/express/node_modules/qs/Readme.md
  45. +3 −0 node_modules/express/node_modules/qs/examples.js
  46. +104 −76 node_modules/express/node_modules/qs/lib/querystring.js
  47. +9 −2 node_modules/express/node_modules/qs/package.json
  48. +6 −6 node_modules/express/package.json
  49. +27 −22 node_modules/express/testing/index.js
  50. +4 −0 node_modules/sax/README.md
  51. +37 −12 node_modules/sax/lib/sax.js
  52. +3 −3 node_modules/sax/package.json
  53. +11 −3 node_modules/sax/test/index.js
  54. +19 −0 node_modules/vows/README.md
  55. +50 −20 node_modules/vows/bin/vows
  56. +18 −7 node_modules/vows/lib/assert/error.js
  57. +74 −11 node_modules/vows/lib/vows.js
  58. +8 −0 node_modules/vows/lib/vows/console.js
  59. +21 −0 node_modules/vows/lib/vows/context.js
  60. +18 −1 node_modules/vows/lib/vows/reporters/json.js
  61. +0 −2 node_modules/vows/lib/vows/reporters/spec.js
  62. +83 −22 node_modules/vows/lib/vows/suite.js
  63. +6 −3 node_modules/vows/node_modules/eyes/lib/eyes.js
  64. +3 −2 node_modules/vows/node_modules/eyes/package.json
  65. +5 −4 node_modules/vows/node_modules/eyes/test/eyes-test.js
  66. +1 −1 node_modules/vows/package.json
  67. +1 −1 node_modules/vows/test/assert-test.js
  68. +148 −0 node_modules/vows/test/vows-test.js
  69. +10 −2 node_modules/winston/.travis.yml
  70. +55 −6 node_modules/winston/README.md
  71. +2 −1 node_modules/winston/lib/winston.js
  72. +2 −2 node_modules/winston/lib/winston/common.js
  73. +9 −3 node_modules/winston/lib/winston/container.js
  74. +23 −3 node_modules/winston/lib/winston/logger.js
  75. +14 −1 node_modules/winston/lib/winston/transports/file.js
  76. +14 −6 node_modules/winston/lib/winston/transports/loggly.js
  77. +53 −11 node_modules/winston/node_modules/async/README.md
  78. +1 −1 node_modules/winston/node_modules/async/dist/async.min.js
  79. +32 −17 node_modules/winston/node_modules/async/lib/async.js
  80. +2 −2 node_modules/winston/node_modules/async/package.json
  81. +91 −7 node_modules/winston/node_modules/async/test/test-async.js
  82. +4 −1 node_modules/winston/node_modules/colors/MIT-LICENSE.txt
  83. +58 −11 node_modules/winston/node_modules/colors/ReadMe.md
  84. +65 −26 node_modules/winston/node_modules/colors/colors.js
  85. +62 −8 node_modules/winston/node_modules/colors/example.html
  86. +59 −14 node_modules/winston/node_modules/colors/example.js
  87. +1 −1 node_modules/winston/node_modules/colors/package.json
  88. +6 −3 node_modules/winston/node_modules/eyes/lib/eyes.js
  89. +3 −2 node_modules/winston/node_modules/eyes/package.json
  90. +5 −4 node_modules/winston/node_modules/eyes/test/eyes-test.js
  91. +2 −2 node_modules/winston/node_modules/loggly/README.md
  92. +2 −2 node_modules/winston/node_modules/loggly/lib/loggly/config.js
  93. +100 −5 node_modules/winston/node_modules/loggly/node_modules/request/README.md
  94. +386 −181 node_modules/winston/node_modules/loggly/node_modules/request/main.js
  95. +3 −2 node_modules/winston/node_modules/loggly/node_modules/request/package.json
  96. +31 −2 node_modules/winston/node_modules/loggly/node_modules/request/tests/server.js
  97. +55 −38 node_modules/winston/node_modules/loggly/node_modules/request/tests/test-body.js
  98. +1 −1 node_modules/winston/node_modules/loggly/node_modules/request/tests/test-errors.js
  99. +155 −109 node_modules/winston/node_modules/loggly/node_modules/request/tests/test-pipes.js
  100. +64 −61 node_modules/winston/node_modules/loggly/node_modules/request/tests/test-timeout.js
  101. +3 −3 node_modules/winston/node_modules/loggly/package.json
  102. +3 −3 node_modules/winston/node_modules/pkginfo/package.json
  103. +2 −2 node_modules/winston/node_modules/pkginfo/test/pkginfo-test.js
  104. +1 −1 node_modules/winston/package.json
  105. +2 −2 node_modules/winston/test/helpers.js
  106. +39 −2 node_modules/winston/test/logger-test.js
  107. +9 −3 node_modules/winston/test/transports/loggly-test.js
  108. +2 −2 node_modules/winston/test/winston-test.js
  109. +24 −9 server.js
  110. +55 −0 test/cf_test/testcfbinding.js
  111. +1 −0 test/cf_test/utils.js
  112. +124 −0 test/common_test/testbasic.js
  113. +107 −0 test/common_test/testcontainername.js
  114. +165 −0 test/common_test/testcopyfile.js
  115. +190 −0 test/common_test/testgetfile.js
Sorry, we could not display the entire diff because it was too big.
View
84 README.md
@@ -1,18 +1,20 @@
# node.js blob service gateway.
-Copyright (c) 2011 VMware, Inc.
+Copyright (c) 2011-2012 VMware, Inc.
-The blob service provides an HTTP endpoint to an underlying storage provider. A driver model is used for different providers. Currently the available driver includes a local file system (FS) driver.
+The blob service provides an S3-compatible HTTP endpoint to an underlying storage provider. A driver model is used for different providers. Currently the available drivers include S3 (Amazon web services) or a local file system (FS) driver.
## Authors
- Sonic Wang (wangs@vmware.com)
## Features
- RESTful web service
-- plugin model (currently support local fs)
+- S3 compatibility
+- plugin model (currently support local fs, s3)
- streaming in/out blobs
- basic blob operations: create/delete/get/copy
-- create/list/delete containers
+- create/list/delete buckets
+- s3 driver: enumerate with prefix/delimiter/marker/max-keys
- user defined meta data
## API Documentation
@@ -55,8 +57,11 @@ now edit `config.json`
## FS driver setup
The FS driver uses directories and files starting at a "root" location in the local file system. For scalability, the fs driver design supports multiple instances of the gateway operating on the same files. Metadata and blobs live in separate physical files in different directories. New files do not replace existing files until they have been fully uploaded and persisted. Old files are cleaned up using garbage collection processes which run in the background.
+## S3 driver setup
+The S3 driver requires a valid id/key pair from your Amazon S3 storage account (no additional metadata storage is used)
+
## Configuration via config.json
-The gateway and its drivers can be configured via config.json which is read at startup. Values that need to be configured are indicated with `<value>`. Drivers are assumed to live under `./drivers/<type>` -- currently only `fs` is included.
+The gateway and its drivers can be configured via config.json which is read at startup. Values that need to be configured are indicated with `<value>`. Drivers are assumed to live under `./drivers/<type>` -- currently only `fs` and `s3` driver types are included.
{
"drivers": [
@@ -79,6 +84,15 @@ The gateway and its drivers can be configured via config.json which is read at s
"obj_limit" : <maximum number of objects allowed to store, default is unlimited>
}
}
+ },
+ {
+ "<s3-driver-name>": {
+ "type": "s3",
+ "option": {
+ "key": "<s3 key>",
+ "secret": "<s3 secret>"
+ }
+ }
}
],
"port": "<port>",
@@ -87,18 +101,22 @@ The gateway and its drivers can be configured via config.json which is read at s
"logfile": "<pathname for the log>",
"keyID": "<any id for auth>",
"secretID": "<any secret for auth>",
- "auth": "<basic, digest; other values mean disabled>",
- "debug": true
+ "auth": "<basic, digest or s3; other values mean disabled>",
+ "debug": true,
+ "account_file" : "./account.json",
+ "account_api" : true
}
`current driver` is used to specify the driver to be used by the service at startup. Only 1 driver can be in use at any time. If no default is specified the first driver will be used.
`logfile` specifies the path to the log file. The `logtype` has to be [winston](https://github.com/indexzero/winston).
-`keyID` and `secretID` and `auth` are used to control front-end authentication. If either the key or id is not present or if auth is not set to a proper auth type then authentication is disabled. Currently the following types are supported: "basic", and "digest". "basic" and "digest" implementation follows rfc2617.
+`keyID` and `secretID` and `auth` are used to control front-end authentication. If either the key or id is not present or if auth is not set to a proper auth type then authentication is disabled. Currently the following types are supported: "basic", "digest", and "s3". "basic" and "digest" implementation follows rfc2617.
`debug` is used to log request and response headers to the console for debugging purposes. Its value is treated as boolean.
+`account_file` is the place where vblob instance stores the CloudFoundry service binding credentials. After a vblob service instance is provisioned in CloundFoundry, user can binding multiple CF apps to the instance. This file indicates vblob where to load and update the binding crendentials so that the apps can properly authenticate with the vblob instance. `account_api` controls whether the CF bind/unbind API is on/off in the instance. For a single node deployment, this is always set to true. For a multi-node deployment, only one node should turn on this switch.
+
## Usage
node server.js [-f path_to_config_file]
@@ -119,11 +137,11 @@ The following curl commands assume:
- authentication is NOT enabled. (set `"auth":"disabled"` in config.json)
- the node.js process is running on localhost and listening on port 3000.
-### Listing all containers
+### Listing all buckets
curl http://localhost:3000 -v
-### Listing files in a container
+### Listing objects in a bucket
curl http://localhost:3000/container1 -v
@@ -133,11 +151,11 @@ One could add a query to the URL. Currently four criteria are supported: prefix;
The above query will also list virtual folders in result as well.
-### Create a container
+### Create a bucket
curl http://localhost:3000/container1 -X PUT -v
-### Delete a container
+### Delete a bucket
curl http://localhost:3000/container1 -X DELETE -v
@@ -145,17 +163,17 @@ The above query will also list virtual folders in result as well.
curl http://localhost:3000/container1/file1.txt -X PUT -T file1.txt -v
-Currently user-defined meta data is supported. All user meta keys start with prefix `x-blb-meta-`. E.g.:
+Currently user-defined meta data is supported. All user meta keys start with prefix `x-amz-meta-`. E.g.:
- curl http://localhost:3000/container1/file1.txt -X PUT -T file1.txt -H "x-blb-meta-comment:hello_world"
+ curl http://localhost:3000/container1/file1.txt -X PUT -T file1.txt -H "x-amz-meta-comment:hello_world"
### Copying a file
- curl http://localhost:3000/container1/file1.txt -X PUT -H "x-blb-copy-from:/container2/file2.txt"
+ curl http://localhost:3000/container1/file1.txt -X PUT -H "x-amz-copy-source:/container2/file2.txt"
The above request will direct gateway to copy file2.txt in container2 to file1.txt in container1. Currently only intra-driver copy is supported. This means both container1 and container2 must be within the same driver(backend). This operation will copy meta data as well. All user-defined meta data in file2.txt will be copied to file1.txt.
-This operation will return code `200`. In addition, the response body includes a JSON format file. It has two fields: `LastModified`, and `ETag`.
+This operation will return code `200`. In addition, the response body includes a JSON format object. It has two fields: `LastModified`, and `ETag`.
### Deleting a file
@@ -169,6 +187,40 @@ Currently additional header `range` is supported for single range read as well.
curl http://localhost:3000/container1/file1.txt -H "range:bytes=123-892" -v
+## Using s3-curl with gateway authentication
+
+Instead of using curl with authentication disabled, the gateway can also be accessed in an authenticated fashion via s3-curl.pl which is a utility for making signed requests to s3 available on [aws](http://aws.amazon.com/code/128).
+
+Credentials for authentication via s3-curl are stored in `.s3curl` as follows
+
+ %awsSecretAccessKeys = (
+
+ # real s3 account
+ s3 => {
+ id => '<s3-id>',
+ key => '<s3-key',
+ },
+
+ gateway => {
+ id => '<gateway-id>',
+ key => '<gateway-key>',
+ },
+ );
+
+Requests are then performed by specifying which credentials to use on the command line. Parameters for `curl` go after the `--`. E.g.
+
+ ./s3curl.pl --id gateway -- -X DELETE -v http://localhost:3000/BucketToDelete
+
+A small modification is required to add the endpoint where the gateway is running to the list of endpoints in the perl script. E.g. if you are running on localhost, you would add localhost to the @endpoints array as follows:
+
+ # begin customizing here
+ my @endpoints = ( 's3.amazonaws.com',
+ 's3-us-west-1.amazonaws.com',
+ 's3-eu-west-1.amazonaws.com',
+ 's3-ap-southeast-1.amazonaws.com',
+ 's3-ap-northeast-1.amazonaws.com',
+ 'localhost' );
+
## Server Tuning
When the gateway is handling a large number of concurrent requests, it may open too many file descriptors. It is suggested to increase the file descriptor limit. E.g.: in unix-type systems:
View
53 common/s3-auth.js
@@ -0,0 +1,53 @@
+/*!
+* Copyright(c) 2010 LearnBoost <dev@learnboost.com>
+* Portions Copyright (c) 2011-2012 VMware, Inc.
+* MIT Licensed
+*/
+var utils = require('./s3/utils');
+var auth = require('./s3/auth');
+var join = require('path').join;
+
+var validate = function(keyID, secretID, method, targets, headers, signature){
+ var content_md5 = "";
+ var content_type = "";
+ var cnt = 1;
+ if (method==="PUT" && targets.filename) //only creating file checkes md5
+ { cnt+=1; }
+ if(true){
+ var keys = Object.keys(headers);
+ for (var idx=0, cnt2=0; idx<keys.length && cnt2<cnt;idx++) {
+ if (cnt === 2 && keys[idx].match(/^content-md5$/i)) { cnt2++; content_md5=headers[keys[idx]]; }
+ else if (keys[idx].match(/^content-type$/i)) { cnt2++; content_type=headers[keys[idx]]; }
+ }
+ }
+ // Authorization header
+ //resource: "/" for listing containers; otherwise container or file level operations
+ var date = headers.Date; //use string form from header, no transformation
+ if (date === undefined) { date = headers.date; }
+ if (date === undefined) { return false; }
+ var Authorization = auth.authorization({
+ key: keyID
+ , secret: secretID
+ , verb: method
+ , md5 : content_md5
+ , contentType: content_type
+ , date: date
+ , resource: auth.canonicalizeResource((targets.container===undefined || targets.container === null)?'/':(targets.filename ?/* join('/', targets.container, targets.filename)*/ '/'+targets.container+'/'+targets.filename+utils.to_query_string(targets.query):join('/',targets.container)+utils.to_query_string(targets.query)))
+ , contentType: content_type
+ , amazonHeaders: auth.canonicalizeHeaders(headers)
+ });
+ return Authorization === signature;
+};
+
+module.exports.authenticate = function(creds, method, targets, headers, signature, resp){
+ var key = null;
+ if (signature) {
+ key = signature.substring(4,signature.indexOf(':'));
+ }
+ if (!key || !creds[key] || validate(key, creds[key], method, targets, headers, signature) === false) {
+ resp.resp_code = 401; resp.resp_header = {}; resp.resp_body = {Error:{Code:"Unauthorized",Message:"Signature does not match"}};
+ return false;
+ }
+ return true;
+};
+
View
165 common/s3/auth.js
@@ -0,0 +1,165 @@
+/*!
+* knox - auth
+* Copyright(c) 2010 LearnBoost <dev@learnboost.com>
+* Portions Copyright (c) 2011-2012 VMware, Inc.
+* MIT Licensed
+*/
+
+/**
+ * Module dependencies.
+ */
+
+var crypto = require('crypto');
+var url = require('url');
+
+/**
+ * Return an "Authorization" header value with the given `options`
+ * in the form of "AWS <key>:<signature>"
+ *
+ * @param {Object} options
+ * @return {String}
+ * @api private
+ */
+
+exports.authorization = function(options){
+ return 'AWS ' + options.key + ':' + exports.sign(options);
+};
+
+/**
+ * Simple HMAC-SHA1 Wrapper
+ *
+ * @param {Object} options
+ * @return {String}
+ * @api private
+ */
+
+exports.hmacSha1 = function(options){
+ return crypto.createHmac('sha1', options.secret).update(options.message).digest('base64');
+};
+
+/**
+ * Create a base64 sha1 HMAC for `options`.
+ *
+ * @param {Object} options
+ * @return {String}
+ * @api private
+ */
+
+exports.sign = function(options){
+ options.message = exports.stringToSign(options);
+ //console.log('message to sign: '+options.message);
+ return exports.hmacSha1(options);
+};
+
+/**
+ * Create a base64 sha1 HMAC for `options`.
+ *
+ * Specifically to be used with S3 presigned URLs
+ *
+ * @param {Object} options
+ * @return {String}
+ * @api private
+ */
+
+exports.signQuery = function(options){
+ options.message = exports.queryStringToSign(options);
+ return exports.hmacSha1(options);
+};
+
+/**
+ * Return a string for sign() with the given `options`.
+ *
+ * Spec:
+ *
+ * <verb>\n
+ * <md5>\n
+ * <content-type>\n
+ * <date>\n
+ * [headers\n]
+ * <resource>
+ *
+ * @param {Object} options
+ * @return {String}
+ * @api private
+ */
+
+exports.stringToSign = function(options){
+ var headers = options.amazonHeaders || '';
+ if (headers) { headers += '\n'; }
+ return [
+ options.verb
+ , options.md5
+ , options.contentType
+ , options.date//.toUTCString()
+ , headers + options.resource
+ ].join('\n');
+};
+
+/**
+ * Return a string for sign() with the given `options`, but is meant exclusively
+ * for S3 presigned URLs
+ *
+ * Spec:
+ *
+ * <date>\n
+ * <resource>
+ *
+ * @param {Object} options
+ * @return {String}
+ * @api private
+ */
+
+exports.queryStringToSign = function(options){
+ return 'GET\n\n\n' +
+ options.date + '\n' +
+ options.resource;
+};
+
+/**
+ * Perform the following:
+ *
+ * - ignore non-amazon headers
+ * - lowercase fields
+ * - sort lexicographically
+ * - trim whitespace between ":"
+ * - join with newline
+ *
+ * @param {Object} headers
+ * @return {String}
+ * @api private
+ */
+
+exports.canonicalizeHeaders = function(headers){
+ var buf = []
+ , fields = Object.keys(headers).sort();
+ for (var i = 0, len = fields.length; i < len; ++i) {
+ var field = fields[i]
+ , val = headers[field];
+ field = field.toLowerCase();
+ if (0 !== field.indexOf('x-amz')) { continue; }
+ buf.push(field + ':' + val);
+ }
+ return buf.join('\n');
+};
+
+/**
+ * Perform the following:
+ *
+ * - ignore non sub-resources
+ * - sort lexicographically
+ *
+ * @param {String} resource
+ * @return {String}
+ * @api private
+ */
+exports.canonicalizeResource = function(resource){
+ var urlObj = url.parse(resource, true);
+ var buf = urlObj.pathname;
+ var qbuf = [];
+ Object.keys(urlObj.query).forEach(function (qs) {
+ if (['acl', 'location', 'logging', 'notification', 'partNumber', 'policy', 'requestPayment', 'torrent', 'uploadId', 'uploads', 'versionId', 'versioning', 'versions', 'website','response-content-type', 'response-content-language','response-expires','response-cache-control','response-content-disposition','response-content-encoding'].indexOf(qs) !== -1) {
+ qbuf.push(qs + (urlObj.query[qs] !== '' ? '=' + /*encodeURIComponent*/(urlObj.query[qs]) : ''));
+ }
+ });
+ return buf + (qbuf.length !== 0 ? '?' + qbuf.sort().join('&') : '');
+};
View
71 common/s3/utils.js
@@ -0,0 +1,71 @@
+/*!
+* knox - utils
+* Copyright(c) 2010 LearnBoost <dev@learnboost.com>
+* Portions Copyright (c) 2011-2012 VMware, Inc.
+* MIT Licensed
+*/
+
+exports.to_query_string = function(options) {
+ if (options === null || options === undefined) { return ''; }
+ var filter = ['acl','notification','partNumber','policy','requestPayment','torrent', 'uploadId', 'uploads', 'versionId', 'versioning', 'versions', 'website','prefix','max-keys','marker','delimiter','location','logging','response-content-type', 'response-content-language','response-expires','response-cache-control','response-content-disposition','response-content-encoding'];
+ var keys = Object.keys(options);
+ var query_string = '';
+ for (var i = 0, len = keys.length; i < len; ++i) {
+ var key = keys[i];
+ var lkey = key.toLowerCase();
+ if (filter.indexOf(lkey) !== -1) {
+ if (query_string === '') { query_string += '?'; } else { query_string += '&'; }
+ query_string += lkey + (options[key]?('=' + encodeURIComponent(options[key])):"");
+ }
+ }
+ return query_string;
+};
+
+/**
+ * Merge object `b` with object `a`.
+ *
+ * @param {Object} a
+ * @param {Object} b
+ * @return {Object} a
+ * @api private
+ */
+
+exports.merge = function(a, b){
+ var keys = Object.keys(b);
+ for (var i = 0, len = keys.length; i < len; ++i) {
+ var key = keys[i];
+ a[key] = b[key];
+ }
+ return a;
+};
+
+/**
+ * Base64.
+ */
+
+exports.base64 = {
+
+ /**
+ * Base64 encode the given `str`.
+ *
+ * @param {String} str
+ * @return {String}
+ * @api private
+ */
+
+ encode: function(str){
+ return new Buffer(str).toString('base64');
+ },
+
+ /**
+ * Base64 decode the given `str`.
+ *
+ * @param {String} str
+ * @return {String}
+ * @api private
+ */
+
+ decode: function(str){
+ return new Buffer(str, 'base64').toString();
+ }
+};
View
9 config.json.sample
@@ -19,6 +19,15 @@
"obj_limit" : <maximum number of objects allowed to store, default is unlimited>
}
}
+ },
+ {
+ "s3-sonic": {
+ "type": "s3",
+ "option": {
+ "key": "<s3 key>",
+ "secret": "<s3 secret>"
+ }
+ }
}
],
"port": "3000",
View
188 doc/blob service driver api.md
@@ -1,30 +1,35 @@
# Blob Service driver API
-Copyright (c) 2011 VMware, Inc.
+Copyright (c) 2011-2012 VMware, Inc.
## constructor
driver = createDriver(options, callback)
-`createDriver` is a factory method, and the only function exported by each driver module. The function returns an instance of a driver interface configured with the specified connection options. The Blob service is currently configured to call this method for FS type driver upon startup.
+`createDriver` is a factory method, and the only function exported by each driver module. The function returns an instance of a driver interface configured with the specified connection options. The Blob service is currently configured to call this method for s3 and FS type drivers upon startup.
`options` is a hash variable that contains credentials for specific drivers. These corresponds to the `options` specified in config.json for each driver.
In addition, all drivers receive:
-- `options.logger`: a logger file on which to call error(), debug(), info(), and warn() with a message string for logging purposes.
+- `options.logger`: a logger object on which to call error(), debug(), info(), and warn() with a message string for logging purposes.
The FS driver receives:
- `options.root`: the root folder storing all the blobs
-createDriver() returns the driver file immediately, but internally it may still be waiting to finish initializing itself. The driver should also return itself in callback(driver) when it is done initializing.
+The S3 driver receives:
+
+- `options.key` : s3 account id
+- `options.secret` : secret for the above id
+
+createDriver() returns the driver object immediately, but internally it may still be waiting to finish initializing itself. The driver should also return itself in callback(driver) when it is done initializing.
## General request/response flow
-The driver api supports passage of request and response information between the client and the back-end service via an `options` file passed into the driver on some methods (see specific methods below), and via a `response_header` file on the return. Note that the names of headers are passed to and from driver methods as lower-cased JSON keys.
+The driver api supports passage of request and response information between the client and the back-end service via an `options` object passed into the driver on some methods (see specific methods below), and via a `response_header` object on the return. Note that the names of headers are passed to and from driver methods as lower-cased JSON keys.
-Inbound streams are passed to the driver as a stream file on which the driver can register `data` and `end` event handlers or connect via a `pipe()`. Similarly, outbound streams are returned by the driver as a `response_data` stream which the gateway can `pipe()` back to the client.
+Inbound streams are passed to the driver as a stream object on which the driver can register `data` and `end` event handlers or connect via a `pipe()`. Similarly, outbound streams are returned by the driver as a `response_data` stream which the gateway can `pipe()` back to the client.
Each interface method includes a callback allowing requests to be processed by the driver asynchronously. The callback has the following signature:
@@ -32,8 +37,10 @@ Each interface method includes a callback allowing requests to be processed by t
- `reponse_code` : HTTP response code
- `response_header` : array of HTTP headers
-- `response_body` : response file for non-streaming responses -- will be converted to XML by the gateway
-- `response_data` : null except for responses which stream back data -- must be an file that supports pipe()
+- `response_body` : response object for non-streaming responses -- will be converted to XML by the gateway
+- `response_data` : null except for responses which stream back data -- must be an object that supports pipe()
+
+In order to return s3-compatible XML, the JSON object returned in `response_body` must match the s3 XML response schema for the respective operation i.e all keys are mapped directly to element names in XML.
## errors
@@ -42,96 +49,111 @@ Drivers should return errors via the callback (not via throw) as follows.
callback (http-response-code, null, {"Error": { "Code": error-code, "Message": error-message }});
-## container operations
+## bucket operations
-### container list
+### bucket list
container_list (callback)
-Enumerates all containers owned by the client.
+Enumerates all buckets owned by the client.
-The response body is a javascript file with the following structure: Note that there may be multiple files in the Container array, and CreationDate values follow the ISO convention for XML date strings.
+The response body is a javascript object with the following structure: Note that there may be multiple objects in the Bucket array, and CreationDate values follow the ISO convention for XML date strings.
- { ContainerList: { Containers: { Container: [ {Name: "Container1", CreationDate: "2011-10-01T01:20:36.000Z"}, ... ] }}}
+ { ListAllMyBucketsResult: { Buckets: { Bucket: [ {Name: "Bucket1", CreationDate: "2011-10-01T01:20:36.000Z"}, ... ] }}}
-The javascript file is converted to XML by the gateway before being sent back to the client.
+The javascript object is converted to XML by the gateway before being sent back to the client.
Note that the gateway may insert Owner information if it has any.
<?xml version="1.0" encoding="UTF-8"?>
- <ContainerList xmlns="https://github.com/vmware-bdc/vblob/">
- <Containers>
- <Container>
- <Name>Container1</Name>
+ <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Owner>
+ <ID>..</ID>
+ <DisplayName>...</DisplayName>
+ </Owner>
+ <Buckets>
+ <Bucket>
+ <Name>Bucket1</Name>
<CreationDate>2011-10-01T01:20:36.000Z</CreationDate>
- </Container>
- <Container>
- <Name>Container2</Name>
+ </Bucket>
+ <Bucket>
+ <Name>Bucket2</Name>
<CreationDate>2011-10-01T01:20:37.000Z</CreationDate>
- </Container>
+ </Bucket>
...
- </Containers>
- </ContainerList>
+ </Buckets>
+ </ListAllMyBucketsResult>
-### container create
+### bucket create
- container_create (container_name, options, data_stream, callback)
+ container_create (bucket_name, options, data_stream, callback)
-Creates a new container.
+Creates a new bucket.
-- `options` are parameters parsed from the request
-- `data_stream` is an optional payload -- this parameter is currently ignored by fs driver.
-- If the container already exists for the same account, the response is the same as a successfull creation (return 200).
-- If the container name is invalid, return 400 and an error code `InvalidContainerName`.
+- `options` are parameters parsed from the request such as `x-amz-acl` -- this parameter is currently ignored by both drivers.
+- `data_stream` is an optional payload for this like `BucketConfiguration` -- this parameter is also currently ignored by both drivers.
+- If the bucket already exists for the same account, the response is the same as a successfull creation (return 200).
+- The request may be rejected on s3 if the bucket name is already taken by another user -- returns 409 and error code `BucketAlreadyExists`
+- If the maximum number buckets is reached, return 400 and an error code `TooManyBuckets`. The default limit in S3 and the FS driver is 100 buckets per account.
+- If the bucket name is invalid, return 400 and an error code `InvalidBucketName`.
-### container delete
+### bucket delete
- container_delete (container_name, callback)
+ container_delete (bucket_name, callback)
-Driver should check if a container is empty. If not, driver should return 409 with error code `ContainerNotEmpty`
+Driver should check if a bucket is empty. If not, driver should return 409 with error code `BucketNotEmpty`
-### container options
+### bucket options
- container_options (container_name, options, callback)
+ container_options (bucket_name, options, callback)
Currently not implemented in any drivers, but intended to retrieve options and policies such as `location` and `logging` or ACLs.
-## file operations
+## object operations
-### file list
+### object list
- file_list (container_name, options, callback)
+ file_list (bucket_name, options, callback)
-Returns a list of file keys in a container.
+Returns a list of object keys in a bucket.
+This operation is only supported by the S3 driver currently.
`options` are query parameters parsed from request query
-- `options.marker` : specifying the starting key for listing files
-- `options.prefix` : specifying the prefix of files that should be listed
-- `options.delimiter` : specifying the delimiter of files that should be listed
-- `options.max-keys` : specifying the max number of files to be listed in on query
+- `options.marker` : specifying the starting key for listing objects
+- `options.prefix` : specifying the prefix of objects that should be listed
+- `options.delimiter` : specifying the delimiter of objects that should be listed
+- `options.max-keys` : specifying the max number of objcts to be listed in on query
-The result returned in the response_body of the callback is a JSON file with the following structure. Note that `Contents` array can contain a variable number of files.
+The result returned in the response_body of the callback is a JSON object with the following structure. Note that `Contents` array can contain a variable number of objects.
{
- "FileList": {
- "Name": "Container1",
+ "ListBucketResult": {
+ "Name": "Bucket1",
"Prefix": {},
"Marker": {},
"MaxKeys": "1000",
"IsTruncated": "false",
"Contents": [
{
- "Key": "file_key1",
+ "Key": "object_key1",
"LastModified": "2011-07-27T17:29:23.000Z",
"ETag": "\"0c0e7e404e17edd568853997813f9354\"",
"Size": "47392",
+ "Owner": {
+ "ID": "03b14d14cb23421fe7400e872df6fd8969efbb32f4584207fa5e5cf5ae580ca8",
+ "DisplayName": "someone"
+ },
"StorageClass": "STANDARD"
},
{
- "Key": "file_key2",
+ "Key": "object_key2",
"LastModified": "2011-07-27T21:16:00.000Z",
"ETag": "\"d1919d0a02a24aa16412301e9e9dfbe4\"",
"Size": "58144",
+ "Owner": {
+ "ID": "03b14d14cb23421fe7400e872df6fd8969efbb32f4584207fa5e5cf5ae580ca8",
+ "DisplayName": "someone"
+ },
"StorageClass": "STANDARD"
},
...
@@ -140,35 +162,43 @@ The result returned in the response_body of the callback is a JSON file with the
This response is the translated by the server into XML.
<?xml version ="1.0" encoding="UTF-8"?>
- <FileList xmlns="https://github.com/vmware-bdc/vblob/">
- <Name>Container1</Name>
+ <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Name>Bucket1</Name>
<Prefix/>
<Marker/>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
- <Key>file_key1</Key>
+ <Key>object_key1</Key>
<LastModified>2011-07-27T17:29:23.000Z</LastModified>
<ETag>"0c0e7e404e17edd568853997813f9354"</ETag>
<Size>47392</Size>
+ <Owner>
+ <ID>03b14d14cb23421fe7400e872df6fd8969efbb32f4584207fa5e5cf5ae580ca8</ID>
+ <DisplayName>someone</DisplayName>
+ </Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
- <Key>file_key2</Key>
+ <Key>object_key2</Key>
<LastModified>2011-07-27T21:16:00.000Z</LastModified>
<ETag>"d1919d0a02a24aa16412301e9e9dfbe4"</ETag>
<Size>58144</Size>
+ <Owner>
+ <ID>03b14d14cb23421fe7400e872df6fd8969efbb32f4584207fa5e5cf5ae580ca8</ID>
+ <DisplayName>someone</DisplayName>
+ </Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
...
- </FileList>
+ </ListBucketResult>
-### file read
+### object read
-Returns an file or part of an file, streamed back via `callback` through `response_data`.
+Returns an object or part of an object, streamed back via `callback` through `response_data`.
If options
- file_read (container_name, file_key, options, callback)
+ file_read (bucket_name, object_key, options, callback)
`options` are parameters parsed from the request (mostly headers)
@@ -182,19 +212,29 @@ The following options cause the request to behave conditionally based on the mod
- `options.if-match`
- `options.if-none-match`
-### file create
+The following options are used to override response headers. The driver should return them verbatim in the `response_headers` parameter of the callback, and, if appropriate, also pass them on to the underlying storage service (like s3) so that they can have the desired effect in any HTTP proxy/cache intermediaries.
+
+- `options.response-content-type`
+- `options.response-content-language`
+- `options.response-expires`
+- `options.response-cache-control`
+- `options.response-content-disposition`
+- `options.response-content-encoding`
+
-Creates or replaces an file with a particular file_key. If multiple concurrent requests occur, the last request to finish will win.
+### object create
- file_create (container_name, file_key, options, metadata, data_stream, callback)
+Creates or replaces an object with a particular object_key. If multiple concurrent requests occur, the last request to finish will win.
+
+ file_create (bucket_name, object_key, options, metadata, data_stream, callback)
`options` are parameters parsed from the request (mostly headers)
-- `options.content-md5` : this is the base64 encoding of the md5 hash (unlike ETag which is in hex). If this is present the creation of the file will be conditional the value of this option matching the md5 hash of the incoming data stream.
+- `options.content-md5` : this is the base64 encoding of the md5 hash (unlike ETag which is in hex). If this is present the creation of the object will be conditional the value of this option matching the md5 hash of the incoming data stream.
`metadata` are additional names/values stored together with the blob and returned as headers in an file_read request
-- `metadata.x-blb-meta-*` : user-defined metadata
+- `metadata.x-amz-meta-*` : user-defined metadata
- `metadata.content-type`
- `metadata.content-length` : this header is required
- `metadata.cache-control`
@@ -203,18 +243,26 @@ Creates or replaces an file with a particular file_key. If multiple concurrent r
- `metadata.expires`
-### file copy
+### object copy
+
+Copies an object to another object with a different key, OR replaces the metadata on an object. This operation does not have an input stream.
+
+ file_copy (bucket_name, object_key, source_bucket_name, source_object_key, options, metadata, callback)
+
+`source_bucket_name` and `source_object_key` specify the source, which may be the same as the `bucket_name` and `object_key`
-Copies an file to another file with a different key, OR replaces the metadata on an file. This operation does not have an input stream.
+`options.x-amz-metadata-directive` is optional and has either value `COPY` or `REPLACE`. When copying an object to itself, the value must be `REPLACE`, which causes existing metadata stored with the object to be replaced with new metadata from `metadata`. If the value is `COPY` metadata is copied from the existing object instead of coming from the request.
- file_copy (container_name, file_key, source_container_name, source_file_key, options, metadata, callback)
+The following options cause the request to behave conditionally based on the modification date or ETag (MD5 hash) just like the corresponding headers on file_create.
-`source_container_name` and `source_file_key` specify the source, which may be the same as the `container_name` and `file_key`
+- `options.x-amz-copy-source-if-modified-since`
+- `options.x-amz-copy-source-if-unmodified-since`
+- `options.x-amz-copy-source-if-match`
+- `options.x-amz-copy-source-if-none-match`
-`options.x-blb-metadata-copy-or-replace` is optional and has either value `COPY` or `REPLACE`. When copying an file to itself, the value must be `REPLACE`, which causes existing metadata stored with the file to be replaced with new metadata from `metadata`. If the value is `COPY` metadata is copied from the existing file instead of coming from the request.
-### file delete
+### object delete
- file_delete (container_name, file_key, callback)
+ file_delete (bucket_name, object_key, callback)
-Deletes an file
+Deletes an object
View
352 doc/blob service rest api.md
@@ -1,26 +1,57 @@
# Blob Service REST API
-Copyright (c) 2011 VMware, Inc.
+Copyright (c) 2011-2012 VMware, Inc.
-This document describes the REST api supported by the [Blob service] for accessing files and containers from Cloud Foundry applications. Please refer to the project readme.md if you are looking for instructions on how to deploy and use the Blob service as a standalone process without using the Cloud Foundry infrastructure.
+This document describes the REST api supported by the [Blob service] for accessing objects and buckets from Cloud Foundry applications. Please refer to the project readme.md if you are looking for instructions on how to deploy and use the Blob service as a standalone process without using the Cloud Foundry infrastructure.
## Introduction
+The Blob service has been designed to offer a useful subset of the S3 API with maximal compatibility with S3 including XML elements and error codes.
+
This release is intended for applications running inside a Cloud Foundry environment like cloudfoundry.com or Micro Cloud Foundry. The service cannot currently be reached directly from the Internet.
-There is currently 1 provider of storage for the Blob service: file system. These are configured as drivers in the Blob service. Only one of the drivers can currently be active in a single deployed instance of the service. Requests to the Blob service are translated into operations on the underlying storage via the installed driver.
+There are currently 2 providers of storage for the Blob service: S3 or file system. These are configured as drivers in the Blob service. Only one of the drivers can currently be active in a single deployed instance of the service. Requests to the Blob service are translated into operations on the underlying storage via the installed driver.
+
+NOTE: Much of the information in this document is very similar to the respective portions of the S3 API, documented on the [Amazon Web Services website for S3](http://docs.amazonwebservices.com/AmazonS3/latest/API/),
+
+#### Supported S3 features
+The following core features of the S3 API are supported. More details and examples are provided below.
+
+- REST api addressing using `http://<endpoint>/<bucket-name>/<object-key>` syntax
+- bucket operations (create, delete, list)
+- object operations (create using PUT, GET, GET range, HEAD, replace using PUT, DELETE, copy using PUT, list using GET on bucket)
+- metadata (timestamps, etags, predefined headers, user-defined headers)
+- conditional operations (based on etag or date)
-#### Request Authentication
-Requests have to be authenticated using basic or digest http access authentication. Keys for signing requests will be provisioned by Cloud Foundry and injected into the environment of Applications just like the service endpoint. For authentication implementation, please refer to RFC2617.
+#### Unsupported S3 features
+The following S3 API features are currently unsupported
-#### REST endpoint and container names
+- virtual-host style buckets using `http://<bucket-name>.<endpoint>/<object-key>` syntax
+- https
+- multiple cloud-foundry locations in a single instance of the FS driver
+- multiple amazon aws locations with a single instance of the S3 driver
+- POST requests to upload objects
+- ACLs on objects or ACL policies on buckets
+- logging or notifications
+- object versions
+- multi-part uploads
+- requestor-pays buckets
+- "website" bucket configuration (default and error docs)
+- torrent api
+- SOAP api
- http://<service-endpoint>/<container-name>/...
+#### Request signing
+Requests have to be signed using the same method as S3 requests. Keys for signing requests will be provisioned by Cloud Foundry and injected into the environment of Applications just like the service endpoint. For details on signing requests see the [aws developer documentation](http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html).
-The Blob service does *not* enforce uniqueness of container names across users. Since each Cloud Foundry user can deploy and bind to their own instance of the Blob service, this restriction is not necessary, even when the service is enhanced to support "virtual host" style endpoints.
+#### REST endpoint and bucket names
+Libraries written to work with S3 should be usable as long they can be directed to the Blob service endpoint (host:port) as provisioned by Cloud Foundry. This information will be made accessible to applications in Cloud Foundry via their environment. The Blob service does not support "virtual host" style endpoints with bucket names in the `Host` header. Bucket names are always specified in the query path of the URL. E.g.
-For interoperability, the Blob service restricts container names as follows:
+ http://<service-endpoint>/<bucket-name>/...
-- no capital letters
+The Blob service does *not* enforce uniqueness of bucket names across users. Since each Cloud Foundry user can deploy and bind to their own instance of the Blob service, this restriction is not necessary, even when the service is enhanced to support "virtual host" style endpoints.
+
+For interoperability, the Blob service restricts bucket names following the DNS-compatible recommendations in the [S3 guidelines for bucket names](http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html).
+
+- no capital letters (to be able to use virtual host)
- starting with lower case letters or numbers
- 3 ~ 63 chars
- no "_"
@@ -30,14 +61,38 @@ For interoperability, the Blob service restricts container names as follows:
- no IP address
- no "-" at end
-## Working with containers
-### List containers
+#### Additional interfaces
+The following non-s3 API has been provided for configuration purposes.
+
+ PUT /~s3/config
+
+This PUT provides a way for users to install their s3 credentials into a running blob service gateway. The payload of the PUT must be valid JSON of the form:
+
+ {
+ "key": "<s3-key>",
+ "secret": "<s3-secret>"
+ }
+
+After provisioning a new blob service instance with the s3 driver in Cloud Foundry, users will see the following 403 error on all bucket and object requests until they provide their s3 credentials through the /~s3/config interface.
+
+ <?xml version="1.0"?>
+ <Error>
+ <Code>InvalidAccessKeyId</Code>
+ <Message>The AWS Access Key Id you provided does not exist in our records.</Message>
+ <RequestId>23C339093A762DB9</RequestId>
+ <HostId>ITsnu6iASQP/ZnI7tJ+46Pq2Kjq1lXHOy2NDQq7mTer0A9FdMjTC3knd1dU4Zf6c</HostId>
+ <AWSAccessKeyId>dummy</AWSAccessKeyId>
+ </Error>
+
+## Working with buckets
+
+### List buckets
GET / HTTP/1.1
Host: localhost:3000
Date: Thu, 18 Aug 2011 15:38:02 +0000
- Authorization: <Basic or Digest>
+ Authorization: AWS jleschner:/70rBY2QIhk761qyIizKc9TTOzE=
Sample response _(xml indented for readability)_
@@ -48,38 +103,71 @@ Sample response _(xml indented for readability)_
Server: blob gw
<?xml version="1.0" encoding="UTF-8"?>
- <ContainerList xmlns="https://github.com/vmware-bdc/vblob/">
- <Containers>
- <Container>
- <Name>Container1</Name>
+ <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Owner>
+ <ID>..</ID>
+ <DisplayName>...</DisplayName>
+ </Owner>
+ <Buckets>
+ <Bucket>
+ <Name>Bucket1</Name>
<CreationDate>2011-10-01T01:20:36.000Z</CreationDate>
- </Container>
- <Container>
- <Name>Container2</Name>
+ </Bucket>
+ <Bucket>
+ <Name>Bucket2</Name>
<CreationDate>2011-10-01T01:20:37.000Z</CreationDate>
- </Container>
- </Containers>
- </ContainerList>
+ </Bucket>
+ </Buckets>
+ </ListAllMyBucketsResult>
+
+
+### Create bucket
+ PUT http://<service-endpoint>/<bucket-name>
-### Create container
+This will create a new bucket assuming that it does not already exist. If the bucket already exists for the current user, success is returned (the request is idempotent).
- PUT http://<service-endpoint>/<container-name>
+With the S3 driver if the bucket already exists in another S3 account, a failure response is returned (see below). This constraint does not apply to the FS driver.
-This will create a new container assuming that it does not already exist. If the container already exists for the current user, success is returned (the request is idempotent).
+Payloads such as XML for `<CreateBucketConfiguration>...` are currently ignored also for the S3 driver. Users should access S3 directly (e.g. via the console to manage buckets in different locations).
+For more details see the [aws documentation](http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketPUT.html)
+
+Other request headers, in particuler `x-amz-acl` are not currently passed through to S3 either.
+
Sample response success:
HTTP/1.1 200 OK
Connection: close
date: Thu Aug 18 2011 08:55:47 GMT-0700 (PDT)
Server: FS
+ x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
content-length: 0
- location: /container3
+ location: /bucket3
+
+Sample response failure from S3: Bucket name not available
-### Delete container
+ HTTP/1.1 409 Conflict
+ x-amz-request-id: 9B66FD00A584C98B
+ x-amz-id-2: P2LaCh+MRxGSDvS4x7/vCQ/xtHTzwnHGATYvxAtRYTKJcQi12djDO4v9aFh9rfTI
+ content-type: application/xml
+ date: Thu, 18 Aug 2011 16:28:48 GMT
+ server: AmazonS3
+ Connection: close
+
+ <?xml version="1.0" encoding="UTF-8"?>
+ <Error>
+ <Code>BucketAlreadyExists</Code>
+ <Message>The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.</Message>
+ <BucketName>www.photal.com</BucketName>
+ <RequestId>9B66FD00A584C98B</RequestId>
+ <HostId>P2LaCh+MRxGSDvS4x7/vCQ/xtHTzwnHGATYvxAtRYTKJcQi12djDO4v9aFh9rfTI</HostId>
+ </Error>
- DELETE http://<service-endpoint>/<container-name>
+### Delete bucket
+
+ DELETE http://<service-endpoint>/<bucket-name>
Sample response (success)
@@ -87,20 +175,39 @@ Sample response (success)
Connection: close
date: Thu Aug 18 2011 09:11:07 GMT-0700 (PDT)
Server: FS
+ x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
-Sample no such container response from FS driver
+Sample response: failure, no such bucket _(XML indented for readability)_
+
+ HTTP/1.1 404 Not Found
+ Connection: close
+ Content-Type: application/xml
+ Date: Thu, 18 Aug 2011 16:16:04 GMT
+ Server: blob gw
<?xml version="1.0" encoding="UTF-8"?>
<Error>
- <Code>NoSuchContainer</Code>
- <Message>No such container on disk</Message>
+ <Code>NoSuchBucket</Code>
+ <Message>The specified bucket does not exist</Message>
+ <BucketName>jltest6</BucketName>
+ <RequestId>40DF6D9131B0F8F9</RequestId>
+ <HostId>luisdxWwZiBt7grFTbDFFIqlVYuxoggdLI5tDQ+l0qqhR4uaA6I5+nTJ2dnlCzf0</HostId>
</Error>
+Sample no such bucket response from FS driver
+
+ <?xml version="1.0" encoding="UTF-8"?>
+ <Error>
+ <Code>NoSuchBucket</Code>
+ <Message>No such bucket on disk</Message>
+ </Error>
-### Listing keys of files in one container
-Containers may contain many files. Enumerating the files in a Container returns an alphabetically sorted list of keys and additional metadata about each item.
- GET http://<service-endpoint>/<container-name>
+### Listing keys of objects in one bucket
+Buckets may contain many objects. Enumerating the objects in a Bucket returns an alphabetically sorted list of keys and additional metadata about each item. This feature is currently unsupported by the FS driver.
+
+ GET http://<service-endpoint>/<bucket-name>
Sample response
@@ -110,11 +217,13 @@ Sample response
content-type: application/xml
date: Thu Aug 18 2011 10:36:58 GMT-0700 (PDT)
Server: FS
+ x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
Transfer-Encoding: chunked
<?xml version="1.0" encoding="UTF-8"?>
- <FileList xmlns="https://github.com/vmware-bdc/vblob/">
- <Name>container1</Name>
+ <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Name>bucket1</Name>
<Prefix/>
<Marker/>
<MaxKeys>1000</MaxKeys>
@@ -124,6 +233,7 @@ Sample response
<LastModified>Thu Aug 18 2011 10:28:54 GMT-0700 (PDT)</LastModified>
<ETag>"9fff58b7a9575dea85e7ca6ddbe31125"</ETag>
<Size>60421</Size>
+ <Owner/>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
@@ -131,11 +241,12 @@ Sample response
<LastModified>Thu Aug 18 2011 10:35:57 GMT-0700 (PDT)</LastModified>
<ETag>"9fff58b7a9575dea85e7ca6ddbe31125"</ETag>
<Size>60421</Size>
+ <Owner/>
<StorageClass>STANDARD</StorageClass>
</Contents>
- </FileList>
+ </ListBucketResult>
-When there are more than 1000 files, multiple requests need to be made, each returning up to 1000 keys. Requests are parameterized with:
+When there are more than 1000 objects, multiple requests need to be made, each returning up to 1000 keys. Requests are parameterized with:
- `marker`: item after which to start enumerating
- `maxkeys`: request for no more than this number of items
@@ -145,28 +256,48 @@ For simulating hierarchies of items, the following parameters are supported
- `prefix`: restricts results to keys matching the prefix string
- `delimiter`: (typically '/'), returns a deduplicated list of substrings in keys satisfying both prefix and delimiter (in addition to keys with just prefix)
-## File operations
+for more details see the [S3 documentation on enumerating](http://docs.amazonwebservices.com/AmazonS3/latest/dev/ListingKeysUsingAPIs.html).
+
+### Unsupported bucket operations
+
+The following request parameters are not currently supported by the Blob service. In S3, these are specified as special parameters after the ? in the URL for REST requests and most of the operations use an XML document for both GET and PUT. The exception on S3 is ?policy documents which are transmitted as JSON.
-Files are identified by keys which are specified in the request after the container name.
+The current implementation of the Blob service, ignores these parameters and treats requests which have these parameters as bucket list or bucket creation requests.
+
+- `?acl`
+- `?policy`
+- `?location`
+- `?logging`
+- `?notification`
+- `?requestPayment`
+- `?uploads`
+- `?versioning`
+- `?versions`
+- `?website`
+
+## Object operations
+
+Objects are identified by keys which are specified in the request after the bucket name.
- Creation dates are sizes are automatically maintained by the system.
-- Additional metadata can be stored with each file using extra headers.
+- Additional metadata can be stored with each object using extra headers.
-### Read File
+### Read Object
-Files can be read by a client via the GET method
+Objects can be read by a client via the GET method
- GET http://<service-endpoint>/<container-name>/<file-key>
+ GET http://<service-endpoint>/<bucket-name>/<object-key>
Sample exchange
- curl -v http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -v http://localhost:3000/bucket1/foo.txt
- > GET /container1/foo.txt HTTP/1.1
+ > GET /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Thu, 18 Aug 2011 17:42:40 +0000
+ > Authorization: AWS jleschner:m/jcNUupkajVuffsWItil0VFbII=
< HTTP/1.1 200 OK
< X-Powered-By: Express
@@ -174,6 +305,8 @@ Sample exchange
< content-type: binary/octet-stream
< date: Thu Aug 18 2011 10:42:40 GMT-0700 (PDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
< Content-Length: 12
< Last-Modified: Thu Aug 18 2011 10:41:53 GMT-0700 (PDT)
< ETag: 6f5902ac237024bdd0c176cb93063dc4
@@ -183,13 +316,14 @@ Sample exchange
For partial reads, standard HTTP range headers are supported. E.g.
- curl -v -H "Range: bytes=0-3" http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -v -H "Range: bytes=0-3" http://localhost:3000/bucket1/foo.txt
- > GET /container1/foo.txt HTTP/1.1
+ > GET /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 15:19:35 +0000
+ > Authorization: AWS jleschner:hrspG1TMaWtyZjbdlkCJj8rJL9M=
> Range: bytes=0-3
< HTTP/1.1 206 Partial Content
@@ -198,6 +332,8 @@ For partial reads, standard HTTP range headers are supported. E.g.
< content-type: binary/octet-stream
< date: Mon Sep 12 2011 11:19:35 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
< Content-Length: 4
< Last-Modified: Mon Sep 12 2011 11:17:51 GMT-0400 (EDT)
< ETag: 7c12772809c1c0c3deda6103b10fdfa0
@@ -206,17 +342,26 @@ For partial reads, standard HTTP range headers are supported. E.g.
1234
+It is possible to override some of the reponse headers by including the following special headers in a request.
+
+- `response-content-type`
+- `response-content-length`
+- `response-cache-control`
+- `response-content-disposition`
+- `response-content-encoding`
+- `response-expires`
+
##### Conditional GET
-The following HTTP headers are supported for retrieving files conditionally:
+The following HTTP headers are supported for retrieving objects conditionally:
-- `if-modified-since` : will validate that the modification date of the file is more recent than the specified date
-- `if-unmodified-since` : will validate that the modification date of the file is older
-- `if-match` : will validate that the file's ETag matches the specified value
-- `if-none-match` : will validate that the file's ETag value does NOT match
+- `if-modified-since` : will validate that the modification date of the object is more recent than the specified date
+- `if-unmodified-since` : will validate that the modification date of the object is older
+- `if-match` : will validate that the object's ETag matches the specified value
+- `if-none-match` : will validate that the object's ETag value does NOT match
E.g.
- curl -H "If-Match:bogus-etag" http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -H "If-Match:bogus-etag" http://localhost:3000/bucket1/foo.txt
<?xml version="1.0" encoding="UTF-8"?>
<Error>
@@ -226,13 +371,14 @@ E.g.
and
- curl -H "If-None-Match:7c12772809c1c0c3deda6103b10fdfa0" -v http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -H "If-None-Match:7c12772809c1c0c3deda6103b10fdfa0" -v http://localhost:3000/bucket1/foo.txt
- > GET /container1/foo.txt HTTP/1.1
+ > GET /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 17:48:26 +0000
+ > Authorization: AWS jleschner:3JrxOmoaxTOQnUSGRWGiE3qYH0M=
> If-None-Match:7c12772809c1c0c3deda6103b10fdfa0
< HTTP/1.1 304 Not Modified
@@ -241,16 +387,19 @@ and
< content-type: application/xml
< date: Mon Sep 12 2011 13:48:26 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
and
- curl -H "If-Modified-Since:Mon Sep 12 2011 11:17:50 GMT-0400 (EDT)" -v http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -H "If-Modified-Since:Mon Sep 12 2011 11:17:50 GMT-0400 (EDT)" -v http://localhost:3000/bucket1/foo.txt
- > GET /container1/foo.txt HTTP/1.1
+ > GET /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 17:44:41 +0000
+ > Authorization: AWS jleschner:ifmwb6RGWPIPiWKoLLHjB+lB9rM=
> If-Modified-Since:Mon Sep 12 2011 11:17:50 GMT-0400 (EDT)
< HTTP/1.1 200 OK
@@ -259,6 +408,8 @@ and
< content-type: binary/octet-stream
< date: Mon Sep 12 2011 13:44:42 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
< Content-Length: 11
< Last-Modified: Mon Sep 12 2011 11:17:51 GMT-0400 (EDT)
< ETag: 7c12772809c1c0c3deda6103b10fdfa0
@@ -268,13 +419,14 @@ and
but
- curl -H "If-Unmodified-Since:Mon Sep 12 2011 11:17:50 GMT-0400 (EDT)" -v http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -H "If-Unmodified-Since:Mon Sep 12 2011 11:17:50 GMT-0400 (EDT)" -v http://localhost:3000/bucket1/foo.txt
- > GET /container1/foo.txt HTTP/1.1
+ > GET /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 17:50:25 +0000
+ > Authorization: AWS jleschner:zZTQy1DrlL4fYAgGM5Tsh6L/cgQ=
> If-Unmodified-Since:Mon Sep 12 2011 11:17:50 GMT-0400 (EDT)
< HTTP/1.1 412 Precondition Failed
@@ -283,6 +435,8 @@ but
< content-type: application/xml
< date: Mon Sep 12 2011 13:50:25 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
< Transfer-Encoding: chunked
<?xml version="1.0" encoding="UTF-8"?>
@@ -293,13 +447,14 @@ but
and
- curl -v -H "If-Modified-Since:Mon Sep 12 2011 11:17:51 GMT-0400 (EDT)" http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -v -H "If-Modified-Since:Mon Sep 12 2011 11:17:51 GMT-0400 (EDT)" http://localhost:3000/bucket1/foo.txt
- > GET /container1/foo.txt HTTP/1.1
+ > GET /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 15:31:03 +0000
+ > Authorization: AWS jleschner:rKacAr+fyx2ZLryFcu3udbsafn0=
> If-Modified-Since:Mon Sep 12 2011 11:17:51 GMT-0400 (EDT)
< HTTP/1.1 304 Not Modified
@@ -308,18 +463,21 @@ and
< content-type: application/xml
< date: Mon Sep 12 2011 11:31:03 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
### HEAD requests
HEAD requests are identical to GET requests except they never return a response payload. Note that the response Content-Length header is set to the length of the stored content even though no content is actually included.
- curl -v -X HEAD http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -v -X HEAD http://localhost:3000/bucket1/foo.txt
- > HEAD /container1/foo.txt HTTP/1.1
+ > HEAD /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 17:58:28 +0000
+ > Authorization: AWS jleschner:mr1YmAf0U8ohzYhQyuqd1XZKiLE=
< HTTP/1.1 200 OK
< X-Powered-By: Express
@@ -327,30 +485,33 @@ HEAD requests are identical to GET requests except they never return a response
< content-type: binary/octet-stream
< date: Mon Sep 12 2011 13:58:28 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
< Content-Length: 11
< Last-Modified: Mon Sep 12 2011 11:17:51 GMT-0400 (EDT)
< ETag: 7c12772809c1c0c3deda6103b10fdfa0
< Accept-Ranges: bytes
-### Create or Replace or Copy file
+### Create or Replace or Copy object
-Files are created, replaced, or copied with a PUT request. If the file-key already exists, the service will replace the existing file with the new one.
+Objects are created, replaced, or copied with a PUT request. If the object-key already exists, the service will replace the existing object with the new one.
-PUT operations are all-or-nothing. There is currently no support for partial writes, or resumable writes. Concurrent PUTs with the same key result in a single winner. Currently there is no versioning support for catching multiple files created with the same key.
+PUT operations are all-or-nothing. There is currently no support for partial writes, or resumable writes. Concurrent PUTs with the same key result in a single winner. Currently there is no versioning support for catching multiple objects created with the same key.
PUT requests must have a content-length header. There is no support for streaming uploads without predefined length.
- PUT http://<service-endpoint>/<container-name>/<file-key>
+ PUT http://<service-endpoint>/<bucket-name>/<object-key>
Sample exchange
- curl -v -X PUT -T foo.txt http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -v -X PUT -T foo.txt http://localhost:3000/bucket1/foo.txt
- > PUT /container1/foo.txt HTTP/1.1
+ > PUT /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 15:17:51 +0000
+ > Authorization: AWS jleschner:M/CW8hMXjdpzcn8zpz5GCgzhD/I=
> Content-Length: 11
> Expect: 100-continue
@@ -360,14 +521,16 @@ Sample exchange
< Connection: close
< date: Mon Sep 12 2011 11:17:51 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
< ETag: 7c12772809c1c0c3deda6103b10fdfa0
< content-length: 0
If the request includes an (optional) `content-md5` header, the value of this header is compared to the MD5 hash of the blob for integrity.
-The following optional headers in a PUT requests result in metadata being stored with the file, for retrieval in future GET requests.
+The following optional headers in a PUT requests result in metadata being stored with the object, for retrieval in future GET requests.
-- x-blb-meta-? : for user defined metadata
+- x-amz-meta-? : for user defined metadata
- cache-control : standard HTTP
- content-disposition : standard HTTP
- content-encoding : standard HTTP
@@ -375,18 +538,21 @@ The following optional headers in a PUT requests result in metadata being stored
- expires : standard HTTP
- content-type : standard HTTP
-Using a special header `x-blb-copy-from` allows the PUT to make a copy of an existing file, with a different key say or different metadata, or in a different container, without re-submitting the actual blob.
+Note that the `x-amz-acl` header is not currently supported.
+
+Using a special header `x-amz-copy-source` allows the PUT to make a copy of an existing object, with a different key say or different metadata, or in a different bucket, without re-submitting the actual blob.
E.g.
- curl -v -X PUT -H "x-blb-copy-from: /container1/foo.txt" http://localhost:3000/container1/foo2.txt
+ $ ./s3curl.pl --id localtest -- -v -X PUT -H "x-amz-copy-source: /bucket1/foo.txt" http://localhost:3000/bucket1/foo2.txt
- > PUT /container1/foo2.txt HTTP/1.1
+ > PUT /bucket1/foo2.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Mon, 12 Sep 2011 20:34:21 +0000
- > x-blb-copy-from: /container1/foo.txt
+ > Authorization: AWS jleschner:fB5qMWnSVI/Ws6SkxkaSFdoZ4y8=
+ > x-amz-copy-source: /bucket1/foo.txt
< HTTP/1.1 200 OK
< X-Powered-By: Express
@@ -394,35 +560,40 @@ E.g.
< content-type: application/xml
< date: Mon Sep 12 2011 16:34:21 GMT-0400 (EDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
< ETag: 7c12772809c1c0c3deda6103b10fdfa0
< content-length:
<?xml version="1.0" encoding="UTF-8"?>
- <FileCopy xmlns="https://github.com/vmware-bdc/vblob/">
+ <CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<LastModified>2011-09-12T20:31:35.000Z</LastModified>
<ETag>&quot;7c12772809c1c0c3deda6103b10fdfa0&quot;</ETag>
- </FileCopy>
+ </CopyObjectResult>
-### Delete file
+### Delete object
- DELETE http://<service-endpoint>/<container-name>/<file-key>
+ DELETE http://<service-endpoint>/<bucket-name>/<object-key>
Sample exchange (successful)
- curl -X DELETE -v http://localhost:3000/container1/foo.txt
+ $ ./s3curl.pl --id localtest -- -X DELETE -v http://localhost:3000/bucket1/foo.txt
- > DELETE /container1/foo.txt HTTP/1.1
+ > DELETE /bucket1/foo.txt HTTP/1.1
> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3
> Host: localhost:3000
> Accept: */*
> Date: Thu, 18 Aug 2011 19:32:21 +0000
+ > Authorization: AWS jleschner:xjno3GAuIpARe1vBKbCrTJnNU1M=
< HTTP/1.1 204 No Content
< X-Powered-By: Express
< Connection: close
< date: Thu Aug 18 2011 12:32:21 GMT-0700 (PDT)
< Server: FS
+ < x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ < x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
Sample response (error)
@@ -432,6 +603,8 @@ Sample response (error)
content-type: application/xml
date: Thu Aug 18 2011 12:33:53 GMT-0700 (PDT)
Server: FS
+ x-amz-request-id: 1D2E3A4D5B6E7E8F9
+ x-amz-id-2: 3F+E1E4B1D5A9E2DB6E5E3F5D8E9
content-length: 108
<?xml version="1.0" encoding="UTF-8"?>
@@ -439,3 +612,16 @@ Sample response (error)
<Code>NoSuchFile</Code>
<Message>No such file</Message>
</Error>
+
+## Unsupported Object operations
+
+The Blob service does not currently support any POST operations. These will be provided when the service is extended to support direct access via signed URLs from the public Internet. For more details see [aws POST doc](http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?RESTObjectPOST.html)
+
+In addition, the following GET, HEAD, and PUT request parameters are not currently supported by the Blob service.
+
+- `?acl` -- see [aws acl doc](http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?RESTObjectPUTacl.html)
+- `?versionID` -- see [aws versioning doc](http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?Versioning.html)
+- `?torrent` -- see [aws torrent doc](http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?RESTObjectGETtorrent.html)
+- `?uploads`, `&PartNumber` and `&UploadID` -- see [aws multi-part uploads doc](http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html)
+
+
View
127 drivers/fs/blob_fs.js
@@ -1,5 +1,5 @@
/*
-Copyright (c) 2011 VMware, Inc.
+Copyright (c) 2011-2012 VMware, Inc.
Author: wangs@vmware.com
*/
var fs = require('fs');
@@ -83,7 +83,8 @@ function start_compactor(option,fb)
{
var node_exepath = option.node_exepath ? option.node_exepath : "node";
var ec_exepath = option.ec_exepath ? option.ec_exepath : "drivers/fs/fs_ec.js";
- var ec_interval = option.ec_interval ? option.ec_interval : "1500";
+ var ec_interval;
+ try { if (isNaN(ec_interval = parseInt(option.ec_interval,10))) throw 'isNaN'; } catch (err) { ec_interval = 1500; }
var ec_status = 0;
fb.ecid = setInterval(function() {
if (ec_status === 1) return; //already a gc process running
@@ -92,7 +93,7 @@ function start_compactor(option,fb)
function(error,stdout, stderr) {
ec_status = 0; //finished set to 0
} );
- }, parseInt(ec_interval,10));
+ }, ec_interval);
}
@@ -103,10 +104,13 @@ function start_gc(option,fb)
var node_exepath = option.node_exepath ? option.node_exepath : "node";
var gc_exepath = option.gc_exepath ? option.gc_exepath : "drivers/fs/fs_gc.js";
var gcfc_exepath = option.gcfc_exepath ? option.gcfc_exepath : "drivers/fs/fs_gcfc.js";
- var gc_interval = option.gc_interval ? option.gc_interval : "600000";
- var gcfc_interval = option.gcfc_interval ? option.gcfc_interval : "1500";
+ var gc_interval;
+ var gcfc_interval;
+ var gctmp_interval;
+ try { if (isNaN(gc_interval = parseInt(option.gc_interval,10))) throw 'isNaN'; } catch (err) { gc_interval = 600000; }
+ try { if (isNaN(gcfc_interval = parseInt(option.gcfc_interval,10))) throw 'isNaN'; } catch (err) { gcfc_interval = 1500; }
+ try { if (isNaN(gctmp_interval = parseInt(option.gctmp_interval,10))) throw 'isNaN'; } catch (err) { gctmp_interval = 3600000; }
var gctmp_exepath = option.gctmp_exepath ? option.gctmp_exepath : "drivers/fs/fs_gctmp.js";
- var gctmp_interval = option.gctmp_interval ? option.gctmp_interval : "3600000";
fb.gcid = setInterval(function() {
if (gc_status === 1) return; //already a gc process running
gc_status = 1;
@@ -114,7 +118,7 @@ function start_gc(option,fb)
function(error,stdout, stderr) {
gc_status = 0; //finished set to 0
} );
- }, parseInt(gc_interval,10));
+ }, gc_interval);
//gc from cache
var gcfc_status = 0;
@@ -134,7 +138,7 @@ function start_gc(option,fb)
fs.unlink(tmp_fn,function() {} );
} );
});
- }, parseInt(gcfc_interval,10));
+ }, gcfc_interval);
//gc tmp
var gctmp_status = 0;
fb.gctmpid = setInterval(function() {
@@ -144,13 +148,13 @@ function start_gc(option,fb)
function(error,stdout, stderr) {
gctmp_status = 0; //finished set to 0
} );
- }, parseInt(gctmp_interval,10));
+ }, gctmp_interval);
}
function start_quota_gathering(fb)
{
fs.readdir(fb.root_path, function(err, dirs) {
- if (err) {
+ if (err) {
setTimeout(start_quota_gathering, 1000, fb);
return;
}
@@ -243,18 +247,18 @@ FS_blob.prototype.container_create = function(container_name,callback,fb)
{
fs.mkdirSync(c_path+"/"+ENUM_FOLDER,"0775");
}
- fs.writeFile(c_path+"/"+ENUM_FOLDER+"/base", "{}");
+ fs.writeFileSync(c_path+"/"+ENUM_FOLDER+"/base", "{}");
if (Path.existsSync(c_path+"/ts") === false) //double check ts
{
fb.logger.debug( ("timestamp "+c_path+"/ts does not exist. Need to create one"));
- fs.writeFile(c_path+"/ts", "DEADBEEF");
+ fs.writeFileSync(c_path+"/ts", "DEADBEEF");
} else
{
fb.logger.debug( ("timestamp "+c_path+"/ts exists!"));
}
} catch (err1) {
var resp = {};
- error_msg(500,"InternalError","Cannot create container because: "+err1,resp);
+ error_msg(500,"InternalError","Cannot create bucket because: "+err1,resp);
callback(resp.resp_code, resp.resp_header, resp.resp_body,null);
return;
}
@@ -302,7 +306,7 @@ FS_blob.prototype.container_delete = function(container_name,callback,fb)
);
} else {
var resp = {};
- error_msg(409,"ContainerNotEmpty","The container you tried to delete is not empty.",resp);
+ error_msg(409,"BucketNotEmpty","The bucket you tried to delete is not empty.",resp);
resp_code = resp.resp_code; resp_header = resp.resp_header; resp_body = resp.resp_body;
callback(resp_code, resp_header, resp_body, null);
}
@@ -325,7 +329,7 @@ function container_exists(container_name, callback,fb)
if (!Path.existsSync(c_path)) {
fb.logger.error( ("no such container_name"));
var resp = {};
- error_msg(404,"NoSuchContainer","No such container on disk",resp);
+ error_msg(404,"NoSuchBucket","No such bucket on disk",resp);
resp_code = resp.resp_code; resp_header = resp.resp_header; resp_body = resp.resp_body;
callback(resp_code, resp_header, resp_body, null);
return false;
@@ -377,7 +381,7 @@ function remove_uploaded_file(fdir_path)
function create_prefix_folders(prefix_array, callback)
{
var resp = {};
- error_msg(404,"NoSuchContainer","Container does not exist.",resp);
+ error_msg(404,"NoSuchBucket","Bucket does not exist.",resp);
var path_pref = null;
for (var idx = 0; idx < prefix_array.length; idx++) {
if (path_pref === null) path_pref = prefix_array[idx];
@@ -405,8 +409,7 @@ FS_blob.prototype.file_create = function (container_name,filename,create_options
var c_path = this.root_path + "/" + container_name;
if (container_exists(container_name,callback,fb) === false) return;
//QUOTA
- if (this.quota && this.used_quota + parseInt(create_meta_data["content-length"],10) > this.quota ||
- this.obj_limit && this.obj_count >= this.obj_limit) {
+ if (this.quota && this.used_quota + parseInt(create_meta_data["content-length"],10) > this.quota || this.obj_limit && this.obj_count >= this.obj_limit) {
error_msg(500,"UsageExceeded","Usage will exceed the quota",resp);
callback(resp.resp_code, resp.resp_header, resp.resp_body, null);
return;
@@ -498,7 +501,7 @@ FS_blob.prototype.file_create = function (container_name,filename,create_options
var keys = Object.keys(create_meta_data);
for (var idx = 0; idx < keys.length; idx++) {
var obj_key = keys[idx];
- if (obj_key.match(/^x-blb-meta-/i)) {
+ if (obj_key.match(/^x-amz-meta-/i)) {
var sub_key = obj_key.substr(11);
sub_key = "vblob_meta_" + sub_key;
opts[sub_key] = create_meta_data[obj_key];
@@ -575,7 +578,7 @@ FS_blob.prototype.file_create_meta = function (container_name, filename, temp_pa
if (err) {
fb.logger.error( ("In creating file "+filename+" meta in container_name "+container_name+" "+err));
if (resp !== null) {
- error_msg(404,"NoSuchContainer",err,resp);
+ error_msg(404,"NoSuchBucket",err,resp);
callback(resp.resp_code, resp.resp_header, resp.resp_body, null);
}
fs.unlink(temp_path,function(err) {});
@@ -588,7 +591,7 @@ FS_blob.prototype.file_create_meta = function (container_name, filename, temp_pa
resp.resp_code = 200; resp.resp_body = null;
fb.logger.debug( ('is_copy: ' + is_copy));
if (is_copy) {
- resp.resp_body = {"FileCopy":{"LastModified":new Date(doc.vblob_update_time).toISOString(),"ETag":'"'+opt.vblob_file_etag+'"'}};
+ resp.resp_body = {"CopyObjectResult":{"LastModified":new Date(doc.vblob_update_time).toISOString(),"ETag":'"'+opt.vblob_file_etag+'"'}};
resp.resp_header = header;
} else {
resp.resp_header = header;
@@ -610,7 +613,8 @@ FS_blob.prototype.file_create_meta = function (container_name, filename, temp_pa
if (!gc_hash[container_name][doc.vblob_file_fingerprint]) gc_hash[container_name][doc.vblob_file_fingerprint] = {ver:[doc.vblob_file_version], fn:doc.vblob_file_name}; else gc_hash[container_name][doc.vblob_file_fingerprint].ver.push(doc.vblob_file_version);
//step 6 mv to versions
var prefix1 = doc.vblob_file_version.substr(0,PREFIX_LENGTH), prefix2 = doc.vblob_file_version.substr(PREFIX_LENGTH,PREFIX_LENGTH);
- fs.rename(temp_path, fb.root_path + "/"+container_name+"/versions/" + prefix1 + "/" + prefix2 + "/" + doc.vblob_file_version,function (err) {
+ //link to version, so version link > 1, gc won't remove it at this point
+ fs.link(temp_path, fb.root_path + "/"+container_name+"/versions/" + prefix1 + "/" + prefix2 + "/" + doc.vblob_file_version,function (err) {
if (err) {
fb.logger.error( ("In creating file "+filename+" meta in container_name "+container_name+" "+err));
if (resp !== null) {
@@ -622,6 +626,8 @@ FS_blob.prototype.file_create_meta = function (container_name, filename, temp_pa
//step 7 ln -f meta/key versions/version_id
var child = exec('ln -f '+fb.root_path + "/"+container_name+"/versions/" + prefix1 + "/" + prefix2 + "/" + doc.vblob_file_version+" "+ fb.root_path + "/"+container_name+"/meta/" + prefix1 + "/" + prefix2 + "/" + doc.vblob_file_fingerprint,
function (error, stdout, stderr) {
+ //now we can remove temp
+ fs.unlink(temp_path,function(err) {});
//step 8 respond
callback(resp.resp_code, resp.resp_header, resp.resp_body,null);
}
@@ -687,19 +693,32 @@ FS_blob.prototype.file_copy = function (container_name,filename,source_container
var blob_path = c_path + "/blob/" + prefix_path + version_id;
var src_meta_path = src_path + "/meta/" + src_prefix_path + src_key_fingerprint;
+ var etag_match=null, etag_none_match=null, date_modified=null, date_unmodified=null;
var meta_dir=null;
if (true){
var keys = Object.keys(options);
for (var idx = 0; idx < keys.length; idx++)
{
- if (keys[idx].match(/^x-blb-metadata-copy-or-replace$/i))
+ if (keys[idx].match(/^x-amz-copy-source-if-match$/i))
+ { etag_match = options[keys[idx]]; }
+ else if (keys[idx].match(/^x-amz-copy-source-if-none-match$/i))
+ { etag_none_match = options[keys[idx]]; }
+ else if (keys[idx].match(/^x-amz-copy-source-if-unmodified-since$/i))
+ { date_unmodified = options[keys[idx]]; }
+ else if (keys[idx].match(/^x-amz-copy-source-if-modified-since$/i))
+ { date_modified = options[keys[idx]]; }
+ else if (keys[idx].match(/^x-amz-metadata-directive$/i))
{ meta_dir = options[keys[idx]]; }
}
}
if (meta_dir === null) { meta_dir = 'COPY'; }
else { meta_dir = meta_dir.toUpperCase(); }
- if ((meta_dir !== 'COPY' && meta_dir !== 'REPLACE')) {
- error_msg(400,"NotImplemented","The headers are not supported",resp);
+ if ((meta_dir !== 'COPY' && meta_dir !== 'REPLACE') ||
+ (etag_match && date_modified) ||
+ (etag_none_match && date_unmodified) ||
+ (date_modified && date_unmodified) ||
+ (etag_match && etag_none_match) ) {
+ error_msg(400,"NotImplemented","The headers are not supported",resp); //same as S3 does
callback(resp.resp_code, resp.resp_header, resp.resp_body, null);
return;
}
@@ -718,7 +737,7 @@ FS_blob.prototype.file_copy = function (container_name,filename,source_container
var obj = JSON.parse(data);
//QUOTA
if (source_container !== container_name || source_file !== filename) {
- if (fb.quota && fb.used_quota + obj.vblob_file_size > fb.quota ||
+ if (fb.quota && fb.used_quota + obj.vblob_file_size > fb.quota ||
fb.obj_limit && fb.obj_count >= fb.obj_limit) {
error_msg(500,"UsageExceeded","Usage will exceed quota",resp);
callback(resp.resp_code, resp.resp_header, resp.resp_body, null);
@@ -726,6 +745,26 @@ FS_blob.prototype.file_copy = function (container_name,filename,source_container
}
}
if (true) {
+ //check etag, last modified
+ var check_modified = true;
+ var t1,t2;
+ if (date_modified) {
+ t1 = new Date(date_modified).valueOf();
+ t2 = new Date(obj.vblob_update_time).valueOf();
+ check_modified = t2 > t1 || t1 > new Date().valueOf();
+ } else if (date_unmodified) {
+ t1 = new Date(date_unmodified).valueOf();
+ t2 = new Date(obj.vblob_update_time).valueOf();
+ check_modified = t2 <= t1;
+ }
+ if ((etag_match && obj.vblob_file_etag !== etag_match) ||
+ (etag_none_match && obj.vblob_file_etag === etag_none_match) ||
+ check_modified === false)
+ {
+ error_msg(412,"PreconditionFailed","At least one of the preconditions you specified did not hold.",resp);
+ callback(resp.resp_code, resp.resp_header, resp.resp_body, null);
+ return;
+ }
var keys,keys2; var idx; //supress warning
var dest_obj = {};
//TODO: more meta to copy (cache-control, encoding, disposition, expires, etc.)
@@ -756,8 +795,8 @@ FS_blob.prototype.file_copy = function (container_name,filename,source_container
keys = Object.keys(metadata);
for (idx = 0; idx < keys.length; idx++) {
var key = keys[idx];
- if (key.match(/^x-blb-meta-/i)) {
- var key2 = key.replace(/^x-blb-meta-/i,"vblob_meta_");
+ if (key.match(/^x-amz-meta-/i)) {
+ var key2 = key.replace(/^x-amz-meta-/i,"vblob_meta_");
dest_obj[key2] = metadata[key];
} else if (!key.match(/^content-length/i)) dest_obj[key] = metadata[key];
}
@@ -851,7 +890,7 @@ FS_blob.prototype.file_read = function (container_name, filename, options, callb
if (modified_since === false ||
etag_none_match && etag_none_match === obj.vblob_file_etag)
{
- error_msg(304,'NotModified','The file is not modified',resp);
+ error_msg(304,'NotModified','The object is not modified',resp);
resp.resp_header.etag = obj.vblob_file_etag; resp.resp_header["last-modified"] = obj.vblob_update_time;
callback(resp.resp_code, resp.resp_header, /*resp.resp_body*/ null, null); //304 should not have body
return;
@@ -865,13 +904,22 @@ FS_blob.prototype.file_read = function (container_name, filename, options, callb
var obj_key = keys[idx];
if (obj_key.match(/^vblob_meta_/)) {
var sub_key = obj_key.substr(11);
- sub_key = "x-blb-meta-" + sub_key;
+ sub_key = "x-amz-meta-" + sub_key;
header[sub_key] = obj[obj_key];
} else if (obj_key.match(/^vblob_/) === null) {
//other standard attributes
header[obj_key] = obj[obj_key];
}
}
+ //override with response-xx
+ keys = Object.keys(options);
+ for (var idx2 = 0; idx2 < keys.length; idx2++) {
+ var obj_key2 = keys[idx2];
+ if (obj_key2.match(/^response-/)) {
+ var sub_key2 = obj_key2.substr(9);
+ header[sub_key2] = options[obj_key2];
+ }
+ }
header["Accept-Ranges"] = "bytes";
var st;
if (range !== null && range !== undefined) {
@@ -892,7 +940,7 @@ FS_blob.prototype.file_read = function (container_name, filename, options, callb
st = fs.createReadStream(c_path+"/"+obj.vblob_file_path, range);
st.on('error', function(err) {
st = null;
- error_msg(503,'SlowDown','The file is being updated too frequently, try later',resp);
+ error_msg(503,'SlowDown','The object is being updated too frequently, try later',resp);
callback(resp.resp_code, resp.resp_header, resp.resp_body, null);
});
st.on('open', function(fd) {
@@ -916,7 +964,7 @@ FS_blob.prototype.file_read = function (container_name, filename, options, callb
st.on('error', function(err) {//RETRY??
st = null;
fb.logger.error( ("file "+obj.vblob_file_version+" is purged by gc already!"));
- //error_msg(508,'SlowDown','The file is being updated too frequently, try later',resp);
+ //error_msg(508,'SlowDown','The object is being updated too frequently, try later',resp);
//callback(resp.resp_code, resp.resp_header, resp.resp_body, null);
setTimeout(function(fb1) { fb1.file_read(container_name, filename, options, callback,fb1); }, Math.floor(Math.random()*1000) + 100,fb);
});
@@ -964,7 +1012,8 @@ function query_files(container_name, options, callback, fb)
}
idx2 = low;
}
-