diff --git a/lib/api/apiUtils/bucket/parseWhere.js b/lib/api/apiUtils/bucket/parseWhere.js index 858976eea3..281697076e 100644 --- a/lib/api/apiUtils/bucket/parseWhere.js +++ b/lib/api/apiUtils/bucket/parseWhere.js @@ -31,6 +31,7 @@ */ const exprMapper = { '=': '$eq', + '!=': '$ne', '<>': '$ne', '>': '$gt', '<': '$lt', diff --git a/lib/api/apiUtils/bucket/validateSearch.js b/lib/api/apiUtils/bucket/validateSearch.js index fa942a11d0..c7285c5b5d 100644 --- a/lib/api/apiUtils/bucket/validateSearch.js +++ b/lib/api/apiUtils/bucket/validateSearch.js @@ -1,26 +1,49 @@ const Parser = require('sql-where-parser'); -const parser = new Parser(); const { errors } = require('arsenal'); const objModel = require('arsenal').models.ObjectMD; +const BINARY_OP = 2; +const sqlConfig = { + operators: [ + { + '=': BINARY_OP, + '<': BINARY_OP, + '>': BINARY_OP, + '<>': BINARY_OP, + '<=': BINARY_OP, + '>=': BINARY_OP, + '!=': BINARY_OP, + }, + { LIKE: BINARY_OP }, + { AND: BINARY_OP }, + { OR: BINARY_OP }, + ], + tokenizer: { + shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'], + shouldMatch: ['"', '\'', '`'], + shouldDelimitBy: [' ', '\n', '\r', '\t'], + }, +}; +const parser = new Parser(sqlConfig); + function _validateTree(whereClause, possibleAttributes) { let invalidAttribute; function _searchTree(node) { - const operator = Object.keys(node)[0]; - - if (operator === 'AND') { - _searchTree(node[operator][0]); - _searchTree(node[operator][1]); - } else if (operator === 'OR') { - _searchTree(node[operator][0]); - _searchTree(node[operator][1]); + if (typeof node !== 'object') { + invalidAttribute = node; } else { - const field = node[operator][0]; - - if (!possibleAttributes[field] && - !field.startsWith('x-amz-meta-')) { - invalidAttribute = field; + const operator = Object.keys(node)[0]; + if (operator === 'AND' || operator === 'OR') { + _searchTree(node[operator][0]); + _searchTree(node[operator][1]); + } else { + const field = node[operator][0]; + if (!field.startsWith('tags.') && + !possibleAttributes[field] && + !field.startsWith('x-amz-meta-')) { + invalidAttribute = field; + } } } } @@ -32,8 +55,8 @@ function _validateTree(whereClause, possibleAttributes) { * validateSearchParams - validate value of ?search= in request * @param {string} searchParams - value of search params in request * which should be jsu sql where clause - * For metadata: userMd.`x-amz-meta-color`=\"blue\" - * For tags: tags.`x-amz-meta-color`=\"blue\" + * For metadata: x-amz-meta-color=\"blue\" + * For tags: tags.x-amz-meta-color=\"blue\" * For any other attribute: `content-length`=5 * @return {undefined | error} undefined if validates or arsenal error if not */ @@ -43,9 +66,11 @@ function validateSearchParams(searchParams) { ast = parser.parse(searchParams); } catch (e) { if (e) { - return errors.InvalidArgument - .customizeDescription('Invalid sql where clause ' + - 'sent as search query'); + return { + error: errors.InvalidArgument + .customizeDescription('Invalid sql where clause ' + + 'sent as search query'), + }; } } const possibleAttributes = objModel.getAttributes(); diff --git a/lib/services.js b/lib/services.js index 29a5943d48..8292b468f6 100644 --- a/lib/services.js +++ b/lib/services.js @@ -101,6 +101,7 @@ const services = { const md = new ObjectMD(); // This should be object creator's canonical ID. md.setOwnerId(authInfo.getCanonicalID()) + .setKey(objectKey) .setCacheControl(cacheControl) .setContentDisposition(contentDisposition) .setContentEncoding(contentEncoding) diff --git a/package.json b/package.json index 46cb66d658..7f67e89f24 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js", "ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js", "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management", + "ft_search": "cd tests/functional/aws-node-sdk && mocha -t 90000 test/mdSearch", "install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7", "lint": "eslint $(git ls-files '*.js')", "lint_md": "mdlint $(git ls-files '*.md')", diff --git a/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js b/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js new file mode 100644 index 0000000000..e99a67ec35 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js @@ -0,0 +1,101 @@ +const s3Client = require('./utils/s3SDK'); +const { runAndCheckSearch, runIfMongo } = require('./utils/helpers'); + +const objectKey = 'findMe'; +const hiddenKey = 'leaveMeAlone'; +const objectTagData = 'item-type=main'; +const hiddenTagData = 'item-type=dessert'; +const userMetadata = { food: 'pizza' }; +const updatedUserMetadata = { food: 'cake' }; + +runIfMongo('Basic search', () => { + const bucketName = `basicsearchmebucket${Date.now()}`; + before(done => { + s3Client.createBucket({ Bucket: bucketName }, err => { + if (err) { + return done(err); + } + return s3Client.putObject({ Bucket: bucketName, Key: objectKey, + Metadata: userMetadata, Tagging: objectTagData }, err => { + if (err) { + return done(err); + } + return s3Client.putObject({ Bucket: bucketName, + Key: hiddenKey, Tagging: hiddenTagData }, done); + }); + }); + }); + + after(done => { + s3Client.deleteObjects({ Bucket: bucketName, Delete: { Objects: [ + { Key: objectKey }, + { Key: hiddenKey }], + } }, + err => { + if (err) { + return done(err); + } + return s3Client.deleteBucket({ Bucket: bucketName }, done); + }); + }); + + it('should list object with searched for system metadata', done => { + const encodedSearch = encodeURIComponent(`key="${objectKey}"`); + return runAndCheckSearch(s3Client, bucketName, + encodedSearch, objectKey, done); + }); + + it('should list object with searched for user metadata', done => { + const encodedSearch = + encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, + objectKey, done); + }); + + it('should list object with searched for tag metadata', done => { + const encodedSearch = + encodeURIComponent('tags.item-type="main"'); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, + objectKey, done); + }); + + it('should return empty listing when no object has user md', done => { + const encodedSearch = + encodeURIComponent('x-amz-meta-food="nosuchfood"'); + return runAndCheckSearch(s3Client, bucketName, + encodedSearch, null, done); + }); + + describe('search when overwrite object', () => { + before(done => { + s3Client.putObject({ Bucket: bucketName, Key: objectKey, + Metadata: updatedUserMetadata }, done); + }); + + it('should list object with searched for updated user metadata', + done => { + const encodedSearch = + encodeURIComponent('x-amz-meta-food' + + `="${updatedUserMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, + objectKey, done); + }); + }); +}); + +runIfMongo('Search when no objects in bucket', () => { + const bucketName = `noobjectbucket${Date.now()}`; + before(done => { + s3Client.createBucket({ Bucket: bucketName }, done); + }); + + after(done => { + s3Client.deleteBucket({ Bucket: bucketName }, done); + }); + + it('should return empty listing when no objects in bucket', done => { + const encodedSearch = encodeURIComponent(`key="${objectKey}"`); + return runAndCheckSearch(s3Client, bucketName, + encodedSearch, null, done); + }); +}); diff --git a/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js b/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js new file mode 100644 index 0000000000..9eb62caea8 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js @@ -0,0 +1,64 @@ +const assert = require('assert'); +const async = require('async'); + +function _deleteVersionList(s3Client, versionList, bucket, callback) { + if (versionList === undefined || versionList.length === 0) { + return callback(); + } + const params = { Bucket: bucket, Delete: { Objects: [] } }; + versionList.forEach(version => { + params.Delete.Objects.push({ + Key: version.Key, VersionId: version.VersionId }); + }); + + return s3Client.deleteObjects(params, callback); +} + +const testUtils = {}; + +testUtils.runIfMongo = process.env.S3METADATA === 'mongodb' ? + describe : describe.skip; + +testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, + keyToFind, done) => { + const searchRequest = s3Client.listObjects({ Bucket: bucketName }); + searchRequest.on('build', () => { + searchRequest.httpRequest.path = + `${searchRequest.httpRequest.path}?search=${encodedSearch}`; + }); + searchRequest.on('success', res => { + if (keyToFind) { + assert(res.data.Contents[0], 'should be Contents listed'); + assert.strictEqual(res.data.Contents[0].Key, keyToFind); + assert.strictEqual(res.data.Contents.length, 1); + } else { + assert.strictEqual(res.data.Contents.length, 0); + } + return done(); + }); + searchRequest.on('error', done); + searchRequest.send(); +}; + +testUtils.removeAllVersions = (s3Client, bucket, callback) => { + async.waterfall([ + cb => s3Client.listObjectVersions({ Bucket: bucket }, cb), + (data, cb) => _deleteVersionList(s3Client, data.DeleteMarkers, bucket, + err => cb(err, data)), + (data, cb) => _deleteVersionList(s3Client, data.Versions, bucket, + err => cb(err, data)), + (data, cb) => { + if (data.IsTruncated) { + const params = { + Bucket: bucket, + KeyMarker: data.NextKeyMarker, + VersionIdMarker: data.NextVersionIdMarker, + }; + return this.removeAllVersions(params, cb); + } + return cb(); + }, + ], callback); +}; + +module.exports = testUtils; diff --git a/tests/functional/aws-node-sdk/test/mdSearch/utils/s3SDK.js b/tests/functional/aws-node-sdk/test/mdSearch/utils/s3SDK.js new file mode 100644 index 0000000000..ec2e234797 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/mdSearch/utils/s3SDK.js @@ -0,0 +1,17 @@ +const S3 = require('aws-sdk').S3; + +const config = { + sslEnabled: false, + endpoint: 'http://127.0.0.1:8000', + apiVersions: { s3: '2006-03-01' }, + signatureCache: false, + signatureVersion: 'v4', + region: 'us-east-1', + s3ForcePathStyle: true, + accessKeyId: 'accessKey1', + secretAccessKey: 'verySecretKey1', +}; + +const client = new S3(config); + +module.exports = client; diff --git a/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js b/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js new file mode 100644 index 0000000000..54eef93dc7 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js @@ -0,0 +1,61 @@ +const s3Client = require('./utils/s3SDK'); +const { runAndCheckSearch, removeAllVersions, runIfMongo } = + require('./utils/helpers'); + +const userMetadata = { food: 'pizza' }; +const updatedMetadata = { food: 'salad' }; +const masterKey = 'master'; + +runIfMongo('Search in version enabled bucket', () => { + const bucketName = `versionedbucket${Date.now()}`; + const VersioningConfiguration = { + MFADelete: 'Disabled', + Status: 'Enabled', + }; + before(done => { + s3Client.createBucket({ Bucket: bucketName }, err => { + if (err) { + return done(err); + } + return s3Client.putBucketVersioning({ Bucket: bucketName, + VersioningConfiguration }, err => { + if (err) { + return done(err); + } + return s3Client.putObject({ Bucket: bucketName, + Key: masterKey, Metadata: userMetadata }, done); + }); + }); + }); + + after(done => { + removeAllVersions(s3Client, bucketName, + err => { + if (err) { + return done(err); + } + return s3Client.deleteBucket({ Bucket: bucketName }, done); + }); + }); + + it('should list just master object with searched for metadata', done => { + const encodedSearch = + encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, + encodedSearch, masterKey, done); + }); + + describe('New version overwrite', () => { + before(done => { + s3Client.putObject({ Bucket: bucketName, + Key: masterKey, Metadata: updatedMetadata }, done); + }); + + it('should list just master object with updated metadata', done => { + const encodedSearch = + encodeURIComponent(`x-amz-meta-food="${updatedMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, + encodedSearch, masterKey, done); + }); + }); +}); diff --git a/tests/unit/utils/validateSearch.js b/tests/unit/utils/validateSearch.js index 8946221c8f..458a2e6e3a 100644 --- a/tests/unit/utils/validateSearch.js +++ b/tests/unit/utils/validateSearch.js @@ -106,6 +106,12 @@ describe('validate search where clause', () => { result: errors.InvalidArgument.customizeDescription('Search ' + 'param contains unknown attribute: madeUp'), }, + { + it: 'should disallow unsupported query operators', + searchParams: 'x-amz-meta-dog BETWEEN "labrador"', + result: errors.InvalidArgument.customizeDescription( + 'Invalid sql where clause sent as search query'), + }, ]; tests.forEach(test => {