Rf/zenko 250 portmdsearchtest

lib/api/apiUtils/bucket/parseWhere.js

@@ -31,6 +31,7 @@
  */
 const exprMapper = {
     '=': '$eq',
+    '!=': '$ne',
     '<>': '$ne',
     '>': '$gt',
     '<': '$lt',

lib/api/apiUtils/bucket/validateSearch.js

@@ -1,26 +1,49 @@
 const Parser = require('sql-where-parser');
-const parser = new Parser();
 const { errors } = require('arsenal');
 const objModel = require('arsenal').models.ObjectMD;
 
+const BINARY_OP = 2;
+const sqlConfig = {
+    operators: [
+        {
+            '=': BINARY_OP,
+            '<': BINARY_OP,
+            '>': BINARY_OP,
+            '<>': BINARY_OP,
+            '<=': BINARY_OP,
+            '>=': BINARY_OP,
+            '!=': BINARY_OP,
+        },
+        { LIKE: BINARY_OP },
+        { AND: BINARY_OP },
+        { OR: BINARY_OP },
+    ],
+    tokenizer: {
+        shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'],
+        shouldMatch: ['"', '\'', '`'],
+        shouldDelimitBy: [' ', '\n', '\r', '\t'],
+    },
+};
+const parser = new Parser(sqlConfig);
+
 function _validateTree(whereClause, possibleAttributes) {
     let invalidAttribute;
 
     function _searchTree(node) {
-        const operator = Object.keys(node)[0];
-
-        if (operator === 'AND') {
-            _searchTree(node[operator][0]);
-            _searchTree(node[operator][1]);
-        } else if (operator === 'OR') {
-            _searchTree(node[operator][0]);
-            _searchTree(node[operator][1]);
+        if (typeof node !== 'object') {
+            invalidAttribute = node;
         } else {
-            const field = node[operator][0];
-
-            if (!possibleAttributes[field] &&
-                !field.startsWith('x-amz-meta-')) {
-                invalidAttribute = field;
+            const operator = Object.keys(node)[0];
+            if (operator === 'AND' || operator === 'OR') {
+                _searchTree(node[operator][0]);
+                _searchTree(node[operator][1]);
+            } else {
+                const field = node[operator][0];
+                if (!field.startsWith('tags.') &&
+                    !possibleAttributes[field] &&
+                    !field.startsWith('x-amz-meta-')) {
+                    invalidAttribute = field;
+                }
             }
         }
     }
@@ -32,8 +55,8 @@ function _validateTree(whereClause, possibleAttributes) {
  * validateSearchParams - validate value of ?search= in request
  * @param  {string} searchParams - value of search params in request
  *  which should be jsu sql where clause
- * For metadata: userMd.`x-amz-meta-color`=\"blue\"
- * For tags: tags.`x-amz-meta-color`=\"blue\"
+ * For metadata: x-amz-meta-color=\"blue\"
+ * For tags: tags.x-amz-meta-color=\"blue\"
  * For any other attribute: `content-length`=5
  * @return {undefined | error} undefined if validates or arsenal error if not
  */
@@ -43,9 +66,11 @@ function validateSearchParams(searchParams) {
         ast = parser.parse(searchParams);
     } catch (e) {
         if (e) {
-            return errors.InvalidArgument
-            .customizeDescription('Invalid sql where clause ' +
-            'sent as search query');
+            return {
+                error: errors.InvalidArgument
+                .customizeDescription('Invalid sql where clause ' +
+                'sent as search query'),
+            };
         }
     }
     const possibleAttributes = objModel.getAttributes();

lib/services.js

@@ -101,6 +101,7 @@ const services = {
         const md = new ObjectMD();
         // This should be object creator's canonical ID.
         md.setOwnerId(authInfo.getCanonicalID())
+            .setKey(objectKey)
             .setCacheControl(cacheControl)
             .setContentDisposition(contentDisposition)
             .setContentEncoding(contentEncoding)

package.json

@@ -77,6 +77,7 @@
     "ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
     "ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
     "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management",
+    "ft_search": "cd tests/functional/aws-node-sdk && mocha -t 90000 test/mdSearch",
     "install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
     "lint": "eslint $(git ls-files '*.js')",
     "lint_md": "mdlint $(git ls-files '*.md')",

tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js

@@ -0,0 +1,101 @@
+const s3Client = require('./utils/s3SDK');
+const { runAndCheckSearch, runIfMongo } = require('./utils/helpers');
+
+const objectKey = 'findMe';
+const hiddenKey = 'leaveMeAlone';
+const objectTagData = 'item-type=main';
+const hiddenTagData = 'item-type=dessert';
+const userMetadata = { food: 'pizza' };
+const updatedUserMetadata = { food: 'cake' };
+
+runIfMongo('Basic search', () => {
+    const bucketName = `basicsearchmebucket${Date.now()}`;
+    before(done => {
+        s3Client.createBucket({ Bucket: bucketName }, err => {
+            if (err) {
+                return done(err);
+            }
+            return s3Client.putObject({ Bucket: bucketName, Key: objectKey,
+                Metadata: userMetadata, Tagging: objectTagData }, err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3Client.putObject({ Bucket: bucketName,
+                    Key: hiddenKey, Tagging: hiddenTagData }, done);
+            });
+        });
+    });
+
+    after(done => {
+        s3Client.deleteObjects({ Bucket: bucketName, Delete: { Objects: [
+            { Key: objectKey },
+            { Key: hiddenKey }],
+        } },
+            err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3Client.deleteBucket({ Bucket: bucketName }, done);
+            });
+    });
+
+    it('should list object with searched for system metadata', done => {
+        const encodedSearch = encodeURIComponent(`key="${objectKey}"`);
+        return runAndCheckSearch(s3Client, bucketName,
+            encodedSearch, objectKey, done);
+    });
+
+    it('should list object with searched for user metadata', done => {
+        const encodedSearch =
+            encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`);
+        return runAndCheckSearch(s3Client, bucketName, encodedSearch,
+            objectKey, done);
+    });
+
+    it('should list object with searched for tag metadata', done => {
+        const encodedSearch =
+            encodeURIComponent('tags.item-type="main"');
+        return runAndCheckSearch(s3Client, bucketName, encodedSearch,
+            objectKey, done);
+    });
+
+    it('should return empty listing when no object has user md', done => {
+        const encodedSearch =
+        encodeURIComponent('x-amz-meta-food="nosuchfood"');
+        return runAndCheckSearch(s3Client, bucketName,
+            encodedSearch, null, done);
+    });
+
+    describe('search when overwrite object', () => {
+        before(done => {
+            s3Client.putObject({ Bucket: bucketName, Key: objectKey,
+                Metadata: updatedUserMetadata }, done);
+        });
+
+        it('should list object with searched for updated user metadata',
+            done => {
+                const encodedSearch =
+                encodeURIComponent('x-amz-meta-food' +
+                `="${updatedUserMetadata.food}"`);
+                return runAndCheckSearch(s3Client, bucketName, encodedSearch,
+                objectKey, done);
+            });
+    });
+});
+
+runIfMongo('Search when no objects in bucket', () => {
+    const bucketName = `noobjectbucket${Date.now()}`;
+    before(done => {
+        s3Client.createBucket({ Bucket: bucketName }, done);
+    });
+
+    after(done => {
+        s3Client.deleteBucket({ Bucket: bucketName }, done);
+    });
+
+    it('should return empty listing when no objects in bucket', done => {
+        const encodedSearch = encodeURIComponent(`key="${objectKey}"`);
+        return runAndCheckSearch(s3Client, bucketName,
+            encodedSearch, null, done);
+    });
+});

tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js

@@ -0,0 +1,64 @@
+const assert = require('assert');
+const async = require('async');
+
+function _deleteVersionList(s3Client, versionList, bucket, callback) {
+    if (versionList === undefined || versionList.length === 0) {
+        return callback();
+    }
+    const params = { Bucket: bucket, Delete: { Objects: [] } };
+    versionList.forEach(version => {
+        params.Delete.Objects.push({
+            Key: version.Key, VersionId: version.VersionId });
+    });
+
+    return s3Client.deleteObjects(params, callback);
+}
+
+const testUtils = {};
+
+testUtils.runIfMongo = process.env.S3METADATA === 'mongodb' ?
+    describe : describe.skip;
+
+testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch,
+    keyToFind, done) => {
+    const searchRequest = s3Client.listObjects({ Bucket: bucketName });
+    searchRequest.on('build', () => {
+        searchRequest.httpRequest.path =
+        `${searchRequest.httpRequest.path}?search=${encodedSearch}`;
+    });
+    searchRequest.on('success', res => {
+        if (keyToFind) {
+            assert(res.data.Contents[0], 'should be Contents listed');
+            assert.strictEqual(res.data.Contents[0].Key, keyToFind);
+            assert.strictEqual(res.data.Contents.length, 1);
+        } else {
+            assert.strictEqual(res.data.Contents.length, 0);
+        }
+        return done();
+    });
+    searchRequest.on('error', done);
+    searchRequest.send();
+};
+
+testUtils.removeAllVersions = (s3Client, bucket, callback) => {
+    async.waterfall([
+        cb => s3Client.listObjectVersions({ Bucket: bucket }, cb),
+        (data, cb) => _deleteVersionList(s3Client, data.DeleteMarkers, bucket,
+            err => cb(err, data)),
+        (data, cb) => _deleteVersionList(s3Client, data.Versions, bucket,
+            err => cb(err, data)),
+        (data, cb) => {
+            if (data.IsTruncated) {
+                const params = {
+                    Bucket: bucket,
+                    KeyMarker: data.NextKeyMarker,
+                    VersionIdMarker: data.NextVersionIdMarker,
+                };
+                return this.removeAllVersions(params, cb);
+            }
+            return cb();
+        },
+    ], callback);
+};
+
+module.exports = testUtils;

tests/functional/aws-node-sdk/test/mdSearch/utils/s3SDK.js

@@ -0,0 +1,17 @@
+const S3 = require('aws-sdk').S3;
+
+const config = {
+    sslEnabled: false,
+    endpoint: 'http://127.0.0.1:8000',
+    apiVersions: { s3: '2006-03-01' },
+    signatureCache: false,
+    signatureVersion: 'v4',
+    region: 'us-east-1',
+    s3ForcePathStyle: true,
+    accessKeyId: 'accessKey1',
+    secretAccessKey: 'verySecretKey1',
+};
+
+const client = new S3(config);
+
+module.exports = client;

tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js

@@ -0,0 +1,61 @@
+const s3Client = require('./utils/s3SDK');
+const { runAndCheckSearch, removeAllVersions, runIfMongo } =
+    require('./utils/helpers');
+
+const userMetadata = { food: 'pizza' };
+const updatedMetadata = { food: 'salad' };
+const masterKey = 'master';
+
+runIfMongo('Search in version enabled bucket', () => {
+    const bucketName = `versionedbucket${Date.now()}`;
+    const VersioningConfiguration = {
+        MFADelete: 'Disabled',
+        Status: 'Enabled',
+    };
+    before(done => {
+        s3Client.createBucket({ Bucket: bucketName }, err => {
+            if (err) {
+                return done(err);
+            }
+            return s3Client.putBucketVersioning({ Bucket: bucketName,
+                VersioningConfiguration }, err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3Client.putObject({ Bucket: bucketName,
+                    Key: masterKey, Metadata: userMetadata }, done);
+            });
+        });
+    });
+
+    after(done => {
+        removeAllVersions(s3Client, bucketName,
+            err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3Client.deleteBucket({ Bucket: bucketName }, done);
+            });
+    });
+
+    it('should list just master object with searched for metadata', done => {
+        const encodedSearch =
+        encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`);
+        return runAndCheckSearch(s3Client, bucketName,
+            encodedSearch, masterKey, done);
+    });
+
+    describe('New version overwrite', () => {
+        before(done => {
+            s3Client.putObject({ Bucket: bucketName,
+                Key: masterKey, Metadata: updatedMetadata }, done);
+        });
+
+        it('should list just master object with updated metadata', done => {
+            const encodedSearch =
+            encodeURIComponent(`x-amz-meta-food="${updatedMetadata.food}"`);
+            return runAndCheckSearch(s3Client, bucketName,
+                encodedSearch, masterKey, done);
+        });
+    });
+});

tests/unit/utils/validateSearch.js

@@ -106,6 +106,12 @@ describe('validate search where clause', () => {
             result: errors.InvalidArgument.customizeDescription('Search ' +
                 'param contains unknown attribute: madeUp'),
         },
+        {
+            it: 'should disallow unsupported query operators',
+            searchParams: 'x-amz-meta-dog BETWEEN "labrador"',
+            result: errors.InvalidArgument.customizeDescription(
+                'Invalid sql where clause sent as search query'),
+        },
     ];
 
     tests.forEach(test => {