Fwdport/z/1.0 master

circle.yml

@@ -41,7 +41,7 @@ dependencies:
     - gem install mime-types -v 3.1
     - gem install rspec -v 3.5
     - gem install json
-    - gem install digest
+    # - gem install digest
     # java sdk dependencies
     - sudo apt-get install -y -q default-jdk
 

dataserver.js

@@ -6,7 +6,9 @@ const logger = require('./lib/utilities/logger');
 
 if (config.backends.data === 'file' ||
     (config.backends.data === 'multiple' &&
-     config.backends.metadata !== 'scality')) {
+     config.backends.metadata !== 'scality') &&
+     (config.backends.auth !== 'scality' &&
+      config.backends.metadata !== 'mongodb')) {
     const dataServer = new arsenal.network.rest.RESTServer(
         { bindAddress: config.dataDaemon.bindAddress,
             port: config.dataDaemon.port,

docs/MD_SEARCH.md

@@ -0,0 +1,229 @@
+# Metadata Search Documenation
+
+## Description
+
+This feature enables metadata search to be performed on the metadata of objects
+stored in Zenko.
+
+## Requirements
+
++ MongoDB
+
+## Design
+
+The MD Search feature expands on the existing `GET Bucket` S3 API. It allows
+users to conduct metadata searches by adding the custom Zenko querystring
+parameter, `search`. The `search` parameter is of a pseudo
+SQL WHERE clause structure and supports basic SQL operators:
+ex. `"A=1 AND B=2 OR C=3"` (more complex queries can also be achieved with the
+use of nesting operators, `(` and `)`).
+
+The search process is as follows:
+
++ Zenko receives a `GET` request.
+
+    ```
+    # regular getBucket request
+    GET /bucketname HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # getBucket versions request
+    GET /bucketname?versions HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # search getBucket request
+    GET /bucketname?search=key%3Dsearch-item HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+    ```
+
++ If the request does not contain the query param `search`, a normal bucket
+  listing is performed and a XML result containing the list of objects will be
+  returned as the response.
++ If the request does contain the query parameter `search`, the search string is
+  parsed and validated.
+
+    + If the search string is invalid, an `InvalidArgument` error will be
+      returned as response.
+    + If the search string is valid, it will be parsed and an abstract syntax
+      tree (AST) is generated.
+
++ The AST is then passed to the MongoDB backend to be used as the query filter
+  for retrieving objects in a bucket that satisfies the requested search
+  conditions.
++ The filtered results are then parsed and returned as the response.
+
+The results from MD search is of the same structure as the `GET Bucket`
+results:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+    <Name>bucketname</Name>
+    <Prefix/>
+    <Marker/>
+    <MaxKeys>1000</MaxKeys>
+    <IsTruncated>false</IsTruncated>
+    <Contents>
+        <Key>objectKey</Key>
+        <LastModified>2018-04-19T18:31:49.426Z</LastModified>
+        <ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
+        <Size>0</Size>
+        <Owner>
+            <ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
+            <DisplayName>Bart</DisplayName>
+        </Owner>
+        <StorageClass>STANDARD</StorageClass>
+    </Contents>
+    <Contents>
+        ...
+    </Contents>
+</ListBucketResult>
+```
+
+## Performing MD Search with Zenko
+
+To make a successful request to Zenko, you would need
+
++ Zenko Credentials
++ Sign request with Auth V4
+
+With requirements, you can peform metadata searches by:
+
++ using the `search_bucket` tool in the
+  [Scality/S3](https://github.com/scality/S3) GitHub repository.
++ creating an AuthV4 signed HTTP request to Zenko in the programming language of
+  choice
+
+### Using the S3 Tool
+
+After cloning the [Scality/S3](https://github.com/scality/S3) GitHub repository
+and installing the necessary dependencies, you can run the following command
+in the S3 project root directory to access the search tool.
+
+```
+node bin/search_bucket
+```
+
+This will generate the following output
+
+```
+Usage: search_bucket [options]
+
+Options:
+
+  -V, --version                 output the version number
+  -a, --access-key <accessKey>  Access key id
+  -k, --secret-key <secretKey>  Secret access key
+  -b, --bucket <bucket>         Name of the bucket
+  -q, --query <query>           Search query
+  -h, --host <host>             Host of the server
+  -p, --port <port>             Port of the server
+  -s                            --ssl
+  -v, --verbose
+  -h, --help                    output usage information
+```
+
+In the following examples, our Zenko Server is accessible on endpoint
+`http://127.0.0.1:8000` and contains the bucket `zenkobucket`.
+
+```
+# search for objects with metadata "blue"
+node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+    -q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
+
+# search for objects tagged with "type=color"
+node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+    -q "tags.type=color" -h 127.0.0.1 -p 8000
+```
+
+### Coding Examples
+
+Search requests can be also performed by making HTTP requests authenticated
+with the `AWS Signature version 4` scheme.\
+See the following urls for more information about the V4 authentication scheme.
+
++ http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
++ http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+
+You can also view examples for making requests with Auth V4 in various
+languages [here](../exmaples).
+
+### Specifying Metadata Fields
+
+To search common metadata headers:
+
+```
+    {metadata-key}{supported SQL op}{search value}
+    # example
+    key = blueObject
+    size > 0
+    key LIKE "blue.*"
+```
+
+To search custom user metadata:
+
+```
+    # metadata must be prefixed with "x-amz-meta-"
+    x-amz-meta-{usermetadata-key}{supported SQL op}{search value}
+    # example
+    x-amz-meta-color = blue
+    x-amz-meta-color != red
+    x-amz-meta-color LIKE "b.*"
+```
+
+To search tags:
+
+```
+    # tag searches must be prefixed with "tags."
+    tags.{tag-key}{supported SQL op}{search value}
+    # example
+    tags.type = color
+```
+
+### Differences from SQL
+
+The MD search queries are similar to the `WHERE` clauses of SQL queries, but
+they differ in that:
+
++ MD search queries follow the `PCRE` format
++ Search queries do not require values with hyphens to be enclosed in
+  backticks, ``(`)``
+
+    ```
+        # SQL query
+        `x-amz-meta-search-item` = `ice-cream-cone`
+
+        # MD Search query
+        x-amz-meta-search-item = ice-cream-cone
+    ```
+
++ The search queries do not support all of the SQL operators.
+
+  + Supported SQL Operators: `=`, `<`, `>`, `<=`, `>=`, `!=`,
+    `AND`, `OR`, `LIKE`, `<>`
+  + Unsupported SQL Operators: `NOT`, `BETWEEN`, `IN`, `IS`, `+`,
+    `-`, `%`, `^`, `/`, `*`, `!`
+
+#### Using Regular Expressions in MD Search
+
++ Regular expressions used in MD search differs from SQL in that wildcards are
+  represented with `.*` instead of `%`.
++ Regex patterns must be wrapped in quotes as not doing so can lead to
+  misinterpretation of patterns.
++ Regex patterns can be written in form of the `/pattern/` syntax or
+  just the pattern if one does not require regex options, similar to `PCRE`.
+
+Example regular expressions:
+
+    ```
+    # search for strings containing word substring "helloworld"
+        ".*helloworld.*"
+        "/.*helloworld.*/"
+        "/.*helloworld.*/i"
+    ```

docs/mkdocs.yml

@@ -1,3 +1,4 @@
+---
 # http://www.mkdocs.org/user-guide/configuration/
 # https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes
 

examples/go-md-search.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+    "fmt"
+    "time"
+    "bytes"
+    "net/http"
+    "net/url"
+    "io/ioutil"
+    "github.com/aws/aws-sdk-go/aws/credentials"
+    "github.com/aws/aws-sdk-go/aws/signer/v4"
+)
+
+func main() {
+    // Input AWS access key, secret key
+    aws_access_key_id := "accessKey1"
+    aws_secret_access_key := "verySecretKey1"
+    endpoint := "http://localhost:8000"
+    bucket_name := "bucketname"
+    searchQuery := url.QueryEscape("x-amz-meta-color=blue")
+    buf := bytes.NewBuffer([]byte{})
+
+    requestUrl := fmt.Sprintf("%s/%s?search=%s",
+        endpoint, bucket_name, searchQuery)
+
+    request, err := http.NewRequest("GET", requestUrl, buf)
+    if err != nil {
+        panic(err)
+    }
+    reader := bytes.NewReader(buf.Bytes())
+    credentials := credentials.NewStaticCredentials(aws_access_key_id,
+        aws_secret_access_key, "")
+    signer := v4.NewSigner(credentials)
+    signer.Sign(request, reader, "s3", "us-east-1", time.Now())
+    client := &http.Client{}
+    resp, err := client.Do(request)
+    if err != nil {
+        panic(err)
+    }
+    defer resp.Body.Close()
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println(string(body))
+}

examples/node-md-search.js

@@ -0,0 +1,28 @@
+const { S3 } = require('aws-sdk');
+const config = {
+    sslEnabled: false,
+    endpoint: 'http://127.0.0.1:8000',
+    signatureCache: false,
+    signatureVersion: 'v4',
+    region: 'us-east-1',
+    s3ForcePathStyle: true,
+    accessKeyId: 'accessKey1',
+    secretAccessKey: 'verySecretKey1',
+};
+const s3Client = new S3(config);
+
+const encodedSearch =
+    encodeURIComponent('x-amz-meta-color="blue"');
+const req = s3Client.listObjects({ Bucket: 'bucketname' });
+
+// the build event
+req.on('build', () => {
+    req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
+});
+req.on('success', res => {
+    process.stdout.write(`Result ${res.data}`);
+});
+req.on('error', err => {
+    process.stdout.write(`Error ${err}`);
+});
+req.send();