Merge b502d8b into 9ff5637

.travis.yml

@@ -29,4 +29,4 @@ before_install:
   - sed -i -e 's/^-XX:+UseNUMA/#-XX:+UseNUMA/' ../apache-cassandra-${CASSANDRA_VERSION}/conf/jvm.options
   - bash -x ../apache-cassandra-${CASSANDRA_VERSION}/bin/cassandra
 
-script: sh test/utils/run_tests.sh coverage ${TEST_TARGET} && (npm run-script coveralls || exit 0)
+script: npm run lint && npm run coverage -- ${TEST_TARGET} && (npm run-script coveralls || exit 0)

config.example.wikimedia.yaml

@@ -14,7 +14,7 @@ default_project: &default_project
           password: cassandra
           defaultConsistency: one # or 'localQuorum' for production
           storage_groups:
-            - name: default.group.local
+            - name: test.group.local
               domains: /./
           # ignored in cassandra, but useful in SQLite testing. only used in tests.
           dbname: test.db.sqlite3

projects/dev.yaml

@@ -118,18 +118,5 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
         options: '{{options}}'
 

projects/wmf_enwiki.yaml

@@ -18,12 +18,12 @@ paths:
                   allows us to contact you quickly. Email addresses or URLs
                   of contact pages work well.
 
-                By using this API, you agree to Wikimedia's 
+                By using this API, you agree to Wikimedia's
                 [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
                 [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
                 Unless otherwise specified in the endpoint documentation
                 below, content accessed via this API is licensed under the
-                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) 
+                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
                 and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
                 and you irrevocably agree to release modifications or
                 additions made through this API under these licenses.
@@ -221,19 +221,6 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

projects/wmf_wikipedia.yaml

@@ -18,12 +18,12 @@ paths:
                   allows us to contact you quickly. Email addresses or URLs
                   of contact pages work well.
 
-                By using this API, you agree to Wikimedia's 
+                By using this API, you agree to Wikimedia's
                 [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
                 [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
                 Unless otherwise specified in the endpoint documentation
                 below, content accessed via this API is licensed under the
-                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) 
+                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
                 and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
                 and you irrevocably agree to release modifications or
                 additions made through this API under these licenses.
@@ -242,19 +242,6 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

projects/wmf_wikivoyage.yaml

@@ -179,19 +179,6 @@ paths:
               x-modules:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control}, options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

projects/wmf_wiktionary.yaml

@@ -19,15 +19,15 @@ paths:
                   allows us to contact you quickly. Email addresses or URLs
                   of contact pages work well.
 
-                By using this API, you agree to Wikimedia's 
+                By using this API, you agree to Wikimedia's
                 [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
                 [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
                 Unless otherwise specified in the endpoint documentation
                 below, content accessed via this API is licensed under the
-                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) 
+                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
                 and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
                 and you irrevocably agree to release modifications or
-                additions made through this API under these licenses. 
+                additions made through this API under these licenses.
                 See https://www.mediawiki.org/wiki/REST_API for background and details.
 
                 ### Endpoint documentation
@@ -174,19 +174,6 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

sys/key_value.js

@@ -13,36 +13,6 @@ const URI = HyperSwitch.URI;
 
 const spec = HyperSwitch.utils.loadSpec(`${__dirname}/key_value.yaml`);
 
-// Format a revision response. Shared between different ways to retrieve a
-// revision (latest & with explicit revision).
-function returnRevision(req) {
-    return (dbResult) => {
-        if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {
-            const row = dbResult.body.items[0];
-            let headers = {
-                etag: row.headers.etag || mwUtil.makeETag('0', row.tid)
-            };
-            if (row.headers) {
-                headers = Object.assign(headers, row.headers);
-            }
-            return {
-                status: 200,
-                headers,
-                body: row.value
-            };
-        } else {
-            throw new HTTPError({
-                status: 404,
-                body: {
-                    type: 'not_found',
-                    uri: req.uri,
-                    method: req.method
-                }
-            });
-        }
-    };
-}
-
 class KVBucket {
     createBucket(hyper, req) {
         const schema = this.makeSchema(req.body || {});
@@ -102,58 +72,53 @@ class KVBucket {
                 limit: 1
             }
         };
-        return hyper.get(storeReq).then(returnRevision(req));
+        return hyper.get(storeReq).then((dbResult) => {
+            if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {
+                const row = dbResult.body.items[0];
+                return {
+                    status: 200,
+                    headers: row.headers,
+                    body: row.value
+                };
+            } else {
+                throw new HTTPError({
+                    status: 404,
+                    body: {
+                        type: 'not_found',
+                        uri: req.uri,
+                        method: req.method
+                    }
+                });
+            }
+        });
     }
 
     putRevision(hyper, req) {
-        const rp = req.params;
-        let tid = rp.tid && mwUtil.coerceTid(rp.tid, 'key_value');
-
-        if (!tid) {
-            tid = (mwUtil.parseETag(req.headers && req.headers.etag) || {}).tid;
-            tid = tid || uuid.now().toString();
-        }
-
         if (mwUtil.isNoStoreRequest(req)) {
-            return {
-                status: 202,
-                headers: {
-                    etag: req.headers && req.headers.etag || mwUtil.makeETag('0', tid)
-                }
-            };
+            return { status: 202 };
         }
 
+        const rp = req.params;
+        const tid = uuid.now().toString();
+
+        const headersToStore = {};
+        Object.keys(req.headers)
+        .filter((headerName) => headerName.startsWith('x-store-'))
+        .forEach((headerName) => {
+            headersToStore[headerName.replace(/^x-store-/, '')] = req.headers[headerName];
+        });
+
         const doPut = () => hyper.put({
             uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
             body: {
                 table: rp.bucket,
                 attributes: {
                     key: rp.key,
                     tid,
-                    value: req.body,
-                    headers: req.headers
+                    headers: headersToStore,
+                    value: req.body
                 }
             }
-        })
-        .then((res) => {
-            if (res.status === 201) {
-                return {
-                    status: 201,
-                    headers: {
-                        etag: req.headers && req.headers.etag || mwUtil.makeETag('0', tid)
-                    },
-                    body: {
-                        message: 'Created.',
-                        tid
-                    }
-                };
-            } else {
-                throw res;
-            }
-        })
-        .catch((error) => {
-            hyper.logger.log('error/kv/putRevision', error);
-            return { status: 400 };
         });
 
         if (req.headers['if-none-hash-match']) {
@@ -162,15 +127,14 @@ class KVBucket {
                 uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket, rp.key])
             })
             .then((oldContent) => {
+                // TODO: proper etag-based compare.
                 if (stringify(req.body) === stringify(oldContent.body) &&
-                        (!req.headers['content-type'] ||
-                        req.headers['content-type'] === oldContent.headers['content-type'])) {
+                        (!headersToStore['content-type'] ||
+                        headersToStore['content-type'] === oldContent.headers['content-type'])) {
                     hyper.metrics.increment(`sys_kv_${req.params.bucket}.unchanged_rev_render`);
                     return {
                         status: 412,
-                        headers: {
-                            etag: oldContent.headers.etag
-                        }
+                        headers: oldContent.headers
                     };
                 }
                 throw new HTTPError({ status: 404 });

sys/key_value.yaml

@@ -2,7 +2,11 @@ swagger: '2.0'
 info:
   version: '1.0.0'
   title: RESTBase key-value module
-  description: Revisioned blob storage with HTTP interface, backed by table storage
+  description: >
+    Blob storage with HTTP interface, backed by table storage.
+
+    All headers prepended with `x-store-` are saved on write and returned
+    with the content upon retrieval. All the not prepended headers not stored.
 paths:
   /{bucket}:
     put:

sys/mathoid.js

@@ -4,6 +4,7 @@ const P = require('bluebird');
 const HyperSwitch = require('hyperswitch');
 const URI = HyperSwitch.URI;
 const HTTPError = HyperSwitch.HTTPError;
+const mwUtil = require('../lib/mwUtil');
 
 const FORMATS = ['mml', 'svg', 'png'];
 
@@ -25,7 +26,7 @@ class MathoidService {
         }).then((res) => {
             hash = origHash = res.body;
             // short-circuit if it's a no-cache request
-            if (req.headers && /no-cache/.test(req.headers['cache-control'])) {
+            if (mwUtil.isNoCacheRequest(req)) {
                 return P.reject(new HTTPError({ status: 404 }));
             }
             // check the post storage
@@ -83,7 +84,11 @@ class MathoidService {
                 return P.join(
                     hyper.put({
                         uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid_ng.check', hash]),
-                        headers: checkRes.headers,
+                        headers: {
+                            'x-store-x-resource-location': hash,
+                            'x-store-cache-control': 'no-cache',
+                            'x-store-content-type': 'application/json'
+                        },
                         body: checkRes.body
                     }),
                     indirectionP,
@@ -114,10 +119,16 @@ class MathoidService {
                 }));
             }
             // construct the request object that will be emitted
+            Object.assign(completeBody[format].headers, {
+                'x-resource-location': hash
+            });
+            const storeHeaders = {};
+            Object.keys(completeBody[format].headers).forEach((header) => {
+                storeHeaders[`x-store-${header}`] = completeBody[format].headers[header];
+            });
             const reqObj = {
                 uri: new URI([domain, 'sys', 'key_value', `mathoid_ng.${format}`, hash]),
-                headers: Object.assign(
-                    completeBody[format].headers, { 'x-resource-location': hash }),
+                headers: storeHeaders,
                 body: completeBody[format].body
             };
             if (format === 'png' && reqObj.body && reqObj.body.type === 'Buffer') {