Merge 4a37241 into 8afca46

config.example.wikimedia.yaml

@@ -16,9 +16,11 @@ default_project: &default_project
           storage_groups:
             - name: default.group.local
               domains: /./
+          # ignored in cassandra, but useful in SQLite testing. only used in tests.
+          dbname: test.db.sqlite3
         parsoid:
           host: https://parsoid-beta.wmflabs.org
-          grace_ttl: 1
+          grace_ttl: 1000000
         action:
           apiUriTemplate: "{{'https://{domain}/w/api.php'}}"
           baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}"

projects/dev.yaml

@@ -118,18 +118,5 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
         options: '{{options}}'
 

projects/wmf_enwiki.yaml

@@ -18,12 +18,12 @@ paths:
                   allows us to contact you quickly. Email addresses or URLs
                   of contact pages work well.
 
-                By using this API, you agree to Wikimedia's 
+                By using this API, you agree to Wikimedia's
                 [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
                 [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
                 Unless otherwise specified in the endpoint documentation
                 below, content accessed via this API is licensed under the
-                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) 
+                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
                 and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
                 and you irrevocably agree to release modifications or
                 additions made through this API under these licenses.
@@ -221,19 +221,6 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

projects/wmf_wikipedia.yaml

@@ -18,12 +18,12 @@ paths:
                   allows us to contact you quickly. Email addresses or URLs
                   of contact pages work well.
 
-                By using this API, you agree to Wikimedia's 
+                By using this API, you agree to Wikimedia's
                 [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
                 [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
                 Unless otherwise specified in the endpoint documentation
                 below, content accessed via this API is licensed under the
-                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) 
+                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
                 and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
                 and you irrevocably agree to release modifications or
                 additions made through this API under these licenses.
@@ -242,19 +242,6 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

projects/wmf_wikivoyage.yaml

@@ -179,19 +179,6 @@ paths:
               x-modules:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control}, options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

projects/wmf_wiktionary.yaml

@@ -19,15 +19,15 @@ paths:
                   allows us to contact you quickly. Email addresses or URLs
                   of contact pages work well.
 
-                By using this API, you agree to Wikimedia's 
+                By using this API, you agree to Wikimedia's
                 [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
                 [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
                 Unless otherwise specified in the endpoint documentation
                 below, content accessed via this API is licensed under the
-                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) 
+                [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
                 and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
                 and you irrevocably agree to release modifications or
-                additions made through this API under these licenses. 
+                additions made through this API under these licenses.
                 See https://www.mediawiki.org/wiki/REST_API for background and details.
 
                 ### Endpoint documentation
@@ -174,19 +174,6 @@ paths:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
-            /mobile_bucket:
-              x-modules:
-                - path: sys/multi_content_bucket.js
-                  options:
-                    grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
-                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
-                    table_name_prefix: mobile_ng
-                    main_content_type:
-                      name: lead
-                      value_type: json
-                    dependent_content_types:
-                      - name: remaining
-                        value_type: json
             /events:
               x-modules:
                 - path: sys/events.js

sys/key_rev_value.js

@@ -106,36 +106,6 @@ class KRVBucket {
         return hyper.get(storeReq).then(returnRevision(req));
     }
 
-    listRevisions(hyper, req) {
-        const rp = req.params;
-        return hyper.get({
-            uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
-            body: {
-                table: req.params.bucket,
-                attributes: {
-                    key: req.params.key
-                },
-                proj: ['rev', 'tid'],
-                limit: mwUtil.getLimit(hyper, req)
-            }
-        })
-        .then((res) => ({
-            status: 200,
-
-            headers: {
-                'content-type': 'application/json'
-            },
-
-            body: {
-                items: res.body.items.map((row) => ({
-                    revision: row.rev,
-                    tid: row.tid
-                })),
-                next: res.body.next
-            }
-        }));
-    }
-
     putRevision(hyper, req) {
         const rp = req.params;
         const rev = mwUtil.parseRevision(rp.revision, 'key_rev_value');
@@ -185,7 +155,6 @@ module.exports = (options) => {
         spec, // Re-export from spec module
         operations: {
             createBucket: krvBucket.createBucket.bind(krvBucket),
-            listRevisions: krvBucket.listRevisions.bind(krvBucket),
             getRevision: krvBucket.getRevision.bind(krvBucket),
             putRevision: krvBucket.putRevision.bind(krvBucket)
         }

sys/key_rev_value.yaml

@@ -14,10 +14,6 @@ paths:
     put:
       operationId: putRevision
 
-  /{bucket}/{key}/:
-    get:
-      operationId: listRevisions
-
   /{bucket}/{key}/{revision}:
     <<: *bucket_key
 

sys/key_value.js

@@ -13,36 +13,6 @@ const URI = HyperSwitch.URI;
 
 const spec = HyperSwitch.utils.loadSpec(`${__dirname}/key_value.yaml`);
 
-// Format a revision response. Shared between different ways to retrieve a
-// revision (latest & with explicit revision).
-function returnRevision(req) {
-    return (dbResult) => {
-        if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {
-            const row = dbResult.body.items[0];
-            let headers = {
-                etag: row.headers.etag || mwUtil.makeETag('0', row.tid)
-            };
-            if (row.headers) {
-                headers = Object.assign(headers, row.headers);
-            }
-            return {
-                status: 200,
-                headers,
-                body: row.value
-            };
-        } else {
-            throw new HTTPError({
-                status: 404,
-                body: {
-                    type: 'not_found',
-                    uri: req.uri,
-                    method: req.method
-                }
-            });
-        }
-    };
-}
-
 class KVBucket {
     createBucket(hyper, req) {
         const schema = this.makeSchema(req.body || {});
@@ -102,83 +72,53 @@ class KVBucket {
                 limit: 1
             }
         };
-        return hyper.get(storeReq).then(returnRevision(req));
-    }
-
-    listRevisions(hyper, req) {
-        const rp = req.params;
-        const storeRequest = {
-            uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
-            body: {
-                table: req.params.bucket,
-                attributes: {
-                    key: req.params.key
-                },
-                proj: ['tid'],
-                limit: 1000
-            }
-        };
-        return hyper.get(storeRequest)
-        .then((res) => ({
-            status: 200,
-            headers: {
-                'content-type': 'application/json'
-            },
-            body: {
-                items: res.body.items.map((row) => row.tid)
+        return hyper.get(storeReq).then((dbResult) => {
+            if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {
+                const row = dbResult.body.items[0];
+                return {
+                    status: 200,
+                    headers: row.headers,
+                    body: row.value
+                };
+            } else {
+                throw new HTTPError({
+                    status: 404,
+                    body: {
+                        type: 'not_found',
+                        uri: req.uri,
+                        method: req.method
+                    }
+                });
             }
-        }));
+        });
     }
 
     putRevision(hyper, req) {
-        const rp = req.params;
-        let tid = rp.tid && mwUtil.coerceTid(rp.tid, 'key_value');
-
-        if (!tid) {
-            tid = (mwUtil.parseETag(req.headers && req.headers.etag) || {}).tid;
-            tid = tid || uuid.now().toString();
-        }
-
         if (mwUtil.isNoStoreRequest(req)) {
-            return {
-                status: 202,
-                headers: {
-                    etag: req.headers && req.headers.etag || mwUtil.makeETag('0', tid)
-                }
-            };
+            return { status: 202 };
         }
 
+        const rp = req.params;
+        const tid = uuid.now().toString();
+
+        const headersToStore = {};
+        Object.keys(req.headers)
+        .filter(headerName => headerName.startsWith('x-store-'))
+        .forEach((headerName => {
+            headersToStore[headerName.replace(/^x-store-/, '')] = req.headers[headerName];
+        }));
+
         const doPut = () => hyper.put({
             uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
             body: {
                 table: rp.bucket,
                 attributes: {
                     key: rp.key,
                     tid,
-                    value: req.body,
-                    headers: req.headers
+                    headers: headersToStore,
+                    value: req.body
                 }
             }
-        })
-        .then((res) => {
-            if (res.status === 201) {
-                return {
-                    status: 201,
-                    headers: {
-                        etag: req.headers && req.headers.etag || mwUtil.makeETag('0', tid)
-                    },
-                    body: {
-                        message: 'Created.',
-                        tid
-                    }
-                };
-            } else {
-                throw res;
-            }
-        })
-        .catch((error) => {
-            hyper.logger.log('error/kv/putRevision', error);
-            return { status: 400 };
         });
 
         if (req.headers['if-none-hash-match']) {
@@ -187,15 +127,14 @@ class KVBucket {
                 uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket, rp.key])
             })
             .then((oldContent) => {
+                // TODO: proper etag-based compare.
                 if (stringify(req.body) === stringify(oldContent.body) &&
-                        (!req.headers['content-type'] ||
-                        req.headers['content-type'] === oldContent.headers['content-type'])) {
+                        (!headersToStore['content-type'] ||
+                        headersToStore['content-type'] === oldContent.headers['content-type'])) {
                     hyper.metrics.increment(`sys_kv_${req.params.bucket}.unchanged_rev_render`);
                     return {
                         status: 412,
-                        headers: {
-                            etag: oldContent.headers.etag
-                        }
+                        headers: oldContent.headers
                     };
                 }
                 throw new HTTPError({ status: 404 });
@@ -213,7 +152,6 @@ module.exports = (options) => {
         spec, // Re-export from spec module
         operations: {
             createBucket: kvBucket.createBucket.bind(kvBucket),
-            listRevisions: kvBucket.listRevisions.bind(kvBucket),
             getRevision: kvBucket.getRevision.bind(kvBucket),
             putRevision: kvBucket.putRevision.bind(kvBucket)
         }