Skip to content

Commit

Permalink
Merge d477b8e into 3947c33
Browse files Browse the repository at this point in the history
  • Loading branch information
@Pchelolo
Pchelolo committed Apr 4, 2019
2 parents 3947c33 + d477b8e commit 471adf1
Show file tree
Hide file tree
Showing 14 changed files with 130 additions and 244 deletions.
7 changes: 4 additions & 3 deletions lib/mwUtil.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@ const mwUtil = {};
/**
* Create an etag value of the form
* "<revision>/<tid>/<optional_suffix>"
* @param {Integer} rev page revision number
* @param {string} tid page render UUID
* @param {Integer} [rev] page revision number
* @param {string} [tid] page render UUID
* @param {string} [suffix] optional suffix
* @return {string} the value of the ETag header
*/
mwUtil.makeETag = (rev, tid, suffix) => {
mwUtil.makeETag = (rev = 0, tid, suffix) => {
tid = tid || uuid.now();
let etag = `"${rev}/${tid}`;
if (suffix) {
etag += `/${suffix}`;
Expand Down
14 changes: 0 additions & 14 deletions projects/dev.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,18 +118,4 @@ paths:
- path: sys/mobileapps.js
options: '{{merge({"response_cache_control": options.purged_cache_control},
options.mobileapps)}}'
/mobile_bucket:
x-modules:
- path: sys/multi_content_bucket.js
options:
grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
table_name_prefix: mobile
main_content_type:
name: lead
value_type: json
dependent_content_types:
- name: remaining
value_type: json
options: '{{options}}'

17 changes: 2 additions & 15 deletions projects/wmf_enwiki.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ paths:
allows us to contact you quickly. Email addresses or URLs
of contact pages work well.
By using this API, you agree to Wikimedia's
By using this API, you agree to Wikimedia's
[Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
[Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
Unless otherwise specified in the endpoint documentation
below, content accessed via this API is licensed under the
[CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
[CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
and you irrevocably agree to release modifications or
additions made through this API under these licenses.
Expand Down Expand Up @@ -221,19 +221,6 @@ paths:
- path: sys/mobileapps.js
options: '{{merge({"response_cache_control": options.purged_cache_control},
options.mobileapps)}}'
/mobile_bucket:
x-modules:
- path: sys/multi_content_bucket.js
options:
grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
table_name_prefix: mobile_ng
main_content_type:
name: lead
value_type: json
dependent_content_types:
- name: remaining
value_type: json
/events:
x-modules:
- path: sys/events.js
Expand Down
17 changes: 2 additions & 15 deletions projects/wmf_wikipedia.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ paths:
allows us to contact you quickly. Email addresses or URLs
of contact pages work well.
By using this API, you agree to Wikimedia's
By using this API, you agree to Wikimedia's
[Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
[Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
Unless otherwise specified in the endpoint documentation
below, content accessed via this API is licensed under the
[CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
[CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
and you irrevocably agree to release modifications or
additions made through this API under these licenses.
Expand Down Expand Up @@ -242,19 +242,6 @@ paths:
- path: sys/mobileapps.js
options: '{{merge({"response_cache_control": options.purged_cache_control},
options.mobileapps)}}'
/mobile_bucket:
x-modules:
- path: sys/multi_content_bucket.js
options:
grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
table_name_prefix: mobile_ng
main_content_type:
name: lead
value_type: json
dependent_content_types:
- name: remaining
value_type: json
/events:
x-modules:
- path: sys/events.js
Expand Down
13 changes: 0 additions & 13 deletions projects/wmf_wikivoyage.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,19 +179,6 @@ paths:
x-modules:
- path: sys/mobileapps.js
options: '{{merge({"response_cache_control": options.purged_cache_control}, options.mobileapps)}}'
/mobile_bucket:
x-modules:
- path: sys/multi_content_bucket.js
options:
grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
table_name_prefix: mobile_ng
main_content_type:
name: lead
value_type: json
dependent_content_types:
- name: remaining
value_type: json
/events:
x-modules:
- path: sys/events.js
Expand Down
19 changes: 3 additions & 16 deletions projects/wmf_wiktionary.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,15 @@ paths:
allows us to contact you quickly. Email addresses or URLs
of contact pages work well.
By using this API, you agree to Wikimedia's
By using this API, you agree to Wikimedia's
[Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and
[Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy).
Unless otherwise specified in the endpoint documentation
below, content accessed via this API is licensed under the
[CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
[CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/)
and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses,
and you irrevocably agree to release modifications or
additions made through this API under these licenses.
additions made through this API under these licenses.
See https://www.mediawiki.org/wiki/REST_API for background and details.
### Endpoint documentation
Expand Down Expand Up @@ -174,19 +174,6 @@ paths:
- path: sys/mobileapps.js
options: '{{merge({"response_cache_control": options.purged_cache_control},
options.mobileapps)}}'
/mobile_bucket:
x-modules:
- path: sys/multi_content_bucket.js
options:
grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
table_name_prefix: mobile_ng
main_content_type:
name: lead
value_type: json
dependent_content_types:
- name: remaining
value_type: json
/events:
x-modules:
- path: sys/events.js
Expand Down
135 changes: 59 additions & 76 deletions sys/key_value.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,45 +4,16 @@
* Key-value bucket handler
*/

const uuid = require('cassandra-uuid').TimeUuid;
const crypto = require('crypto');
const stringify = require('fast-json-stable-stringify');
const TimeUUID = require('cassandra-uuid').TimeUuid;
const mwUtil = require('../lib/mwUtil');
const HyperSwitch = require('hyperswitch');
const stringify = require('fast-json-stable-stringify');
const HTTPError = HyperSwitch.HTTPError;
const URI = HyperSwitch.URI;

const spec = HyperSwitch.utils.loadSpec(`${__dirname}/key_value.yaml`);

// Format a revision response. Shared between different ways to retrieve a
// revision (latest & with explicit revision).
function returnRevision(req) {
return (dbResult) => {
if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {
const row = dbResult.body.items[0];
let headers = {
etag: row.headers.etag || mwUtil.makeETag('0', row.tid)
};
if (row.headers) {
headers = Object.assign(headers, row.headers);
}
return {
status: 200,
headers,
body: row.value
};
} else {
throw new HTTPError({
status: 404,
body: {
type: 'not_found',
uri: req.uri,
method: req.method
}
});
}
};
}

class KVBucket {
createBucket(hyper, req) {
const schema = this.makeSchema(req.body || {});
Expand All @@ -56,27 +27,20 @@ class KVBucket {
}

makeSchema(opts) {
const schemaVersionMajor = 5;
const schemaVersionMajor = 6;

return {
// Combine option & bucket version into a monotonically increasing
// combined schema version. By multiplying the bucket version by 1000,
// we increase the chance of catching a reset in the option version.
version: schemaVersionMajor * 1000 + (opts.version || 0),
options: {
compression: opts.compression || [
{
algorithm: 'deflate',
block_size: 256
}
],
updates: opts.updates || {
pattern: 'timeseries'
}
},
attributes: {
key: opts.keyType || 'string',
// Both TID and ETAG are added in case we ever want to support
// CAS using lightweight transactions to support proper
// conditional HTTP requests with `if-modified-since` or `if-match`
tid: 'timeuuid',
etag: 'string',
headers: 'json',
value: opts.valueType || 'blob'
},
Expand All @@ -88,7 +52,13 @@ class KVBucket {

getRevision(hyper, req) {
if (mwUtil.isNoCacheRequest(req)) {
throw new HTTPError({ status: 404 });
throw new HTTPError({
status: 404,
body: {
type: 'not_found',
description: 'Not attempting to fetch content for no-cache request'
}
});
}

const rp = req.params;
Expand All @@ -98,11 +68,28 @@ class KVBucket {
table: rp.bucket,
attributes: {
key: rp.key
},
limit: 1
}
}
};
return hyper.get(storeReq).then(returnRevision(req));
return hyper.get(storeReq).then((dbResult) => {
if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {
const row = dbResult.body.items[0];
return {
status: 200,
headers: row.headers,
body: row.value
};
} else {
throw new HTTPError({
status: 404,
body: {
type: 'not_found',
uri: req.uri,
method: req.method
}
});
}
});
}

listRevisions(hyper, req) {
Expand All @@ -114,7 +101,6 @@ class KVBucket {
attributes: {
key: req.params.key
},
proj: ['tid'],
limit: 1000
}
};
Expand All @@ -131,21 +117,21 @@ class KVBucket {
}

putRevision(hyper, req) {
const rp = req.params;
let tid = rp.tid && mwUtil.coerceTid(rp.tid, 'key_value');

if (!tid) {
tid = (mwUtil.parseETag(req.headers && req.headers.etag) || {}).tid;
tid = tid || uuid.now().toString();
if (mwUtil.isNoStoreRequest(req)) {
return { status: 202 };
}

if (mwUtil.isNoStoreRequest(req)) {
return {
status: 202,
headers: {
etag: req.headers && req.headers.etag || mwUtil.makeETag('0', tid)
}
};
const rp = req.params;
req.headers = req.headers || {};

const tid = TimeUUID.now().toString();
if (!req.headers.etag) {
hyper.logger.log('fatal/kv/putRevision', {
msg: 'No etag header provided to key-value bucket'
});
req.headers.etag = crypto.createHash('sha256')
.update(stringify(req.body))
.digest('hex');
}

const doPut = () => hyper.put({
Expand All @@ -155,6 +141,7 @@ class KVBucket {
attributes: {
key: rp.key,
tid,
etag: req.headers.etag,
value: req.body,
headers: req.headers
}
Expand All @@ -164,23 +151,18 @@ class KVBucket {
if (res.status === 201) {
return {
status: 201,
headers: {
etag: req.headers && req.headers.etag || mwUtil.makeETag('0', tid)
},
body: {
message: 'Created.',
tid
message: 'Created.'
}
};
} else {
throw res;
}
})
.catch((error) => {
hyper.logger.log('error/kv/putRevision', error);
return { status: 400 };
});
.tapCatch((error) => hyper.logger.log('error/kv/putRevision', error));

// TODO: Respect the stored ETag and allow matching on etag - either one provided
// by the client or auto-generated one.
if (req.headers['if-none-hash-match']) {
delete req.headers['if-none-hash-match'];
return hyper.get({
Expand All @@ -191,14 +173,15 @@ class KVBucket {
(!req.headers['content-type'] ||
req.headers['content-type'] === oldContent.headers['content-type'])) {
hyper.metrics.increment(`sys_kv_${req.params.bucket}.unchanged_rev_render`);
return {
throw new HTTPError({
status: 412,
headers: {
etag: oldContent.headers.etag
body: {
type: 'precondition_failed',
description: 'Not replacing existing content'
}
};
});
}
throw new HTTPError({ status: 404 });
return doPut();
})
.catch({ status: 404 }, doPut);
} else {
Expand Down
Loading

0 comments on commit 471adf1

Please sign in to comment.