Skip to content

Commit

Permalink
Merge dbc5693 into 7aadda0
Browse files Browse the repository at this point in the history
  • Loading branch information
Pchelolo committed May 29, 2019
2 parents 7aadda0 + dbc5693 commit 44ce777
Show file tree
Hide file tree
Showing 26 changed files with 373 additions and 91 deletions.
7 changes: 5 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ addons:
- openjdk-8-jre-headless

env:
- CASSANDRA_VERSION=3.11.2 TEST_TARGET=all
- CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=single_process
- CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=single_process
- CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=multi_process
- CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=multi_process

before_install:
- wget https://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz -P ../
Expand All @@ -29,4 +32,4 @@ before_install:
- sed -i -e 's/^-XX:+UseNUMA/#-XX:+UseNUMA/' ../apache-cassandra-${CASSANDRA_VERSION}/conf/jvm.options
- bash -x ../apache-cassandra-${CASSANDRA_VERSION}/bin/cassandra

script: npm run lint && npm run coverage -- ${TEST_TARGET} && (npm run-script coveralls || exit 0)
script: npm run lint && npm run coverage -- ${TEST_TARGET} ${TEST_MODE} && (npm run-script coveralls || exit 0)
89 changes: 89 additions & 0 deletions config.example.storage.wikimedia.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# RESTBase wikimedia example config
default_project: &default_project
x-modules:
- spec:
paths:
/{api:v1}:
x-modules:
- spec:
paths:
/key_value:
x-modules:
- path: sys/key_value.js
/page_revisions:
# We need to forward cookie to the API
x-route-filters:
- path: ./lib/mediawiki_auth_filter.js
x-modules:
- path: sys/page_revisions.js
/{api:sys}:
x-modules:
- spec:
paths:
/table:
x-modules:
- path: sys/table.js
options:
conf:
backend: '{env(RB_TEST_BACKEND, sqlite)}'
hosts: [localhost]
keyspace: system
username: cassandra
password: cassandra
defaultConsistency: one # or 'localQuorum' for production
storage_groups:
- name: test.group.local
domains: /./
dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing
/action:
x-modules:
- path: sys/action.js
options:
apiUriTemplate: "{{'https://{domain}/w/api.php'}}"
baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}"


# The root of the spec tree. Domains tend to share specs by referencing them
# using YAML references.
spec_root: &spec_root
title: "The RESTBase Storage Service"
x-sub-request-filters:
- type: default
name: http
options:
allow:
- pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/
forward_headers: true
paths:
/{domain}: *default_project

# Finally, a standard service-runner config.
info:
name: restbase

services:
- name: restbase
module: hyperswitch
conf:
port: 7232
spec: *spec_root
salt: secret
default_page_size: 125
user_agent: RESTBase
ui_name: RESTBase
ui_url: https://www.mediawiki.org/wiki/RESTBase
ui_title: RESTBase docs

logging:
name: restbase
level: error
streams:
- type: stdout

#metrics:
# type: statsd
# host: localhost
# port: 8125
# batch: true

num_workers: 0
5 changes: 5 additions & 0 deletions config.example.wikimedia.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ default_project: &default_project
x-modules:
- path: projects/wmf_default.yaml
options: &default_options
proxy:
backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys/legacy"}})}'
table:
backend: '{env(RB_TEST_BACKEND, sqlite)}'
hosts: [localhost]
Expand Down Expand Up @@ -110,6 +112,9 @@ spec_root: &spec_root
forward_headers: true
- pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/
forward_headers: true
# Need to forward cookie to backend RESTBase
- pattern: /^https?:\/\/localhost:7232.+/
forward_headers: true
- pattern: /^https?:\/\//
paths:
/{domain:en.wikipedia.org}: *en.wikipedia.org
Expand Down
5 changes: 5 additions & 0 deletions config.test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ default_project: &default_project
- path: test/test_module.yaml
- path: projects/wmf_default.yaml
options: &default_options
proxy:
backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys/legacy"}})}'
table:
backend: '{env(RB_TEST_BACKEND, sqlite)}'
hosts: [localhost]
Expand Down Expand Up @@ -119,6 +121,9 @@ spec_root: &spec_root
forward_headers: true
- pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/
forward_headers: true
# Need to forward cookie to backend RESTBase
- pattern: /^https?:\/\/localhost:7232.+/
forward_headers: true
- pattern: /^https?:\/\//
paths:
/{domain:test.wikipedia.org}: *wikipedia_project
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"dependencies": {
"bluebird": "^3.5.5",
"cassandra-uuid": "^0.1.0",
"colors": "^1.3.3",
"content-type": "git+https://github.com/wikimedia/content-type#master",
"entities": "^1.1.2",
"extend": "^3.0.2",
Expand Down
6 changes: 5 additions & 1 deletion projects/wikimedia.org.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,11 @@ paths:
- path: sys/table.js
options:
conf: '{{options.table}}'
/key_value: &sys_key_value
/key_value:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/key_value:
x-modules:
- path: sys/key_value.js
/post_data: &sys_post_data
Expand Down
10 changes: 9 additions & 1 deletion projects/wmf_default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,18 @@ paths:
- path: sys/table.js
options:
conf: '{{options.table}}'
/key_value: &sys_key_value
/key_value:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/key_value:
x-modules:
- path: sys/key_value.js
/page_revisions:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/page_revisions:
x-modules:
- path: sys/page_revisions.js
/post_data: &sys_post_data
Expand Down
12 changes: 10 additions & 2 deletions projects/wmf_enwiki.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -134,15 +134,23 @@ paths:
x-modules:
- spec:
paths:
/table: &sys_table
/table:
x-modules:
- path: sys/table.js
options:
conf: '{{options.table}}'
/key_value: &sys_key_value
/key_value:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/key_value:
x-modules:
- path: sys/key_value.js
/page_revisions:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/page_revisions:
x-modules:
- path: sys/page_revisions.js
/post_data: &sys_post_data
Expand Down
10 changes: 9 additions & 1 deletion projects/wmf_wikidata.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,18 @@ paths:
- path: sys/table.js
options:
conf: '{{options.table}}'
/key_value: &sys_key_value
/key_value:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/key_value:
x-modules:
- path: sys/key_value.js
/page_revisions:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/page_revisions:
x-modules:
- path: sys/page_revisions.js
/post_data: &sys_post_data
Expand Down
10 changes: 9 additions & 1 deletion projects/wmf_wikipedia.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,18 @@ paths:
- path: sys/table.js
options:
conf: '{{options.table}}'
/key_value: &sys_key_value
/key_value:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/key_value:
x-modules:
- path: sys/key_value.js
/page_revisions:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/page_revisions:
x-modules:
- path: sys/page_revisions.js
/post_data: &sys_post_data
Expand Down
10 changes: 9 additions & 1 deletion projects/wmf_wikivoyage.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,18 @@ paths:
- path: sys/table.js
options:
conf: '{{options.table}}'
/key_value: &sys_key_value
/key_value:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/key_value:
x-modules:
- path: sys/key_value.js
/page_revisions:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/page_revisions:
x-modules:
- path: sys/page_revisions.js
/post_data: &sys_post_data
Expand Down
10 changes: 9 additions & 1 deletion projects/wmf_wiktionary.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,18 @@ paths:
- path: sys/table.js
options:
conf: '{{options.table}}'
/key_value: &sys_key_value
/key_value:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/key_value:
x-modules:
- path: sys/key_value.js
/page_revisions:
x-modules:
- path: sys/backend_proxy.js
options: '{{options.proxy}}'
/legacy/page_revisions:
x-modules:
- path: sys/page_revisions.js
/post_data: &sys_post_data
Expand Down
45 changes: 45 additions & 0 deletions sys/backend_proxy.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
'use strict';

const Template = require('hyperswitch').Template;

module.exports = (options) => {
const backendURITemplate = new Template({
uri: `${options.backend_host_template}/{{path}}`
});
return {
spec: {
paths: {
'/{+path}': {
all: {
operationId: 'proxy'
}
}
}
},
operations: {
proxy: (hyper, req) => {
// Add the proxied module name to the path.
const modName = req.uri.path[req.uri.path.indexOf('sys') + 1];
req.params.path = `${modName}/${req.params.path}`;
return hyper.request({
method: req.method,
uri: backendURITemplate.expand({ request: req }).uri,
headers: req.headers,
body: req.body,
query: req.query
})
.then((res) => {
// TODO: Leave a comment why.
if (res &&
res.headers &&
res.headers['content-type'] &&
res.headers['content-type'].startsWith('application/json') &&
Buffer.isBuffer(res.body)) {
res.body = JSON.parse(res.body.toString('utf8'));
}
return res;
});
}
}
};
};
4 changes: 1 addition & 3 deletions sys/key_value.js
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,7 @@ class KVBucket {

if (req.headers['if-none-hash-match']) {
delete req.headers['if-none-hash-match'];
return hyper.get({
uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket, rp.key])
})
return this.getRevision(hyper, req)
.then((oldContent) => {
// TODO: proper etag-based compare.
if (stringify(req.body) === stringify(oldContent.body) &&
Expand Down
Loading

0 comments on commit 44ce777

Please sign in to comment.