From 99c68025832feb7752c86af52b7aeeef71c063e6 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Mon, 1 Apr 2019 14:37:09 -0300 Subject: [PATCH 1/9] WIP: Add storage RB config. Change-Id: I19bee6cc678b8b051abd575c8a6089696e64da61 --- config.example.storage.wikimedia.yaml | 153 +++++++++++++++++++++ test/features/buckets/key_value_bucket.js | 5 +- test/features/buckets/revisioned_bucket.js | 0 test/test_module.yaml | 4 - test/utils/server.js | 7 +- 5 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 config.example.storage.wikimedia.yaml create mode 100644 test/features/buckets/revisioned_bucket.js diff --git a/config.example.storage.wikimedia.yaml b/config.example.storage.wikimedia.yaml new file mode 100644 index 000000000..7f4696c25 --- /dev/null +++ b/config.example.storage.wikimedia.yaml @@ -0,0 +1,153 @@ +# RESTBase wikimedia example config + +# Load some project templates. These are referenced / shared between domains +# in the root_spec further down. +default_project: &default_project + x-modules: + - spec: + paths: + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: default.group.local + domains: /./ + /parsoid_bucket: + x-modules: + - path: sys/multi_content_bucket.js + options: + grace_ttl: 1 + index_ttl: 864000 + renew_expiring: true + delete_probability: 1 + table_name_prefix: parsoid_ng + main_content_type: + name: html + value_type: blob + dependent_content_types: + - name: data-parsoid + value_type: json + /mobile_bucket: + x-modules: + - path: sys/multi_content_bucket.js + options: + grace_ttl: 1 + delete_probability: 1 + table_name_prefix: mobile_ng + main_content_type: + name: lead + value_type: json + dependent_content_types: + - name: remaining + value_type: json + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /key_rev_value: + x-modules: + - path: sys/key_rev_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + +# The root of the spec tree. Domains tend to share specs by referencing them +# using YAML references. +spec_root: &spec_root + title: "The RESTBase root" + x-request-filters: + - path: lib/security_response_header_filter.js + - path: lib/normalize_headers_filter.js + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true + - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ + forward_headers: true + - pattern: /^https?:\/\// + paths: + /{domain:en.wikipedia.org}: *default_project + /{domain:ru.wikipedia.org}: *default_project + /{domain:de.wikipedia.org}: *default_project + /{domain:es.wikipedia.org}: *default_project + /{domain:it.wikipedia.org}: *default_project + /{domain:da.wikipedia.org}: *default_project + /{domain:sv.wikipedia.org}: *default_project + + # Example for wiktionary + /{domain:en.wiktionary.org}: *default_project + + # global domain + /{domain:wikimedia.org}: *default_project + + # special case for wikidata + /{domain:www.wikidata.org}: *default_project + + # example for non-wikipedia + /{domain:en.wikivoyage.beta.wmflabs.org}: *default_project + + # A robots.txt to make sure that the content isn't indexed. + /robots.txt: + get: + x-request-handler: + - static: + return: + status: 200 + headers: + content-type: text/plain + body: | + User-agent: * + Allow: /*/v1/?doc + Disallow: / + +# Finally, a standard service-runner config. +info: + name: restbase + +services: + - name: restbase + module: hyperswitch + conf: + port: 7232 + spec: *spec_root + salt: secret + default_page_size: 125 + user_agent: RESTBase + ui_name: RESTBase + ui_url: https://www.mediawiki.org/wiki/RESTBase + ui_title: RESTBase docs + +logging: + name: restbase + level: warn + streams: + - type: stdout + +#metrics: +# type: statsd +# host: localhost +# port: 8125 +# batch: true + +ratelimiter: + type: kademlia + # Cluster nodes + seeds: + # Port 3050 used by default + - 127.0.0.1 + +num_workers: 0 diff --git a/test/features/buckets/key_value_bucket.js b/test/features/buckets/key_value_bucket.js index cf4799365..05cd28c41 100644 --- a/test/features/buckets/key_value_bucket.js +++ b/test/features/buckets/key_value_bucket.js @@ -18,12 +18,13 @@ describe('Key value buckets', () => { } function runTests(bucketName) { - const server = new Server(); + const server = new Server(`${__dirname}/../../../config.example.storage.wikimedia.yaml`); let bucketBaseURI; before(() => server.start() .then(() => { bucketBaseURI = - `${server.config.baseURL()}/buckets/${bucketName}/${bucketName}TestingBucket`; + `${server.config.sysURL()}/${bucketName}/${bucketName}TestingBucket`; + console.log(bucketBaseURI); return preq.put({ uri: bucketBaseURI} ); })); after(() => server.stop()); diff --git a/test/features/buckets/revisioned_bucket.js b/test/features/buckets/revisioned_bucket.js new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_module.yaml b/test/test_module.yaml index cf40952ae..9210a8f6d 100644 --- a/test/test_module.yaml +++ b/test/test_module.yaml @@ -74,10 +74,6 @@ paths: uri: /{domain}/sys/post_data/post.test/{hash} x-monitor: false - /buckets/key_value: - x-modules: - - path: sys/key_value.js - /events_no_config: x-modules: - path: sys/events.js diff --git a/test/utils/server.js b/test/utils/server.js index 1b0055ab8..d44823258 100644 --- a/test/utils/server.js +++ b/test/utils/server.js @@ -12,8 +12,10 @@ class TestRestbase extends TestRunner { if (!this._running) { throw new Error('Accessing test service config before starting the service'); } - const hostPort = 'http://localhost:7231'; + const conf = this._runner._impl.config; + const hostPort = `http://localhost:${conf.services[0].conf.port}`; const baseURL = (domain = DEFAULT_DOMAIN) => `${hostPort}/${domain}/v1`; + const sysURL = (domain = DEFAULT_DOMAIN) => `${hostPort}/${domain}/sys`; const bucketURL = (domain) => `${baseURL(domain)}/page`; const apiPath = '/w/api.php'; const apiBase = (domain = DEFAULT_DOMAIN) => `https://${domain}`; @@ -22,12 +24,13 @@ class TestRestbase extends TestRunner { defaultDomain: DEFAULT_DOMAIN, hostPort, baseURL, + sysURL, bucketURL, apiBase, apiPath, apiURL, parsoidURI: 'https://parsoid-beta.wmflabs.org', - conf: this._runner._impl.config + conf } } } From 3680cea58f319283354b5c53ed025d666d3c5741 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Tue, 7 May 2019 14:20:53 -0700 Subject: [PATCH 2/9] Switch summary to proxying to the backend service Change-Id: I08991ef84318e94839a00823fe0c8b06d3ce9382 --- config.example.storage.wikimedia.yaml | 88 +++++-------------- config.test.yaml | 3 + projects/wikimedia.org.yaml | 4 + projects/wmf_default.yaml | 4 + projects/wmf_enwiki.yaml | 6 +- projects/wmf_wikidata.yaml | 4 + projects/wmf_wikipedia.yaml | 4 + projects/wmf_wikivoyage.yaml | 4 + projects/wmf_wiktionary.yaml | 4 + sys/backend_proxy.js | 32 +++++++ sys/key_value.js | 4 +- test/features/buckets/key_value_bucket.js | 4 +- .../features/pagecontent/language_variants.js | 6 +- test/utils/server.js | 2 - v1/summary_new.yaml | 13 +-- 15 files changed, 102 insertions(+), 80 deletions(-) create mode 100644 sys/backend_proxy.js diff --git a/config.example.storage.wikimedia.yaml b/config.example.storage.wikimedia.yaml index 7f4696c25..4e47df457 100644 --- a/config.example.storage.wikimedia.yaml +++ b/config.example.storage.wikimedia.yaml @@ -6,24 +6,10 @@ default_project: &default_project x-modules: - spec: paths: - /{api:sys}: + /{api:v1}: x-modules: - spec: paths: - /table: &sys_table - x-modules: - - path: sys/table.js - options: - conf: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: default.group.local - domains: /./ /parsoid_bucket: x-modules: - path: sys/multi_content_bucket.js @@ -39,67 +25,41 @@ default_project: &default_project dependent_content_types: - name: data-parsoid value_type: json - /mobile_bucket: - x-modules: - - path: sys/multi_content_bucket.js - options: - grace_ttl: 1 - delete_probability: 1 - table_name_prefix: mobile_ng - main_content_type: - name: lead - value_type: json - dependent_content_types: - - name: remaining - value_type: json - /key_value: &sys_key_value + /key_value: x-modules: - path: sys/key_value.js /key_rev_value: x-modules: - path: sys/key_rev_value.js - /page_revisions: + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table x-modules: - - path: sys/page_revisions.js + - path: sys/table.js + options: + conf: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: default.group.local + domains: /./ + dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing + # The root of the spec tree. Domains tend to share specs by referencing them # using YAML references. spec_root: &spec_root - title: "The RESTBase root" + title: "The RESTBase Storage Service" x-request-filters: - path: lib/security_response_header_filter.js - - path: lib/normalize_headers_filter.js - x-sub-request-filters: - - type: default - name: http - options: - allow: - - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ - forward_headers: true - - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ - forward_headers: true - - pattern: /^https?:\/\// paths: - /{domain:en.wikipedia.org}: *default_project - /{domain:ru.wikipedia.org}: *default_project - /{domain:de.wikipedia.org}: *default_project - /{domain:es.wikipedia.org}: *default_project - /{domain:it.wikipedia.org}: *default_project - /{domain:da.wikipedia.org}: *default_project - /{domain:sv.wikipedia.org}: *default_project - - # Example for wiktionary - /{domain:en.wiktionary.org}: *default_project - - # global domain - /{domain:wikimedia.org}: *default_project - - # special case for wikidata - /{domain:www.wikidata.org}: *default_project - - # example for non-wikipedia - /{domain:en.wikivoyage.beta.wmflabs.org}: *default_project - + /{domain}: *default_project # A robots.txt to make sure that the content isn't indexed. /robots.txt: get: @@ -133,7 +93,7 @@ services: logging: name: restbase - level: warn + level: error streams: - type: stdout diff --git a/config.test.yaml b/config.test.yaml index 31032409d..8d1583f5c 100644 --- a/config.test.yaml +++ b/config.test.yaml @@ -7,6 +7,9 @@ default_project: &default_project - path: test/test_module.yaml - path: projects/wmf_default.yaml options: &default_options + proxy: + #backend_host_template: "{{'/{domain}/sys'}}" + backend_host_template: "{{'http://localhost:7232/{domain}/v1'}}" table: backend: '{env(RB_TEST_BACKEND, sqlite)}' hosts: [localhost] diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml index 99cae3ee3..a9b8908ea 100644 --- a/projects/wikimedia.org.yaml +++ b/projects/wikimedia.org.yaml @@ -83,6 +83,10 @@ paths: x-modules: - spec: paths: + /proxy: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' /mathoid: x-modules: - path: sys/mathoid.js diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml index b1c21ee04..6c43bc241 100644 --- a/projects/wmf_default.yaml +++ b/projects/wmf_default.yaml @@ -107,6 +107,10 @@ paths: x-modules: - spec: paths: + /proxy: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml index 242655145..8d41461d8 100644 --- a/projects/wmf_enwiki.yaml +++ b/projects/wmf_enwiki.yaml @@ -134,7 +134,11 @@ paths: x-modules: - spec: paths: - /table: &sys_table + /proxy: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /table: x-modules: - path: sys/table.js options: diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml index 471969351..14bf4d89a 100644 --- a/projects/wmf_wikidata.yaml +++ b/projects/wmf_wikidata.yaml @@ -89,6 +89,10 @@ paths: x-modules: - spec: paths: + /proxy: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js diff --git a/projects/wmf_wikipedia.yaml b/projects/wmf_wikipedia.yaml index 090f8c0f6..dc6921d8d 100644 --- a/projects/wmf_wikipedia.yaml +++ b/projects/wmf_wikipedia.yaml @@ -134,6 +134,10 @@ paths: x-modules: - spec: paths: + /proxy: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js diff --git a/projects/wmf_wikivoyage.yaml b/projects/wmf_wikivoyage.yaml index 6e02b5da6..165b934c9 100644 --- a/projects/wmf_wikivoyage.yaml +++ b/projects/wmf_wikivoyage.yaml @@ -129,6 +129,10 @@ paths: x-modules: - spec: paths: + /proxy: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml index 194a99acb..8a0075503 100644 --- a/projects/wmf_wiktionary.yaml +++ b/projects/wmf_wiktionary.yaml @@ -123,6 +123,10 @@ paths: x-modules: - spec: paths: + /proxy: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js new file mode 100644 index 000000000..4cc331a9e --- /dev/null +++ b/sys/backend_proxy.js @@ -0,0 +1,32 @@ +'use strict'; + +const Template = require('hyperswitch').Template; + +module.exports = (options) => { + const backendURITemplate = new Template({ + uri: `${options.backend_host_template}/{{path}}` + }); + return { + spec: { + paths: { + '/{+path}': { + all: { + operationId: 'proxy' + } + } + } + }, + operations: { + proxy: (hyper, req) => { + req.uri = backendURITemplate.expand({ request: req }).uri.toString(); + return hyper.request({ + method: req.method, + uri: backendURITemplate.expand({ request: req }).uri.toString(), + headers: req.headers, + body: req.body, + query: req.query + }); + } + } + }; +}; diff --git a/sys/key_value.js b/sys/key_value.js index da489e2c9..64270e189 100644 --- a/sys/key_value.js +++ b/sys/key_value.js @@ -133,9 +133,7 @@ class KVBucket { if (req.headers['if-none-hash-match']) { delete req.headers['if-none-hash-match']; - return hyper.get({ - uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket, rp.key]) - }) + return this.getRevision(hyper, req) .then((oldContent) => { // TODO: proper etag-based compare. if (stringify(req.body) === stringify(oldContent.body) && diff --git a/test/features/buckets/key_value_bucket.js b/test/features/buckets/key_value_bucket.js index 05cd28c41..aa1195eb8 100644 --- a/test/features/buckets/key_value_bucket.js +++ b/test/features/buckets/key_value_bucket.js @@ -22,9 +22,7 @@ describe('Key value buckets', () => { let bucketBaseURI; before(() => server.start() .then(() => { - bucketBaseURI = - `${server.config.sysURL()}/${bucketName}/${bucketName}TestingBucket`; - console.log(bucketBaseURI); + bucketBaseURI = `${server.config.baseURL()}/${bucketName}/${bucketName}TestingBucket`; return preq.put({ uri: bucketBaseURI} ); })); after(() => server.stop()); diff --git a/test/features/pagecontent/language_variants.js b/test/features/pagecontent/language_variants.js index a20f50181..154c612ad 100644 --- a/test/features/pagecontent/language_variants.js +++ b/test/features/pagecontent/language_variants.js @@ -1,5 +1,6 @@ 'use strict'; +const P = require('bluebird'); const assert = require('../../utils/assert.js'); const preq = require('preq'); const Server = require('../../utils/server.js'); @@ -8,9 +9,10 @@ const variantsPageTitle = 'RESTBase_Testing_Page'; describe('Language variants', function() { this.timeout(20000); const server = new Server(); + const backendServer = new Server(`${__dirname}/../../../config.example.storage.wikimedia.yaml`); - before(() => server.start()); - after(() => server.stop()); + before(() => P.join(server.start(), backendServer.start())); + after(() => P.join(server.stop(), backendServer.stop())); it('should request html with impossible variants', () => { return preq.get({ uri: `${server.config.bucketURL()}/html/Main_Page`}) diff --git a/test/utils/server.js b/test/utils/server.js index d44823258..8493c83fa 100644 --- a/test/utils/server.js +++ b/test/utils/server.js @@ -15,7 +15,6 @@ class TestRestbase extends TestRunner { const conf = this._runner._impl.config; const hostPort = `http://localhost:${conf.services[0].conf.port}`; const baseURL = (domain = DEFAULT_DOMAIN) => `${hostPort}/${domain}/v1`; - const sysURL = (domain = DEFAULT_DOMAIN) => `${hostPort}/${domain}/sys`; const bucketURL = (domain) => `${baseURL(domain)}/page`; const apiPath = '/w/api.php'; const apiBase = (domain = DEFAULT_DOMAIN) => `https://${domain}`; @@ -24,7 +23,6 @@ class TestRestbase extends TestRunner { defaultDomain: DEFAULT_DOMAIN, hostPort, baseURL, - sysURL, bucketURL, apiBase, apiPath, diff --git a/v1/summary_new.yaml b/v1/summary_new.yaml index 6379ab879..4ca9ac484 100644 --- a/v1/summary_new.yaml +++ b/v1/summary_new.yaml @@ -93,8 +93,10 @@ paths: x-setup-handler: # Set up a simple key-value bucket. - init: - method: put - uri: /{domain}/sys/key_value/page_summary + method: 'put' + uri: /{domain}/sys/proxy/key_value/page_summary + headers: + content-type: application/json body: valueType: json x-request-handler: @@ -103,7 +105,7 @@ paths: method: get headers: cache-control: '{{cache-control}}' - uri: /{domain}/sys/key_value/page_summary/{request.params.title} + uri: /{domain}/sys/proxy/key_value/page_summary/{request.params.title} catch: status: 404 return_if: @@ -133,9 +135,10 @@ paths: - store_and_return: request: method: put - uri: /{domain}/sys/key_value/page_summary/{request.params.title} + uri: /{domain}/sys/proxy/key_value/page_summary/{request.params.title} headers: - if-none-hash-match": '*' + if-none-hash-match: '*' + content-type: 'application/json' cache-control: '{{request.headers.cache-control}}' x-store-etag: '{{extract.headers.etag}}' x-store-content-language: '{{extract.headers.content-language}}' From 122a202d0965bf789dfbddaa4da279084c554f4c Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Thu, 23 May 2019 19:59:59 +0300 Subject: [PATCH 3/9] Switch everything to multi-config Change-Id: I989ad30725f087c9e41a914677b7bfcd2e617583 --- .travis.yml | 7 +- config.example.storage.wikimedia.yaml | 64 +++++--------- config.example.wikimedia.yaml | 5 ++ config.test.yaml | 6 +- projects/wikimedia.org.yaml | 10 +-- projects/wmf_default.yaml | 14 ++-- projects/wmf_enwiki.yaml | 14 ++-- projects/wmf_wikidata.yaml | 14 ++-- projects/wmf_wikipedia.yaml | 14 ++-- projects/wmf_wikivoyage.yaml | 14 ++-- projects/wmf_wiktionary.yaml | 14 ++-- sys/backend_proxy.js | 27 +++++- sys/mathoid.js | 25 ++++-- sys/mobileapps.js | 1 - sys/parsoid.js | 20 ++--- sys/post_data.js | 3 + test/features/buckets/key_value_bucket.js | 40 ++++++++- test/features/buckets/revisioned_bucket.js | 0 test/features/pagecontent/access_checks.js | 14 +++- .../features/pagecontent/language_variants.js | 6 +- test/utils/run_tests.sh | 84 +++++++++---------- test/utils/server.js | 19 ++++- v1/definition.yaml | 5 +- v1/mathoid.yaml | 8 +- v1/summary_new.yaml | 6 +- 25 files changed, 275 insertions(+), 159 deletions(-) delete mode 100644 test/features/buckets/revisioned_bucket.js diff --git a/.travis.yml b/.travis.yml index 66ecf7e8f..8c71562ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,10 @@ addons: - openjdk-8-jre-headless env: - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=all + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=single_process + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=single_process + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=multi_process + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=multi_process before_install: - wget https://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz -P ../ @@ -29,4 +32,4 @@ before_install: - sed -i -e 's/^-XX:+UseNUMA/#-XX:+UseNUMA/' ../apache-cassandra-${CASSANDRA_VERSION}/conf/jvm.options - bash -x ../apache-cassandra-${CASSANDRA_VERSION}/bin/cassandra -script: npm run lint && npm run coverage -- ${TEST_TARGET} && (npm run-script coveralls || exit 0) +script: npm run lint && npm run coverage -- ${TEST_TARGET} ${TEST_MODE} && (npm run-script coveralls || exit 0) diff --git a/config.example.storage.wikimedia.yaml b/config.example.storage.wikimedia.yaml index 4e47df457..c59bf7377 100644 --- a/config.example.storage.wikimedia.yaml +++ b/config.example.storage.wikimedia.yaml @@ -1,7 +1,4 @@ # RESTBase wikimedia example config - -# Load some project templates. These are referenced / shared between domains -# in the root_spec further down. default_project: &default_project x-modules: - spec: @@ -10,32 +7,20 @@ default_project: &default_project x-modules: - spec: paths: - /parsoid_bucket: - x-modules: - - path: sys/multi_content_bucket.js - options: - grace_ttl: 1 - index_ttl: 864000 - renew_expiring: true - delete_probability: 1 - table_name_prefix: parsoid_ng - main_content_type: - name: html - value_type: blob - dependent_content_types: - - name: data-parsoid - value_type: json /key_value: x-modules: - path: sys/key_value.js - /key_rev_value: + /page_revisions: + # We need to forward cookie to the API + x-route-filters: + - path: ./lib/mediawiki_auth_filter.js x-modules: - - path: sys/key_rev_value.js + - path: sys/page_revisions.js /{api:sys}: x-modules: - spec: paths: - /table: &sys_table + /table: x-modules: - path: sys/table.js options: @@ -47,32 +32,30 @@ default_project: &default_project password: cassandra defaultConsistency: one # or 'localQuorum' for production storage_groups: - - name: default.group.local + - name: test.group.local domains: /./ dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing + /action: + x-modules: + - path: sys/action.js + options: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" # The root of the spec tree. Domains tend to share specs by referencing them # using YAML references. spec_root: &spec_root title: "The RESTBase Storage Service" - x-request-filters: - - path: lib/security_response_header_filter.js + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true paths: /{domain}: *default_project - # A robots.txt to make sure that the content isn't indexed. - /robots.txt: - get: - x-request-handler: - - static: - return: - status: 200 - headers: - content-type: text/plain - body: | - User-agent: * - Allow: /*/v1/?doc - Disallow: / # Finally, a standard service-runner config. info: @@ -103,11 +86,4 @@ logging: # port: 8125 # batch: true -ratelimiter: - type: kademlia - # Cluster nodes - seeds: - # Port 3050 used by default - - 127.0.0.1 - num_workers: 0 diff --git a/config.example.wikimedia.yaml b/config.example.wikimedia.yaml index ffdf8ead4..e10c11f51 100644 --- a/config.example.wikimedia.yaml +++ b/config.example.wikimedia.yaml @@ -6,6 +6,8 @@ default_project: &default_project x-modules: - path: projects/wmf_default.yaml options: &default_options + proxy: + backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys/legacy"}})}' table: backend: '{env(RB_TEST_BACKEND, sqlite)}' hosts: [localhost] @@ -110,6 +112,9 @@ spec_root: &spec_root forward_headers: true - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ forward_headers: true + # Need to forward cookie to backend RESTBase + - pattern: /^https?:\/\/localhost:7232.+/ + forward_headers: true - pattern: /^https?:\/\// paths: /{domain:en.wikipedia.org}: *en.wikipedia.org diff --git a/config.test.yaml b/config.test.yaml index 8d1583f5c..0d7f756e3 100644 --- a/config.test.yaml +++ b/config.test.yaml @@ -8,8 +8,7 @@ default_project: &default_project - path: projects/wmf_default.yaml options: &default_options proxy: - #backend_host_template: "{{'/{domain}/sys'}}" - backend_host_template: "{{'http://localhost:7232/{domain}/v1'}}" + backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys/legacy"}})}' table: backend: '{env(RB_TEST_BACKEND, sqlite)}' hosts: [localhost] @@ -122,6 +121,9 @@ spec_root: &spec_root forward_headers: true - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ forward_headers: true + # Need to forward cookie to backend RESTBase + - pattern: /^https?:\/\/localhost:7232.+/ + forward_headers: true - pattern: /^https?:\/\// paths: /{domain:test.wikipedia.org}: *wikipedia_project diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml index a9b8908ea..33cafbd50 100644 --- a/projects/wikimedia.org.yaml +++ b/projects/wikimedia.org.yaml @@ -83,10 +83,6 @@ paths: x-modules: - spec: paths: - /proxy: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' /mathoid: x-modules: - path: sys/mathoid.js @@ -96,7 +92,11 @@ paths: - path: sys/table.js options: conf: '{{options.table}}' - /key_value: &sys_key_value + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/key_value: x-modules: - path: sys/key_value.js /post_data: &sys_post_data diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml index 6c43bc241..c47b3587a 100644 --- a/projects/wmf_default.yaml +++ b/projects/wmf_default.yaml @@ -107,19 +107,23 @@ paths: x-modules: - spec: paths: - /proxy: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js options: conf: '{{options.table}}' - /key_value: &sys_key_value + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/key_value: x-modules: - path: sys/key_value.js /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/page_revisions: x-modules: - path: sys/page_revisions.js /post_data: &sys_post_data diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml index 8d41461d8..8bfc041f0 100644 --- a/projects/wmf_enwiki.yaml +++ b/projects/wmf_enwiki.yaml @@ -134,19 +134,23 @@ paths: x-modules: - spec: paths: - /proxy: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' /table: x-modules: - path: sys/table.js options: conf: '{{options.table}}' - /key_value: &sys_key_value + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/key_value: x-modules: - path: sys/key_value.js /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/page_revisions: x-modules: - path: sys/page_revisions.js /post_data: &sys_post_data diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml index 14bf4d89a..264120acc 100644 --- a/projects/wmf_wikidata.yaml +++ b/projects/wmf_wikidata.yaml @@ -89,19 +89,23 @@ paths: x-modules: - spec: paths: - /proxy: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js options: conf: '{{options.table}}' - /key_value: &sys_key_value + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/key_value: x-modules: - path: sys/key_value.js /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/page_revisions: x-modules: - path: sys/page_revisions.js /post_data: &sys_post_data diff --git a/projects/wmf_wikipedia.yaml b/projects/wmf_wikipedia.yaml index dc6921d8d..5eac81f71 100644 --- a/projects/wmf_wikipedia.yaml +++ b/projects/wmf_wikipedia.yaml @@ -134,19 +134,23 @@ paths: x-modules: - spec: paths: - /proxy: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js options: conf: '{{options.table}}' - /key_value: &sys_key_value + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/key_value: x-modules: - path: sys/key_value.js /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/page_revisions: x-modules: - path: sys/page_revisions.js /post_data: &sys_post_data diff --git a/projects/wmf_wikivoyage.yaml b/projects/wmf_wikivoyage.yaml index 165b934c9..9938340f9 100644 --- a/projects/wmf_wikivoyage.yaml +++ b/projects/wmf_wikivoyage.yaml @@ -129,19 +129,23 @@ paths: x-modules: - spec: paths: - /proxy: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js options: conf: '{{options.table}}' - /key_value: &sys_key_value + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/key_value: x-modules: - path: sys/key_value.js /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/page_revisions: x-modules: - path: sys/page_revisions.js /post_data: &sys_post_data diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml index 8a0075503..2e24c1a30 100644 --- a/projects/wmf_wiktionary.yaml +++ b/projects/wmf_wiktionary.yaml @@ -123,19 +123,23 @@ paths: x-modules: - spec: paths: - /proxy: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' /table: &sys_table x-modules: - path: sys/table.js options: conf: '{{options.table}}' - /key_value: &sys_key_value + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/key_value: x-modules: - path: sys/key_value.js /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /legacy/page_revisions: x-modules: - path: sys/page_revisions.js /post_data: &sys_post_data diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js index 4cc331a9e..49d5ecd02 100644 --- a/sys/backend_proxy.js +++ b/sys/backend_proxy.js @@ -18,13 +18,36 @@ module.exports = (options) => { }, operations: { proxy: (hyper, req) => { - req.uri = backendURITemplate.expand({ request: req }).uri.toString(); + // Add the proxied module name to the path. + // The proxy is mounted at paths like `/sys/key_value`. Here + // out of a request path we want to find out the name of the module + // that is being proxied (key_value) from the example and add it + // to the proxy target path. + const modName = req.uri.path[req.uri.path.indexOf('sys') + 1]; + req.params.path = `${modName}/${req.params.path}`; return hyper.request({ method: req.method, - uri: backendURITemplate.expand({ request: req }).uri.toString(), + uri: backendURITemplate.expand({ request: req }).uri, headers: req.headers, body: req.body, query: req.query + }) + .then((res) => { + // We are slowly switching to storing data as binary blobs + // in the storage component. When running multi-process, + // the storage component will return the appropriate + // content-type and deserializing the response will + // happen internally in preq. However, when running + // single process, preq is not involved, so for consistency + // we need to manually deserialize the returned blob. + if (res && + res.headers && + res.headers['content-type'] && + res.headers['content-type'].startsWith('application/json') && + Buffer.isBuffer(res.body)) { + res.body = JSON.parse(res.body.toString('utf8')); + } + return res; }); } } diff --git a/sys/mathoid.js b/sys/mathoid.js index e8087baf7..329cf4b3f 100644 --- a/sys/mathoid.js +++ b/sys/mathoid.js @@ -82,7 +82,10 @@ class MathoidService { indirectionP = hyper.put({ uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid_ng.hash_table', origHash]), - headers: { 'x-store-content-type': 'text/plain' }, + headers: { + 'content-type': 'text/plain', + 'x-store-content-type': 'text/plain' + }, body: hash }); } @@ -95,7 +98,9 @@ class MathoidService { return P.join( hyper.put({ uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid_ng.check', hash]), - headers: prefixHeaders(checkRes.headers), + headers: Object.assign({ + 'content-type': 'application/json' + }, prefixHeaders(checkRes.headers)), body: checkRes.body }), indirectionP, @@ -131,7 +136,9 @@ class MathoidService { }); const reqObj = { uri: new URI([domain, 'sys', 'key_value', `mathoid_ng.${format}`, hash]), - headers: prefixHeaders(completeBody[format].headers), + headers: Object.assign({ + 'content-type': completeBody[format].headers['content-type'] + }, prefixHeaders(completeBody[format].headers)), body: completeBody[format].body }; if (format === 'png' && reqObj.body && reqObj.body.type === 'Buffer') { @@ -251,11 +258,19 @@ module.exports = (options) => { resources: [ { uri: '/{domain}/sys/post_data/mathoid_ng.input' - }, { + }, + { uri: '/{domain}/sys/key_value/mathoid_ng.hash_table', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'string' } - }, { + }, + { uri: '/{domain}/sys/key_value/mathoid_ng.check', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'json' } } ] diff --git a/sys/mobileapps.js b/sys/mobileapps.js index beccad7eb..76c0fad7d 100644 --- a/sys/mobileapps.js +++ b/sys/mobileapps.js @@ -23,7 +23,6 @@ class MobileApps { uri: new URI([rp.domain, 'sys', 'key_value', BUCKET_NAME, rp.title]) }) .then((res) => { - res.body = JSON.parse(res.body.toString('utf8')); if (!rp.revision || `${mwUtils.parseETag(res.headers.etag).rev}` === `${rp.revision}`) { return res; diff --git a/sys/parsoid.js b/sys/parsoid.js index 79172e948..ae72d7266 100644 --- a/sys/parsoid.js +++ b/sys/parsoid.js @@ -206,7 +206,7 @@ class ParsoidService { uri: this.getStashBucketURI(domain, title, revision, tid) }) .then((res) => { - res = JSON.parse(res.body.toString('utf8')); + res = res.body; res.revid = revision; return res; }); @@ -330,11 +330,7 @@ class ParsoidService { */ _getContentWithFallback(hyper, domain, title, revision, tid) { if (!revision && !tid) { - return hyper.get({ uri: this.getLatestBucketURI(domain, title) }) - .then((res) => { - res.body = JSON.parse(res.body.toString('utf8')); - return res; - }); + return hyper.get({ uri: this.getLatestBucketURI(domain, title) }); } else if (!tid) { return hyper.get({ uri: this.getLatestBucketURI(domain, title) }) .then((res) => { @@ -342,17 +338,12 @@ class ParsoidService { if (revision !== resEtag.rev) { throw new HTTPError({ status: 404 }); } - res.body = JSON.parse(res.body.toString('utf8')); return res; }); } else { return hyper.get({ uri: this.getStashBucketURI(domain, title, revision, tid) }) - .then((res) => { - res.body = JSON.parse(res.body.toString('utf8')); - return res; - }) .catch({ status: 404 }, () => hyper.get({ uri: this.getLatestBucketURI(domain, title) }) .then((res) => { @@ -360,7 +351,6 @@ class ParsoidService { if (revision !== resEtag.rev || tid !== resEtag.tid) { throw new HTTPError({ status: 404 }); } - res.body = JSON.parse(res.body.toString('utf8')); return res; }) ); @@ -817,12 +807,18 @@ module.exports = (options) => { resources: [ { uri: '/{domain}/sys/key_value/parsoid', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'blob' } }, { uri: '/{domain}/sys/key_value/parsoid-stash', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'blob', default_time_to_live: options.grace_ttl diff --git a/sys/post_data.js b/sys/post_data.js index a671ed383..827a2f6d8 100644 --- a/sys/post_data.js +++ b/sys/post_data.js @@ -57,6 +57,9 @@ class PostDataBucket { const rp = req.params; return hyper.put({ uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket]), + headers: { + 'content-type': 'application/json' + }, body: { keyType: 'string', valueType: 'json' diff --git a/test/features/buckets/key_value_bucket.js b/test/features/buckets/key_value_bucket.js index aa1195eb8..e791a3ed3 100644 --- a/test/features/buckets/key_value_bucket.js +++ b/test/features/buckets/key_value_bucket.js @@ -18,16 +18,30 @@ describe('Key value buckets', () => { } function runTests(bucketName) { - const server = new Server(`${__dirname}/../../../config.example.storage.wikimedia.yaml`); + const server = new Server(`${__dirname}/../../../config.example.storage.wikimedia.yaml`, true); let bucketBaseURI; + let stringBaseURI; before(() => server.start() .then(() => { bucketBaseURI = `${server.config.baseURL()}/${bucketName}/${bucketName}TestingBucket`; - return preq.put({ uri: bucketBaseURI} ); + return preq.put({uri: bucketBaseURI}); + }) + .then(() => { + stringBaseURI = `${server.config.baseURL()}/${bucketName}/${bucketName}StringBucket`; + return preq.put({ + uri: stringBaseURI, + headers: { + 'content-type': 'application/json' + }, + body: { + valueType: 'string' + } + }); })); after(() => server.stop()); it('stores a content in a bucket and gets it back', () => { + console.log(bucketBaseURI); const testData = randomString(100); return preq.put({ uri: `${bucketBaseURI}/${testData}`, @@ -44,6 +58,28 @@ describe('Key value buckets', () => { assert.deepEqual(res.body, new Buffer(testData)); }); }); + it('Supports text/plain', () => { + const testData = randomString(100); + return preq.put({ + uri: `${stringBaseURI}/${testData}`, + headers: { + 'content-type': 'text/plain', + 'x-store-content-type': 'text/plain' + }, + body: testData + }) + .then((res) => { + assert.deepEqual(res.status, 201); + return preq.get({ + uri: `${stringBaseURI}/${testData}` + }); + }) + .then((res) => { + assert.deepEqual(res.status, 200); + assert.deepEqual(res.body, testData); + assert.deepEqual(res.headers['content-type'], 'text/plain'); + }); + }); it('throws 404 error if key not found', () => { return preq.get({ diff --git a/test/features/buckets/revisioned_bucket.js b/test/features/buckets/revisioned_bucket.js deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/features/pagecontent/access_checks.js b/test/features/pagecontent/access_checks.js index a3f6a00e2..64dbdbdaa 100644 --- a/test/features/pagecontent/access_checks.js +++ b/test/features/pagecontent/access_checks.js @@ -53,8 +53,18 @@ describe('Access checks', () => { return server.start() // Do a preparation request to force siteinfo fetch so that we don't need to mock it .then(() => P.join( - preq.get({ uri: `${server.config.bucketURL()}/html/Main_Page` }), - preq.get({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/Main_Page` }) + preq.get({ + uri: `${server.config.bucketURL()}/html/Main_Page`, + headers: { + 'cache-control': 'no-cache' + } + }), + preq.get({ + uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/Main_Page`, + headers: { + 'cache-control': 'no-cache' + } + }) )) // Load in the revisions .then(() => { diff --git a/test/features/pagecontent/language_variants.js b/test/features/pagecontent/language_variants.js index 154c612ad..a20f50181 100644 --- a/test/features/pagecontent/language_variants.js +++ b/test/features/pagecontent/language_variants.js @@ -1,6 +1,5 @@ 'use strict'; -const P = require('bluebird'); const assert = require('../../utils/assert.js'); const preq = require('preq'); const Server = require('../../utils/server.js'); @@ -9,10 +8,9 @@ const variantsPageTitle = 'RESTBase_Testing_Page'; describe('Language variants', function() { this.timeout(20000); const server = new Server(); - const backendServer = new Server(`${__dirname}/../../../config.example.storage.wikimedia.yaml`); - before(() => P.join(server.start(), backendServer.start())); - after(() => P.join(server.stop(), backendServer.stop())); + before(() => server.start()); + after(() => server.stop()); it('should request html with impossible variants', () => { return preq.get({ uri: `${server.config.bucketURL()}/html/Main_Page`}) diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index 899beeb0a..8cfcb2bc3 100644 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -3,52 +3,52 @@ mod_dir=$( cd "$( dirname "$0" )"/../.. && pwd )/node_modules mocha="$mod_dir"/mocha/bin/mocha nyc="$mod_dir"/.bin/nyc +test_target=${TEST_TARGET:-$2} +test_mode=${TEST_MODE:-$3} -runTest ( ) { - if [ "$1" = "cassandra" ]; then - echo "Running with Cassandra backend" - if [ `nc -z localhost 9042 < /dev/null; echo $?` != 0 ]; then - echo "Waiting for Cassandra to start..." - while [ `nc -z localhost 9042; echo $?` != 0 ]; do - sleep 1 - done - echo "Cassandra is ready." - fi - export RB_TEST_BACKEND=cassandra - sh ./test/utils/cleandb.sh local_group_test - else - echo "Running with SQLite backend" - export RB_TEST_BACKEND=sqlite - rm -f test.db.sqlite3 - fi +if [ "$1" = "test" ]; then + test_command="${mocha}" +elif [ "$1" = "coverage" ]; then + test_command="${nyc}" --reporter=lcov node_modules/.bin/_mocha +else + echo "Invalid test command. Must be 'test' or 'coverage" + exit 1 +fi - if [ "$2" = "test" ]; then - "${mocha}" - elif [ "$2" = "coverage" ]; then - "${nyc}" --reporter=lcov node_modules/.bin/_mocha +if [ "x$test_target" = "x" ] || [ "$test_target" = "sqlite" ]; then + echo "Running with SQLite backend" + rm -f test.db.sqlite3 +elif [ "$test_target" = "cassandra" ]; then + echo "Running with Cassandra backend" + if [ `nc -z localhost 9042 < /dev/null; echo $?` != 0 ]; then + echo "Waiting for Cassandra to start..." + while [ `nc -z localhost 9042; echo $?` != 0 ]; do + sleep 1 + done + echo "Cassandra is ready." fi -} + export RB_TEST_BACKEND=cassandra + sh ./test/utils/cleandb.sh local_group_test +else + echo "Invalid TEST_TARGET ${test_target}. Must me 'sqlite' or 'cassandra' if specified" + exit 1 +fi -if [ "x$2" = "x" ]; then - # no concrete backend is provided, check for cassandra - `echo exit;` | cqlsh 2> /dev/null - if [ "$?" -eq 0 ]; then - runTest "cassandra" $1 - else - echo "Cassandra not available. Using SQLite backend for tests" - runTest "sqlite" $1 - fi -elif [ "$2" = "sqlite" ]; then - runTest "sqlite" $1 -elif [ "$2" = "cassandra" ]; then - runTest "cassandra" $1 -elif [ "$2" = "all" ]; then - runTest "cassandra" $1 - cassandra_result=$? - runTest "sqlite" $1 - sqlite_result=$? - exit $(($cassandra_result + $sqlite_result)) +if [ "x$test_mode" = "x" ] || [ "$test_mode" = "single_process" ]; then + echo "Running single process mode" +elif [ "$test_mode" = "multi_process" ]; then + echo "Running multi process mode" + export RB_TEST_BACKEND_HOST_TEMPLATE="{{\"http://localhost:7232/{domain}/v1\"}}" +else + echo "Invalid TEST_MODE ${test_mode}. Must be 'single_process' or 'multi_process' if specified" + exit 1 +fi + +if [ "$1" = "test" ]; then + "${mocha}" +elif [ "$1" = "coverage" ]; then + "${nyc}" --reporter=lcov node_modules/.bin/_mocha else - echo "Invalid testing mode" + echo "Invalid test command ${1}. Must be 'test' or 'coverage'" exit 1 fi diff --git a/test/utils/server.js b/test/utils/server.js index 8493c83fa..49a11d4c1 100644 --- a/test/utils/server.js +++ b/test/utils/server.js @@ -1,11 +1,18 @@ 'use strict'; +const P = require('bluebird'); const TestRunner = require('service-runner/test/TestServer'); const DEFAULT_DOMAIN = 'en.wikipedia.org'; class TestRestbase extends TestRunner { - constructor(configPath = `${__dirname}/../../config.test.yaml`) { + constructor(configPath = `${__dirname}/../../config.test.yaml`, forseSkipBackend) { super(configPath); + this._spinBackend = !forseSkipBackend && !!process.env.RB_TEST_BACKEND_HOST_TEMPLATE; + if (this._spinBackend) { + this._backendServer = new TestRestbase( + `${__dirname}/../../config.example.storage.wikimedia.yaml`, + true); + } } get config() { @@ -31,6 +38,16 @@ class TestRestbase extends TestRunner { conf } } + + start() { + const startPromise = this._spinBackend ? this._backendServer.start() : P.resolve(); + return startPromise.then(() => super.start()); + } + + stop() { + const stopPromise = this._spinBackend ? this._backendServer.stop() : P.resolve(); + return stopPromise.then(() => super.stop()); + } } module.exports = TestRestbase; diff --git a/v1/definition.yaml b/v1/definition.yaml index 3b28726cb..232f2a04a 100644 --- a/v1/definition.yaml +++ b/v1/definition.yaml @@ -91,6 +91,8 @@ paths: - init: method: put uri: /{domain}/sys/key_value/term.definition-ng + headers: + content-type: application/json body: valueType: json x-request-handler: @@ -124,7 +126,8 @@ paths: method: put uri: /{domain}/sys/key_value/term.definition-ng/{request.params.term} headers: - if-none-hash-match": '*' + content-type: 'application/json' + if-none-hash-match: '*' cache-control: '{{request.headers.cache-control}}' x-store-etag: '{{extract.headers.etag}}' x-store-content-language: '{{extract.headers.content-language}}' diff --git a/v1/mathoid.yaml b/v1/mathoid.yaml index 28622aa87..29a759fa7 100644 --- a/v1/mathoid.yaml +++ b/v1/mathoid.yaml @@ -191,16 +191,22 @@ paths: x-setup-handler: - init_svg: uri: /wikimedia.org/sys/key_value/mathoid_ng.svg + headers: + content-type: 'application/json' body: keyType: string valueType: string - init_mml: uri: /wikimedia.org/sys/key_value/mathoid_ng.mml + headers: + content-type: 'application/json' body: keyType: string valueType: string - init_png: uri: /wikimedia.org/sys/key_value/mathoid_ng.png + headers: + content-type: 'application/json' body: keyType: string valueType: blob @@ -208,7 +214,7 @@ paths: - check_storage: request: method: get - uri: /wikimedia.org/sys/key_value/mathoid_ng.{$.request.params.format}/{$.request.params.hash} + uri: /wikimedia.org/sys/key_value/mathoid_ng.{format}/{hash} headers: cache-control: '{{ cache-control }}' catch: diff --git a/v1/summary_new.yaml b/v1/summary_new.yaml index 4ca9ac484..fa17e3fb0 100644 --- a/v1/summary_new.yaml +++ b/v1/summary_new.yaml @@ -94,7 +94,7 @@ paths: # Set up a simple key-value bucket. - init: method: 'put' - uri: /{domain}/sys/proxy/key_value/page_summary + uri: /{domain}/sys/key_value/page_summary headers: content-type: application/json body: @@ -105,7 +105,7 @@ paths: method: get headers: cache-control: '{{cache-control}}' - uri: /{domain}/sys/proxy/key_value/page_summary/{request.params.title} + uri: /{domain}/sys/key_value/page_summary/{request.params.title} catch: status: 404 return_if: @@ -135,7 +135,7 @@ paths: - store_and_return: request: method: put - uri: /{domain}/sys/proxy/key_value/page_summary/{request.params.title} + uri: /{domain}/sys/key_value/page_summary/{request.params.title} headers: if-none-hash-match: '*' content-type: 'application/json' From 54e4a6fa0d81f48ddd41a637bdf70b228ca173a0 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Wed, 29 May 2019 11:31:51 +0300 Subject: [PATCH 4/9] Provide a default for backend_host_template Change-Id: I791ef44a398981cfe8d2d0cae363dab8b03d5702 --- sys/backend_proxy.js | 1 + 1 file changed, 1 insertion(+) diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js index 49d5ecd02..4e2fe42ff 100644 --- a/sys/backend_proxy.js +++ b/sys/backend_proxy.js @@ -3,6 +3,7 @@ const Template = require('hyperswitch').Template; module.exports = (options) => { + options.backend_host_template = options.backend_host_template || '/{domain}/sys/legacy'; const backendURITemplate = new Template({ uri: `${options.backend_host_template}/{{path}}` }); From 0e1a6cbecb252ccf06dd023276bfa1e3ce513930 Mon Sep 17 00:00:00 2001 From: Marko Obrovac Date: Fri, 31 May 2019 14:32:14 +0200 Subject: [PATCH 5/9] Consolidate sys/ projects Until now, we would just be adding paths to sys/ in each project as they would change. However, most of these paths for most projects would be exactly the same. This commit consolidates them and groups them in three sub-projects: - business: a sub-project containing the business-logic modules (parsoid, events, etc.) - backend-storage: a sub-project containing back-end storage logic (table, action, page_revisions, key_value) - backend-proxy: a sub-project containing sys/ paths that need proxying from the front-end service to the back-end service. This commit also introduces the `service_type` configuration variable that is used to choose between which modules to load. Its possible values are `frontend`, `backend` and `fullstack`. For now, it selects between backend-storage and backend-proxy modules, but in the future it will also be used to decide whether to load the business sub-project too. --- config.example.storage.wikimedia.yaml | 40 ++++---- config.example.wikimedia.yaml | 3 +- config.test.yaml | 3 +- lib/loader_module.js | 23 +++++ projects/sys/backend-proxy.yaml | 10 ++ projects/sys/backend-storage.yaml | 16 ++++ projects/sys/backend.js | 25 +++++ projects/sys/business.js | 10 ++ projects/sys/business.yaml | 128 ++++++++++++++++++++++++++ projects/wikimedia.org.yaml | 16 +--- projects/wmf_default.yaml | 61 +----------- projects/wmf_enwiki.yaml | 86 +---------------- projects/wmf_wikidata.yaml | 47 +--------- projects/wmf_wikipedia.yaml | 107 +-------------------- projects/wmf_wikivoyage.yaml | 49 +--------- projects/wmf_wiktionary.yaml | 50 +--------- sys/backend_proxy.js | 3 +- test/utils/run_tests.sh | 10 +- test/utils/server.js | 4 +- 19 files changed, 262 insertions(+), 429 deletions(-) create mode 100644 lib/loader_module.js create mode 100644 projects/sys/backend-proxy.yaml create mode 100644 projects/sys/backend-storage.yaml create mode 100644 projects/sys/backend.js create mode 100644 projects/sys/business.js create mode 100644 projects/sys/business.yaml diff --git a/config.example.storage.wikimedia.yaml b/config.example.storage.wikimedia.yaml index c59bf7377..eee13ca7d 100644 --- a/config.example.storage.wikimedia.yaml +++ b/config.example.storage.wikimedia.yaml @@ -18,29 +18,23 @@ default_project: &default_project - path: sys/page_revisions.js /{api:sys}: x-modules: - - spec: - paths: - /table: - x-modules: - - path: sys/table.js - options: - conf: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: test.group.local - domains: /./ - dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing - /action: - x-modules: - - path: sys/action.js - options: - apiUriTemplate: "{{'https://{domain}/w/api.php'}}" - baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + - path: projects/sys/backend.js + options: + service_type: backend + table: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: group.local + domains: /./ + dbname: db.sqlite3 # ignored in cassandra, but useful in SQLite testing + action: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" # The root of the spec tree. Domains tend to share specs by referencing them diff --git a/config.example.wikimedia.yaml b/config.example.wikimedia.yaml index e10c11f51..b8c46103f 100644 --- a/config.example.wikimedia.yaml +++ b/config.example.wikimedia.yaml @@ -6,8 +6,9 @@ default_project: &default_project x-modules: - path: projects/wmf_default.yaml options: &default_options + service_type: '{env(RB_SERVICE_TYPE, fullstack)}' proxy: - backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys/legacy"}})}' + backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys"}})}' table: backend: '{env(RB_TEST_BACKEND, sqlite)}' hosts: [localhost] diff --git a/config.test.yaml b/config.test.yaml index 0d7f756e3..13fa51435 100644 --- a/config.test.yaml +++ b/config.test.yaml @@ -7,8 +7,9 @@ default_project: &default_project - path: test/test_module.yaml - path: projects/wmf_default.yaml options: &default_options + service_type: '{env(RB_SERVICE_TYPE, fullstack)}' proxy: - backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys/legacy"}})}' + backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys"}})}' table: backend: '{env(RB_TEST_BACKEND, sqlite)}' hosts: [localhost] diff --git a/lib/loader_module.js b/lib/loader_module.js new file mode 100644 index 000000000..1f36811bb --- /dev/null +++ b/lib/loader_module.js @@ -0,0 +1,23 @@ +'const strict'; + +module.exports = (spec, options) => { + options = options || {}; + spec = spec || {}; + spec.paths = spec.paths || {}; + if (options.exclude) { + if (!Array.isArray(options.exclude)) { + options.exclude = options.exclude.split(','); + } + options.exclude.forEach((module) => { + module = /^\//.test(module) ? module : `/${module}`; + if (spec.paths[module]) { + delete spec.paths[module]; + } + }); + } + return { + spec, + operations: {}, + globals: { options } + }; +}; diff --git a/projects/sys/backend-proxy.yaml b/projects/sys/backend-proxy.yaml new file mode 100644 index 000000000..3358b31e2 --- /dev/null +++ b/projects/sys/backend-proxy.yaml @@ -0,0 +1,10 @@ +paths: + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options.proxy}}' + diff --git a/projects/sys/backend-storage.yaml b/projects/sys/backend-storage.yaml new file mode 100644 index 000000000..858b71777 --- /dev/null +++ b/projects/sys/backend-storage.yaml @@ -0,0 +1,16 @@ +paths: + /table: + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /key_value: + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js diff --git a/projects/sys/backend.js b/projects/sys/backend.js new file mode 100644 index 000000000..607336dd1 --- /dev/null +++ b/projects/sys/backend.js @@ -0,0 +1,25 @@ +'use strict'; + +const HyperSwitch = require('hyperswitch'); +const loader = require('../../lib/loader_module.js'); + +const specStorage = HyperSwitch.utils.loadSpec(`${__dirname}/backend-storage.yaml`); +const specProxy = HyperSwitch.utils.loadSpec(`${__dirname}/backend-proxy.yaml`); + +module.exports = (options) => { + let spec; + options = options || {}; + options.service_type = options.service_type || 'fullstack'; + switch (options.service_type.toLowerCase()) { + case 'frontend': + spec = specProxy; + break; + case 'backend': + case 'fullstack': + spec = specStorage; + break; + default: + throw new Error('Invalid value for option service_type supplied!'); + } + return loader(spec, options); +}; diff --git a/projects/sys/business.js b/projects/sys/business.js new file mode 100644 index 000000000..3380611da --- /dev/null +++ b/projects/sys/business.js @@ -0,0 +1,10 @@ +'use strict'; + +const HyperSwitch = require('hyperswitch'); +const loader = require('../../lib/loader_module.js'); + +const spec = HyperSwitch.utils.loadSpec(`${__dirname}/business.yaml`); + +module.exports = (options) => { + return loader(spec, options); +}; diff --git a/projects/sys/business.yaml b/projects/sys/business.yaml new file mode 100644 index 000000000..0562b7d89 --- /dev/null +++ b/projects/sys/business.yaml @@ -0,0 +1,128 @@ +paths: + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /post_data: + x-modules: + - path: sys/post_data.js + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + # en wiki + en.wikipedia.org: + - 'User:B-bot/Event_log' + - 'User:DeltaQuad/UAA/Wait' + - 'User:JamesR/AdminStats' + - 'User:Kudpung/Dashboard' + # Various dashboards + - 'User:Breawycker/Wikipedia' + - 'User:Sonia/dashboard' + - 'User:Ocaasi/dashboard' + - 'User:Nolelover' + - 'User:Calmer_Waters' + - '/User%3ARedwolf24\//' + - 'User:Technical_13/dashboard' + - 'Template:Cratstats' + # Cyberbot is creating 90% of null edits + - '/^User:Cyberbot_I\//' + - '/^User:Cyberbot_II\//' + - '/^User:Cyberpower678\//' + - '/^User:Darts170Darts170\//' + - 'صارف:Cyberbot_I/Run/Adminstats' + - 'Defnyddiwr:Cyberbot_I/Run/Adminstats' + - 'User:Pentjuuu!.!/sandbox' + - 'User:AllyD/CSDlog' + - 'User:Peter_I._Vardy/sandbox-13' + - 'User:I_dream_of_horses/CSD_log' + - 'User:MJ180MJ180/sandbox' + - 'Talk:United_States_presidential_election,_2016' + - 'Wikipedia:Reference_desk/Humanities' + - 'Wikipedia:WikiProject_Deletion_sorting/People' + - 'Wikipedia:WikiProject_Deletion_sorting/United_States_of_America' + - 'Wikipedia:Articles_for_creation/Redirects' + - 'Wikipedia:Administrators%27_noticeboard/Incidents' + # Wikipedia + ca.wikipedia.org: + - 'Usuari:TronaBot/log:Activitat_reversors_per_hores' + ceb.wikipedia.org: + - 'Gumagamit:Lsjbot/Anomalier-PRIVAT' + - 'Gumagamit:Lsjbot/Kartrutor2' + de.wikipedia.org: + - '/The_Big_Bang_Theory\/Staffel/' + - 'Wikipedia:Café' + - 'Wikipedia:Defekte_Weblinks/Bot2015-Problem' + - 'Wikipedia_Diskussion:Hauptseite/Schon_gewusst' + - 'Benutzer:Anglo-Araneophilus/Almigdad_Mojalli' + - 'Benutzer:Wartungsstube/Berlin' + - 'Benutzer:Wartungsstube/Musik' + - 'Benutzer:Wartungsstube/Unternehmen' + - 'Benutzer:Wartungsstube/Schifffahrt' + - 'Benutzer:Verum/ege' + - 'Benutzer:Septembermorgen/Bottabelle/Französische_Kantone_N–Z' + - 'Wikipedia:WikiProjekt_Planen_und_Bauen/Zu_überarbeitende_Artikel' + es.wikipedia.org: + - 'Wikipedia:Café/Archivo/Miscelánea/Actual' + fr.wikipedia.org: + - 'Utilisateur:ZéroBot/Log/Erreurs' + - 'Utilisateur:SyntaxTerror/Ajouts_du_modèle_Autorité' + - '/^Utilisateur:[\s\S]+[Bb]rouillon' + - 'Discussion_utilisateur:NaggoBot/CommonsDR' + - 'Projet:France/Annonces/Admissibilité' + - '/Wikipédia:Le_saviez-vous_.+/Anecdotes_proposées/' + hy.wikipedia.org: + - "/Մասնակից:Omicroñ\\'R/" + it.wikipedia.org: + - 'Utente:Effems/Sandbox7' + nl.wikipedia.org: + - 'Gebruiker:Eg-T2g/Kladblok' + pt.wikipedia.org: + - 'Wikipédia:Pedidos/Bloqueio' + ru.wikipedia.org: + - 'Википедия:Форум/Технический' + - 'Портал:Герпетология' + sv.wikipedia.org: + - 'Användare:Lsjbot/Anomalier-PRIVAT' + - 'Användare:Lsjbot/Namnkonflikter-PRIVAT' + ur.wikipedia.org: + - 'نام_مقامات_ایل' + - 'نام_مقامات_ڈی' + - 'نام_مقامات_جے' + - 'نام_مقامات_جی' + - 'نام_مقامات_ایچ' + - 'نام_مقامات_ایم' + - 'نام_مقامات_ایس' + zh.wikipedia.org: + - 'Wikipedia:互助客栈/条目探讨' + - 'Draft:日本人工湖列表' + # Wikisource + pl.wikisource.org: + - '/^Wśród_czarnych\//' + # Wikimedia + commons.wikimedia.org: + - '/Commons:Featured_picture_candidates\//' + - 'Commons:Quality_images/Subject/Places/Natural_structures' + - '/Commons:Undeletion_requests\//' + - '/Commons:WikiProject_Aviation\/recent_uploads\//' + - '/^(?:User|Benutzer):/' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml index 33cafbd50..a9a0835eb 100644 --- a/projects/wikimedia.org.yaml +++ b/projects/wikimedia.org.yaml @@ -81,25 +81,15 @@ paths: /{api:sys}: x-modules: + - path: projects/sys/backend.js + options: '{{options}}' - spec: paths: /mathoid: x-modules: - path: sys/mathoid.js options: '{{options.mathoid}}' - /table: &sys_table - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /key_value: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/key_value: - x-modules: - - path: sys/key_value.js - /post_data: &sys_post_data + /post_data: x-modules: - path: sys/post_data.js /events: diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml index c47b3587a..fd8213678 100644 --- a/projects/wmf_default.yaml +++ b/projects/wmf_default.yaml @@ -105,60 +105,7 @@ paths: /{api:sys}: x-modules: - - spec: - paths: - /table: &sys_table - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /key_value: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/page_revisions: - x-modules: - - path: sys/page_revisions.js - /post_data: &sys_post_data - x-modules: - - path: sys/post_data.js - /action: - x-modules: - - path: sys/action.js - options: "{{options.action}}" - /page_save: - x-modules: - - path: sys/page_save.js - /parsoid: - x-modules: - - path: sys/parsoid.js - options: &parsoid_options - parsoidHost: '{{options.parsoid.host}}' - response_cache_control: '{{options.purged_cache_control}}' - grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' - # A list of pages that we don't currently want to re-render on - # each edit. Most of these are huge bot-edited pages, which are - # rarely viewed in any case. - rerenderBlacklist: - # Wikisource - pl.wikisource.org: - - '/^Wśród_czarnych\//' - # Wikimedia - commons.wikimedia.org: - - '/Commons:Featured_picture_candidates\//' - - 'Commons:Quality_images/Subject/Places/Natural_structures' - - '/Commons:Undeletion_requests\//' - - '/Commons:WikiProject_Aviation\/recent_uploads\//' - - '/^(?:User|Benutzer):/' - /events: - x-modules: - - path: sys/events.js - options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' - options: '{{options}}' + - path: projects/sys/business.js + options: '{{merge(options, {"exclude": "mobileapps"})}}' + - path: projects/sys/backend.js + options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml index 8bfc041f0..8694ac1ad 100644 --- a/projects/wmf_enwiki.yaml +++ b/projects/wmf_enwiki.yaml @@ -132,87 +132,7 @@ paths: /{api:sys}: x-modules: - - spec: - paths: - /table: - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /key_value: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/page_revisions: - x-modules: - - path: sys/page_revisions.js - /post_data: &sys_post_data - x-modules: - - path: sys/post_data.js - /action: - x-modules: - - path: sys/action.js - options: "{{options.action}}" - /page_save: - x-modules: - - path: sys/page_save.js - /parsoid: - x-modules: - - path: sys/parsoid.js - options: &parsoid_options - parsoidHost: '{{options.parsoid.host}}' - response_cache_control: '{{options.purged_cache_control}}' - grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' - # A list of pages that we don't currently want to re-render on - # each edit. Most of these are huge bot-edited pages, which are - # rarely viewed in any case. - rerenderBlacklist: - en.wikipedia.org: - - 'User:B-bot/Event_log' - - 'User:DeltaQuad/UAA/Wait' - - 'User:JamesR/AdminStats' - - 'User:Kudpung/Dashboard' - # Various dashboards - - 'User:Breawycker/Wikipedia' - - 'User:Sonia/dashboard' - - 'User:Ocaasi/dashboard' - - 'User:Nolelover' - - 'User:Calmer_Waters' - - '/User%3ARedwolf24\//' - - 'User:Technical_13/dashboard' - - 'Template:Cratstats' - # Cyberbot is creating 90% of null edits - - '/^User:Cyberbot_I\//' - - '/^User:Cyberbot_II\//' - - '/^User:Cyberpower678\//' - - '/^User:Darts170Darts170\//' - - 'صارف:Cyberbot_I/Run/Adminstats' - - 'Defnyddiwr:Cyberbot_I/Run/Adminstats' - - 'User:Pentjuuu!.!/sandbox' - - 'User:AllyD/CSDlog' - - 'User:Peter_I._Vardy/sandbox-13' - - 'User:I_dream_of_horses/CSD_log' - - 'User:MJ180MJ180/sandbox' - - 'Talk:United_States_presidential_election,_2016' - - 'Wikipedia:Reference_desk/Humanities' - - 'Wikipedia:WikiProject_Deletion_sorting/People' - - 'Wikipedia:WikiProject_Deletion_sorting/United_States_of_America' - - 'Wikipedia:Articles_for_creation/Redirects' - - 'Wikipedia:Administrators%27_noticeboard/Incidents' - /mobileapps: - x-modules: - - path: sys/mobileapps.js - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - /events: - x-modules: - - path: sys/events.js - options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + - path: projects/sys/business.js options: '{{options}}' + - path: projects/sys/backend.js + options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml index 264120acc..c48cd1073 100644 --- a/projects/wmf_wikidata.yaml +++ b/projects/wmf_wikidata.yaml @@ -87,46 +87,7 @@ paths: /{api:sys}: x-modules: - - spec: - paths: - /table: &sys_table - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /key_value: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/page_revisions: - x-modules: - - path: sys/page_revisions.js - /post_data: &sys_post_data - x-modules: - - path: sys/post_data.js - /action: - x-modules: - - path: sys/action.js - options: "{{options.action}}" - /page_save: - x-modules: - - path: sys/page_save.js - /parsoid: - x-modules: - - path: sys/parsoid.js - options: &parsoid_options - parsoidHost: '{{options.parsoid.host}}' - response_cache_control: '{{options.purged_cache_control}}' - grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' - /events: - x-modules: - - path: sys/events.js - options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' - options: '{{options}}' + - path: projects/sys/business.js + options: '{{merge(options, {"exclude": "mobileapps"})}}' + - path: projects/sys/backend.js + options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wikipedia.yaml b/projects/wmf_wikipedia.yaml index 5eac81f71..8694ac1ad 100644 --- a/projects/wmf_wikipedia.yaml +++ b/projects/wmf_wikipedia.yaml @@ -132,108 +132,7 @@ paths: /{api:sys}: x-modules: - - spec: - paths: - /table: &sys_table - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /key_value: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/page_revisions: - x-modules: - - path: sys/page_revisions.js - /post_data: &sys_post_data - x-modules: - - path: sys/post_data.js - /action: - x-modules: - - path: sys/action.js - options: "{{options.action}}" - /page_save: - x-modules: - - path: sys/page_save.js - /parsoid: - x-modules: - - path: sys/parsoid.js - options: &parsoid_options - parsoidHost: '{{options.parsoid.host}}' - response_cache_control: '{{options.purged_cache_control}}' - grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' - # A list of pages that we don't currently want to re-render on - # each edit. Most of these are huge bot-edited pages, which are - # rarely viewed in any case. - rerenderBlacklist: - # Wikipedia - ca.wikipedia.org: - - 'Usuari:TronaBot/log:Activitat_reversors_per_hores' - ceb.wikipedia.org: - - 'Gumagamit:Lsjbot/Anomalier-PRIVAT' - - 'Gumagamit:Lsjbot/Kartrutor2' - de.wikipedia.org: - - '/The_Big_Bang_Theory\/Staffel/' - - 'Wikipedia:Café' - - 'Wikipedia:Defekte_Weblinks/Bot2015-Problem' - - 'Wikipedia_Diskussion:Hauptseite/Schon_gewusst' - - 'Benutzer:Anglo-Araneophilus/Almigdad_Mojalli' - - 'Benutzer:Wartungsstube/Berlin' - - 'Benutzer:Wartungsstube/Musik' - - 'Benutzer:Wartungsstube/Unternehmen' - - 'Benutzer:Wartungsstube/Schifffahrt' - - 'Benutzer:Verum/ege' - - 'Benutzer:Septembermorgen/Bottabelle/Französische_Kantone_N–Z' - - 'Wikipedia:WikiProjekt_Planen_und_Bauen/Zu_überarbeitende_Artikel' - es.wikipedia.org: - - 'Wikipedia:Café/Archivo/Miscelánea/Actual' - fr.wikipedia.org: - - 'Utilisateur:ZéroBot/Log/Erreurs' - - 'Utilisateur:SyntaxTerror/Ajouts_du_modèle_Autorité' - - '/^Utilisateur:[\s\S]+[Bb]rouillon' - - 'Discussion_utilisateur:NaggoBot/CommonsDR' - - 'Projet:France/Annonces/Admissibilité' - - '/Wikipédia:Le_saviez-vous_.+/Anecdotes_proposées/' - hy.wikipedia.org: - - "/Մասնակից:Omicroñ\\'R/" - it.wikipedia.org: - - 'Utente:Effems/Sandbox7' - nl.wikipedia.org: - - 'Gebruiker:Eg-T2g/Kladblok' - pt.wikipedia.org: - - 'Wikipédia:Pedidos/Bloqueio' - ru.wikipedia.org: - - 'Википедия:Форум/Технический' - - 'Портал:Герпетология' - sv.wikipedia.org: - - 'Användare:Lsjbot/Anomalier-PRIVAT' - - 'Användare:Lsjbot/Namnkonflikter-PRIVAT' - ur.wikipedia.org: - - 'نام_مقامات_ایل' - - 'نام_مقامات_ڈی' - - 'نام_مقامات_جے' - - 'نام_مقامات_جی' - - 'نام_مقامات_ایچ' - - 'نام_مقامات_ایم' - - 'نام_مقامات_ایس' - zh.wikipedia.org: - - 'Wikipedia:互助客栈/条目探讨' - - 'Draft:日本人工湖列表' - /mobileapps: - x-modules: - - path: sys/mobileapps.js - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - /events: - x-modules: - - path: sys/events.js - options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + - path: projects/sys/business.js options: '{{options}}' + - path: projects/sys/backend.js + options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wikivoyage.yaml b/projects/wmf_wikivoyage.yaml index 9938340f9..92e944c94 100644 --- a/projects/wmf_wikivoyage.yaml +++ b/projects/wmf_wikivoyage.yaml @@ -127,50 +127,7 @@ paths: /{api:sys}: x-modules: - - spec: - paths: - /table: &sys_table - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /key_value: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/page_revisions: - x-modules: - - path: sys/page_revisions.js - /post_data: &sys_post_data - x-modules: - - path: sys/post_data.js - /action: - x-modules: - - path: sys/action.js - options: "{{options.action}}" - /page_save: - x-modules: - - path: sys/page_save.js - /parsoid: - x-modules: - - path: sys/parsoid.js - options: &parsoid_options - parsoidHost: '{{options.parsoid.host}}' - response_cache_control: '{{options.purged_cache_control}}' - grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' - /mobileapps: - x-modules: - - path: sys/mobileapps.js - options: '{{merge({"response_cache_control": options.purged_cache_control}, options.mobileapps)}}' - /events: - x-modules: - - path: sys/events.js - options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + - path: projects/sys/business.js options: '{{options}}' + - path: projects/sys/backend.js + options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml index 2e24c1a30..2e96656fc 100644 --- a/projects/wmf_wiktionary.yaml +++ b/projects/wmf_wiktionary.yaml @@ -121,51 +121,7 @@ paths: /{api:sys}: x-modules: - - spec: - paths: - /table: &sys_table - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /key_value: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - x-modules: - - path: sys/backend_proxy.js - options: '{{options.proxy}}' - /legacy/page_revisions: - x-modules: - - path: sys/page_revisions.js - /post_data: &sys_post_data - x-modules: - - path: sys/post_data.js - /action: - x-modules: - - path: sys/action.js - options: "{{options.action}}" - /page_save: - x-modules: - - path: sys/page_save.js - /parsoid: - x-modules: - - path: sys/parsoid.js - options: &parsoid_options - parsoidHost: '{{options.parsoid.host}}' - response_cache_control: '{{options.purged_cache_control}}' - grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' - /mobileapps: - x-modules: - - path: sys/mobileapps.js - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - /events: - x-modules: - - path: sys/events.js - options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + - path: projects/sys/business.js options: '{{options}}' + - path: projects/sys/backend.js + options: '{{merge(options, {"exclude": "action"})}}' diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js index 4e2fe42ff..d3099e65c 100644 --- a/sys/backend_proxy.js +++ b/sys/backend_proxy.js @@ -3,7 +3,8 @@ const Template = require('hyperswitch').Template; module.exports = (options) => { - options.backend_host_template = options.backend_host_template || '/{domain}/sys/legacy'; + options = options || {}; + options.backend_host_template = options.backend_host_template || '/{domain}/sys'; const backendURITemplate = new Template({ uri: `${options.backend_host_template}/{{path}}` }); diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index 8cfcb2bc3..8ddbfda3e 100644 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -44,11 +44,5 @@ else exit 1 fi -if [ "$1" = "test" ]; then - "${mocha}" -elif [ "$1" = "coverage" ]; then - "${nyc}" --reporter=lcov node_modules/.bin/_mocha -else - echo "Invalid test command ${1}. Must be 'test' or 'coverage'" - exit 1 -fi +${test_command}; +exit $?; diff --git a/test/utils/server.js b/test/utils/server.js index 49a11d4c1..d88172dd2 100644 --- a/test/utils/server.js +++ b/test/utils/server.js @@ -5,9 +5,9 @@ const TestRunner = require('service-runner/test/TestServer'); const DEFAULT_DOMAIN = 'en.wikipedia.org'; class TestRestbase extends TestRunner { - constructor(configPath = `${__dirname}/../../config.test.yaml`, forseSkipBackend) { + constructor(configPath = `${__dirname}/../../config.test.yaml`, forceSkipBackend) { super(configPath); - this._spinBackend = !forseSkipBackend && !!process.env.RB_TEST_BACKEND_HOST_TEMPLATE; + this._spinBackend = !forceSkipBackend && !!process.env.RB_TEST_BACKEND_HOST_TEMPLATE; if (this._spinBackend) { this._backendServer = new TestRestbase( `${__dirname}/../../config.example.storage.wikimedia.yaml`, From fb542d5e4b25558687ce5740ff14d932df92fe87 Mon Sep 17 00:00:00 2001 From: Marko Obrovac Date: Sat, 1 Jun 2019 15:49:45 +0200 Subject: [PATCH 6/9] Add the v1 proxy for key_value and page_revisions v1/proxy.js adds the `v1/key_value` and `v1/page_revisions` end points for backend and fullstack configurations and it keeps the specification unchanged for frontend configurations. It reuses the sys/backend-proxy sub-project for the end point definitions, thus ensuring the symmetry between frontend and fullstack configurations. The backend_proxy sys module is made generic to allow requests coming for both v1/ and sys/ namespaces. Additionally, it ensures that if it is publicly exposed, a 403 is returned for external requests. --- config.example.storage.wikimedia.yaml | 51 +++++++++---------------- projects/backend.yaml | 9 +++++ projects/wikimedia.org.yaml | 2 + projects/wmf_default.yaml | 2 + projects/wmf_enwiki.yaml | 2 + projects/wmf_wikidata.yaml | 2 + projects/wmf_wikipedia.yaml | 2 + projects/wmf_wikivoyage.yaml | 2 + projects/wmf_wiktionary.yaml | 2 + sys/backend_proxy.js | 44 ++++++++++++++++----- v1/proxy.js | 55 +++++++++++++++++++++++++++ 11 files changed, 130 insertions(+), 43 deletions(-) create mode 100644 projects/backend.yaml create mode 100644 v1/proxy.js diff --git a/config.example.storage.wikimedia.yaml b/config.example.storage.wikimedia.yaml index eee13ca7d..e26c76adc 100644 --- a/config.example.storage.wikimedia.yaml +++ b/config.example.storage.wikimedia.yaml @@ -1,40 +1,23 @@ # RESTBase wikimedia example config default_project: &default_project x-modules: - - spec: - paths: - /{api:v1}: - x-modules: - - spec: - paths: - /key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - # We need to forward cookie to the API - x-route-filters: - - path: ./lib/mediawiki_auth_filter.js - x-modules: - - path: sys/page_revisions.js - /{api:sys}: - x-modules: - - path: projects/sys/backend.js - options: - service_type: backend - table: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: group.local - domains: /./ - dbname: db.sqlite3 # ignored in cassandra, but useful in SQLite testing - action: - apiUriTemplate: "{{'https://{domain}/w/api.php'}}" - baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + - path: projects/backend.yaml + options: + service_type: backend + table: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: group.local + domains: /./ + dbname: db.sqlite3 # ignored in cassandra, but useful in SQLite testing + action: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" # The root of the spec tree. Domains tend to share specs by referencing them diff --git a/projects/backend.yaml b/projects/backend.yaml new file mode 100644 index 000000000..825ebee53 --- /dev/null +++ b/projects/backend.yaml @@ -0,0 +1,9 @@ +paths: + /{api:v1}: + x-modules: + - path: v1/proxy.js + options: '{{options}}' + /{api:sys}: + x-modules: + - path: projects/sys/backend.js + options: '{{options}}' diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml index a9a0835eb..e713feac9 100644 --- a/projects/wikimedia.org.yaml +++ b/projects/wikimedia.org.yaml @@ -78,6 +78,8 @@ paths: - path: v1/availability.yaml options: '{{options.mobileapps}}' options: '{{options}}' + - path: v1/proxy.js + options: '{{options}}' /{api:sys}: x-modules: diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml index fd8213678..877bf757e 100644 --- a/projects/wmf_default.yaml +++ b/projects/wmf_default.yaml @@ -102,6 +102,8 @@ paths: options: host: '{{options.mobileapps.host}}' options: '{{options}}' + - path: v1/proxy.js + options: '{{options}}' /{api:sys}: x-modules: diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml index 8694ac1ad..560c6388c 100644 --- a/projects/wmf_enwiki.yaml +++ b/projects/wmf_enwiki.yaml @@ -129,6 +129,8 @@ paths: options: host: '{{options.mobileapps.host}}' options: '{{options}}' + - path: v1/proxy.js + options: '{{options}}' /{api:sys}: x-modules: diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml index c48cd1073..554f5238c 100644 --- a/projects/wmf_wikidata.yaml +++ b/projects/wmf_wikidata.yaml @@ -84,6 +84,8 @@ paths: - path: v1/lists.js options: '{{options.lists}}' options: '{{options}}' + - path: v1/proxy.js + options: '{{options}}' /{api:sys}: x-modules: diff --git a/projects/wmf_wikipedia.yaml b/projects/wmf_wikipedia.yaml index 8694ac1ad..560c6388c 100644 --- a/projects/wmf_wikipedia.yaml +++ b/projects/wmf_wikipedia.yaml @@ -129,6 +129,8 @@ paths: options: host: '{{options.mobileapps.host}}' options: '{{options}}' + - path: v1/proxy.js + options: '{{options}}' /{api:sys}: x-modules: diff --git a/projects/wmf_wikivoyage.yaml b/projects/wmf_wikivoyage.yaml index 92e944c94..5f5dc7c19 100644 --- a/projects/wmf_wikivoyage.yaml +++ b/projects/wmf_wikivoyage.yaml @@ -124,6 +124,8 @@ paths: options: host: '{{options.mobileapps.host}}' options: '{{options}}' + - path: v1/proxy.js + options: '{{options}}' /{api:sys}: x-modules: diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml index 2e96656fc..27a3c5b16 100644 --- a/projects/wmf_wiktionary.yaml +++ b/projects/wmf_wiktionary.yaml @@ -118,6 +118,8 @@ paths: options: host: '{{options.mobileapps.host}}' options: '{{options}}' + - path: v1/proxy.js + options: '{{options}}' /{api:sys}: x-modules: diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js index d3099e65c..a96e3987e 100644 --- a/sys/backend_proxy.js +++ b/sys/backend_proxy.js @@ -1,32 +1,58 @@ 'use strict'; -const Template = require('hyperswitch').Template; +const P = require('bluebird'); +const HyperSwitch = require('hyperswitch'); + +const Template = HyperSwitch.Template; +const HTTPError = HyperSwitch.HTTPError; module.exports = (options) => { options = options || {}; options.backend_host_template = options.backend_host_template || '/{domain}/sys'; + if (!Object.prototype.hasOwnProperty.call(options, 'use_path_segment')) { + options.use_path_segment = true; + } + if (!Object.prototype.hasOwnProperty.call(options, 'block_external_reqs')) { + options.block_external_reqs = true; + } const backendURITemplate = new Template({ uri: `${options.backend_host_template}/{{path}}` }); + const usePathSegment = options.use_path_segment; + const blockExternalReqs = options.block_external_reqs; return { spec: { paths: { '/{+path}': { all: { - operationId: 'proxy' + operationId: 'proxy', + 'x-monitor': false } } } }, operations: { proxy: (hyper, req) => { - // Add the proxied module name to the path. - // The proxy is mounted at paths like `/sys/key_value`. Here - // out of a request path we want to find out the name of the module - // that is being proxied (key_value) from the example and add it - // to the proxy target path. - const modName = req.uri.path[req.uri.path.indexOf('sys') + 1]; - req.params.path = `${modName}/${req.params.path}`; + if (blockExternalReqs && !hyper._isSysRequest(req) && + req.headers['x-request-class'] === 'external') { + return P.reject(new HTTPError({ + status: 403, + body: { + type: 'forbidden', + title: 'Forbidden', + description: 'You are not allowed to access this URI' + } + })); + } + if (usePathSegment) { + // if usePathSegment is set (true by default), then the proxy module + // will include the path segment preceding the specified path to + // construct the full request URI + const uri = req.uri.toString(); + const uriPrefix = uri.substring(0, uri.indexOf(`/${req.params.path}`)); + const segment = uriPrefix.split('/').pop(); + req.params.path = `${segment}/${req.params.path}`; + } return hyper.request({ method: req.method, uri: backendURITemplate.expand({ request: req }).uri, diff --git a/v1/proxy.js b/v1/proxy.js new file mode 100644 index 000000000..bac85d118 --- /dev/null +++ b/v1/proxy.js @@ -0,0 +1,55 @@ +'use strict'; + +const HyperSwitch = require('hyperswitch'); + +const specProxy = HyperSwitch.utils.loadSpec(`${__dirname}/../projects/sys/backend-proxy.yaml`); + +function addAuthFilter(spec) { + if (!spec.paths['/page_revisions']) { + return; + } + const pageRev = spec.paths['/page_revisions']; + pageRev['x-route-filters'] = pageRev['x-route-filters'] || []; + pageRev['x-route-filters'].push({ + path: './lib/mediawiki_auth_filter.js' + }); +} + +function hidePaths(spec) { + const paths = spec.paths; + Object.keys(paths).forEach((path) => { + paths[path]['x-hidden'] = true; + }); +} + +module.exports = (options) => { + let spec = {}; + options = options || {}; + options.service_type = options.service_type || 'fullstack'; + options.proxy = options.proxy || {}; + if (!Object.prototype.hasOwnProperty.call(options, 'expose_proxy')) { + options.proxy.expose_proxy = true; + } + if (options.proxy.expose_proxy) { + switch (options.service_type.toLowerCase()) { + case 'frontend': + break; + case 'backend': + spec = specProxy; + addAuthFilter(spec); + break; + case 'fullstack': + spec = specProxy; + addAuthFilter(spec); + hidePaths(spec); + break; + default: + throw new Error('Invalid value for option service_type supplied!'); + } + } + return { + spec, + operations: {}, + globals: { options } + }; +}; From 05f4229f7d1d7ac90c9aabbc290a5433df02fcd2 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Mon, 3 Jun 2019 11:08:36 +0300 Subject: [PATCH 7/9] Make monitoring tests properly start/stop. Change-Id: If4980be11cd870d68abfe958aa462683cb4d2d05 --- test/features/specification/monitoring.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/features/specification/monitoring.js b/test/features/specification/monitoring.js index 62417b4fb..6f7030a7d 100644 --- a/test/features/specification/monitoring.js +++ b/test/features/specification/monitoring.js @@ -157,6 +157,7 @@ describe('Monitoring tests', function() { this.timeout(20000); const server = new Server(); before(() => server.start()); + after(() => server.stop()); it('should get the spec', () => { return P.each([{ @@ -181,6 +182,8 @@ describe('Monitoring tests', function() { }) .then((spec) => { const defineTests = () => { + before(() => server.start()); + after(() => server.stop()); constructTests(spec, options, server).forEach((testCase) => { it(testCase.title, () => { const missingParam = /\/{(.+)}/.exec(testCase.request.uri); @@ -200,7 +203,6 @@ describe('Monitoring tests', function() { }; parallel(`Monitoring routes, ${options.domain} domain, new content`, defineTests); parallel(`Monitoring routes, ${options.domain} domain, from storage`, defineTests); - after(() => server.stop()); }); }); }); From 3a020d121db9a198eda5d70ec80445b9881f6910 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Tue, 4 Jun 2019 16:08:00 +0300 Subject: [PATCH 8/9] Introduce 3 running modes: fs, ftfs, ftbe Change-Id: I5db24111bc3f38d71a7f2d1342579636e6664711 --- .travis.yml | 10 +- config.example.storage.wikimedia.yaml | 66 ----- config.example.wikimedia.yaml | 187 ------------- config.frontend.test.yaml | 202 ++++++++++++++ config.fullstack.test.yaml | 251 ++++++++++++++++++ config.storage.test.yaml | 89 +++++++ config.test.yaml | 179 ------------- lib/loader_module.js | 23 -- projects/backend.yaml | 9 - .../{sys/backend-proxy.yaml => proxy.yaml} | 5 +- projects/sys/backend-storage.yaml | 16 -- projects/sys/backend.js | 25 -- projects/sys/business.js | 10 - .../sys/{business.yaml => default.wmf.yaml} | 3 +- projects/v1/default.wmf.yaml | 99 +++++++ projects/v1/enwiki.wmf.yaml | 126 +++++++++ projects/v1/wikidata.wmf.yaml | 82 ++++++ projects/v1/wikimedia.wmf.yaml | 75 ++++++ projects/v1/wikipedia.wmf.yaml | 127 +++++++++ projects/v1/wikivoyage.wmf.yaml | 122 +++++++++ projects/v1/wiktionary.wmf.yaml | 114 ++++++++ projects/wikimedia.org.yaml | 101 ------- projects/wmf_default.yaml | 113 -------- projects/wmf_enwiki.yaml | 140 ---------- projects/wmf_wikidata.yaml | 95 ------- projects/wmf_wikipedia.yaml | 140 ---------- projects/wmf_wikivoyage.yaml | 135 ---------- projects/wmf_wiktionary.yaml | 129 --------- sys/backend_proxy.js | 1 + test/features/buckets/key_value_bucket.js | 8 +- test/features/events/events.js | 34 +-- test/features/post_data.js | 12 +- test/features/schema_tests.js | 2 +- test/test_module.yaml | 163 ++++++------ test/utils/run_tests.sh | 19 +- test/utils/server.js | 42 ++- v1/proxy.js | 55 ---- 37 files changed, 1435 insertions(+), 1574 deletions(-) delete mode 100644 config.example.storage.wikimedia.yaml delete mode 100644 config.example.wikimedia.yaml create mode 100644 config.frontend.test.yaml create mode 100644 config.fullstack.test.yaml create mode 100644 config.storage.test.yaml delete mode 100644 config.test.yaml delete mode 100644 lib/loader_module.js delete mode 100644 projects/backend.yaml rename projects/{sys/backend-proxy.yaml => proxy.yaml} (65%) delete mode 100644 projects/sys/backend-storage.yaml delete mode 100644 projects/sys/backend.js delete mode 100644 projects/sys/business.js rename projects/sys/{business.yaml => default.wmf.yaml} (99%) create mode 100644 projects/v1/default.wmf.yaml create mode 100644 projects/v1/enwiki.wmf.yaml create mode 100644 projects/v1/wikidata.wmf.yaml create mode 100644 projects/v1/wikimedia.wmf.yaml create mode 100644 projects/v1/wikipedia.wmf.yaml create mode 100644 projects/v1/wikivoyage.wmf.yaml create mode 100644 projects/v1/wiktionary.wmf.yaml delete mode 100644 projects/wikimedia.org.yaml delete mode 100644 projects/wmf_default.yaml delete mode 100644 projects/wmf_enwiki.yaml delete mode 100644 projects/wmf_wikidata.yaml delete mode 100644 projects/wmf_wikipedia.yaml delete mode 100644 projects/wmf_wikivoyage.yaml delete mode 100644 projects/wmf_wiktionary.yaml delete mode 100644 v1/proxy.js diff --git a/.travis.yml b/.travis.yml index 8c71562ed..ffd923d12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,12 @@ addons: - openjdk-8-jre-headless env: - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=single_process - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=single_process - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=multi_process - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=multi_process + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=fs + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=fs + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftfs + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftfs + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftbe + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftbe before_install: - wget https://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz -P ../ diff --git a/config.example.storage.wikimedia.yaml b/config.example.storage.wikimedia.yaml deleted file mode 100644 index e26c76adc..000000000 --- a/config.example.storage.wikimedia.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# RESTBase wikimedia example config -default_project: &default_project - x-modules: - - path: projects/backend.yaml - options: - service_type: backend - table: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: group.local - domains: /./ - dbname: db.sqlite3 # ignored in cassandra, but useful in SQLite testing - action: - apiUriTemplate: "{{'https://{domain}/w/api.php'}}" - baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" - - -# The root of the spec tree. Domains tend to share specs by referencing them -# using YAML references. -spec_root: &spec_root - title: "The RESTBase Storage Service" - x-sub-request-filters: - - type: default - name: http - options: - allow: - - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ - forward_headers: true - paths: - /{domain}: *default_project - -# Finally, a standard service-runner config. -info: - name: restbase - -services: - - name: restbase - module: hyperswitch - conf: - port: 7232 - spec: *spec_root - salt: secret - default_page_size: 125 - user_agent: RESTBase - ui_name: RESTBase - ui_url: https://www.mediawiki.org/wiki/RESTBase - ui_title: RESTBase docs - -logging: - name: restbase - level: error - streams: - - type: stdout - -#metrics: -# type: statsd -# host: localhost -# port: 8125 -# batch: true - -num_workers: 0 diff --git a/config.example.wikimedia.yaml b/config.example.wikimedia.yaml deleted file mode 100644 index 1b91f2485..000000000 --- a/config.example.wikimedia.yaml +++ /dev/null @@ -1,187 +0,0 @@ -# RESTBase wikimedia example config - -# Load some project templates. These are referenced / shared between domains -# in the root_spec further down. -default_project: &default_project - x-modules: - - path: projects/wmf_default.yaml - options: &default_options - service_type: '{env(RB_SERVICE_TYPE, fullstack)}' - proxy: - backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys"}})}' - table: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: test.group.local - domains: /./ - # ignored in cassandra, but useful in SQLite testing. only used in tests. - dbname: test.db.sqlite3 - parsoid: - host: https://parsoid-beta.wmflabs.org - grace_ttl: 1000000 - stash_ratelimit: 5 - action: - apiUriTemplate: "{{'https://{domain}/w/api.php'}}" - baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" - graphoid: - host: https://graphoid-beta.wmflabs.org - mathoid: - host: https://mathoid-beta.wmflabs.org - # 10 days Varnish caching, one day client-side - cache-control: s-maxage=864000, max-age=86400 - mobileapps: - host: https://appservice.wmflabs.org - summary: - protocol: https - implementation: mcs - host: https://appservice.wmflabs.org - related: - cache_control: s-maxage=86400, max-age=86400 - citoid: - host: https://citoid-beta.wmflabs.org - recommendation: - host: https://recommendation-api-beta.wmflabs.org - # 10 days Varnish caching, one day client-side - purged_cache_control: s-maxage=1209600, max-age=0, must-revalidate - # Cache control for purged endpoints allowing short-term client caching - purged_cache_control_client_cache: s-maxage=1209600, max-age=300 - pdf: - # Cache PDF for 5 minutes since it's not purged - cache_control: s-maxage=600, max-age=600 - uri: https://proton-beta.wmflabs.org - transform: - cx_host: https://cxserver-beta.wmflabs.org - skip_updates: false - -# A separate project for en.wikipedia because it is more feature-rich -en.wikipedia.org: &en.wikipedia.org - x-modules: - - path: projects/wmf_enwiki.yaml - options: *default_options - -# A different project template, sharing configuration options. -wikimedia.org: &wikimedia.org - x-modules: - - path: projects/wikimedia.org.yaml - options: - <<: *default_options - pageviews: - host: https://wikimedia.org/api/rest_v1/metrics - -wikidata.org: &wikidata.org - x-modules: - - path: projects/wmf_wikidata.yaml - options: *default_options - -wiktionary_project: &wiktionary_project - x-modules: - - path: projects/wmf_wiktionary.yaml - options: *default_options - -wikipedia_project: &wikipedia_project - x-modules: - - path: projects/wmf_wikipedia.yaml - options: *default_options - -wikivoyage_project: &wikivoyage_project - x-modules: - - path: projects/wmf_wikivoyage.yaml - options: *default_options - -# The root of the spec tree. Domains tend to share specs by referencing them -# using YAML references. -spec_root: &spec_root - title: "The RESTBase root" - x-request-filters: - - path: lib/security_response_header_filter.js - - path: lib/normalize_headers_filter.js - x-sub-request-filters: - - type: default - name: http - options: - allow: - - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ - forward_headers: true - - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ - forward_headers: true - # Need to forward cookie to backend RESTBase - - pattern: /^https?:\/\/localhost:7232.+/ - forward_headers: true - - pattern: /^https?:\/\// - paths: - /{domain:en.wikipedia.org}: *en.wikipedia.org - /{domain:ru.wikipedia.org}: *wikipedia_project - /{domain:de.wikipedia.org}: *wikipedia_project - /{domain:es.wikipedia.org}: *wikipedia_project - /{domain:it.wikipedia.org}: *wikipedia_project - /{domain:da.wikipedia.org}: *wikipedia_project - /{domain:sv.wikipedia.org}: *wikipedia_project - - # Example for wiktionary - /{domain:en.wiktionary.org}: *wiktionary_project - - # global domain - /{domain:wikimedia.org}: *wikimedia.org - - # special case for wikidata - /{domain:www.wikidata.org}: *wikidata.org - - # example for non-wikipedia - /{domain:en.wikivoyage.beta.wmflabs.org}: *wikivoyage_project - - # A robots.txt to make sure that the content isn't indexed. - /robots.txt: - get: - x-request-handler: - - static: - return: - status: 200 - headers: - content-type: text/plain - body: | - User-agent: * - Allow: /*/v1/?doc - Disallow: / - -# Finally, a standard service-runner config. -info: - name: restbase - -services: - - name: restbase - module: hyperswitch - conf: - port: 7231 - spec: *spec_root - salt: secret - default_page_size: 125 - user_agent: RESTBase - ui_name: RESTBase - ui_url: https://www.mediawiki.org/wiki/RESTBase - ui_title: RESTBase docs - -logging: - name: restbase - level: warn - streams: - - type: stdout - -#metrics: -# type: statsd -# host: localhost -# port: 8125 -# batch: true - -ratelimiter: - type: kademlia - # Cluster nodes - seeds: - # Port 3050 used by default - - 127.0.0.1 - -num_workers: 0 diff --git a/config.frontend.test.yaml b/config.frontend.test.yaml new file mode 100644 index 000000000..1bbcad983 --- /dev/null +++ b/config.frontend.test.yaml @@ -0,0 +1,202 @@ +default_project: &default_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: test/test_module.yaml + - path: projects/v1/default.wmf.yaml + options: &default_options + parsoid: + host: https://parsoid-beta.wmflabs.org + grace_ttl: 1000000 + action: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + graphoid: + host: https://graphoid-beta.wmflabs.org + mathoid: &mathoid_options + host: https://mathoid-beta.wmflabs.org + # 10 days Varnish caching, one day client-side + cache-control: s-maxage=864000, max-age=86400 + mobileapps: + host: https://appservice.wmflabs.org + summary: + protocol: https + implementation: mcs + host: https://appservice.wmflabs.org + citoid: + host: https://citoid-beta.wmflabs.org + recommendation: + host: https://recommendation-api-beta.wmflabs.org + purged_cache_control: test_purged_cache_control + # Cache control for purged endpoints allowing short-term client caching + purged_cache_control_client_cache: test_purged_cache_control_with_client_caching + pdf: + # Cache PDF for 5 minutes since it's not purged + cache_control: s-maxage=600, max-age=600 + uri: https://proton-beta.wmflabs.org + transform: + cx_host: https://cxserver-beta.wmflabs.org + skip_updates: false + /{api:sys}: &default_sys + x-modules: + - path: projects/sys/default.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"http://localhost:7231/{domain}/v1"}}' + block_external_reqs: false + +wikimedia_project: &wikimedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikimedia.wmf.yaml + options: + <<: *default_options + pageviews: + host: https://wikimedia.org/api/rest_v1/metrics + /{api:sys}: + x-modules: + - path: projects/proxy.yaml + options: + backend_host_template: '{{"http://localhost:7231/{domain}/v1"}}' + block_external_reqs: false + - spec: + paths: + /mathoid: + x-modules: + - path: sys/mathoid.js + options: *mathoid_options + /post_data: + x-modules: + - path: sys/post_data.js + /events: + x-modules: + - path: sys/events.js + +wikipedia_project: &wikipedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikipedia.wmf.yaml + options: *default_options + - path: test/test_module.yaml + options: + events: + uri: http://127.0.0.1:8085/v1/events + topic: resource_change + transcludes_topic: change-prop.transcludes.resource-change + /{api:sys}: *default_sys + +enwiki_project: &enwiki_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/enwiki.wmf.yaml + options: *default_options + /{api:sys}: *default_sys + +wiktionary_project: &wiktionary_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: test/test_module.yaml + - path: projects/v1/wiktionary.wmf.yaml + options: *default_options + /{api:sys}: *default_sys + +# Hacky way to parametrize RESTBase tests. TODO: Move to config? +test: + content_types: + html: '/^text\/html; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/HTML\/[\d.]+"$/' + data-parsoid: '/^application\/json; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/data-parsoid/[\d.]+"$/' + wikitext: '/^text\/plain; charset=utf-8; profile="https:\/\/www.mediawiki.org\/wiki\/Specs\/wikitext\/[\d.]+"$/' + + +# The root of the spec tree. Domains tend to share specs by referencing them +# using YAML references. +spec_root: &spec_root + title: "The RESTBase root" + # Some more general RESTBase info + x-request-filters: + - path: lib/security_response_header_filter.js + - path: lib/normalize_headers_filter.js + + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true + - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ + forward_headers: true + # Need to forward cookie to backend RESTBase + - pattern: /^https?:\/\/localhost:7231.+/ + forward_headers: true + - pattern: /^https?:\/\// + paths: + /{domain:test.wikipedia.org}: *wikipedia_project + # The order is important for tests. + # Redirect tests require en.wiki being not the first wiki in the list. + /{domain:en.wikipedia.org}: *enwiki_project + /{domain:ru.wikipedia.org}: *wikipedia_project + /{domain:de.wikipedia.org}: *wikipedia_project + /{domain:test2.wikipedia.org}: *wikipedia_project + /{domain:commons.wikimedia.org}: *default_project + + # labs, used for most tests + /{domain:en.wikipedia.beta.wmflabs.org}: *wikipedia_project + # Serbian wiki in beta for language variants tests + /{domain:sr.wikipedia.beta.wmflabs.org}: *wikipedia_project + + # For security testing we rely on mocks, so it's OK to use French wiki. + /{domain:fr.wikipedia.org}: + <<: *wikipedia_project + x-route-filters: + - path: ./lib/mediawiki_auth_filter.js + options: + permissions: + - read + # global domain + /{domain:wikimedia.org}: *wikimedia_project + + # Wiktionary has some specific endpoints + /{domain:en.wiktionary.org}: *wiktionary_project + + + +# Finally, a standard service-runner config. +info: + name: restbase + +services: + - name: restbase + module: hyperswitch + conf: + port: 7232 + spec: *spec_root + salt: secret + default_page_size: 1 + user_agent: RESTBase-testsuite + ui_name: RESTBase + ui_url: https://www.mediawiki.org/wiki/RESTBase + ui_title: RESTBase docs + +logging: + name: restbase-test + level: info + streams: + - type: stdout + +num_workers: 0 diff --git a/config.fullstack.test.yaml b/config.fullstack.test.yaml new file mode 100644 index 000000000..673f93b17 --- /dev/null +++ b/config.fullstack.test.yaml @@ -0,0 +1,251 @@ +# RESTBase test config, used in integration tests. + +# Load some project templates. These are referenced / shared between domains +# in the root_spec further down. +default_project: &default_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/default.wmf.yaml + options: &default_options + parsoid: + host: https://parsoid-beta.wmflabs.org + grace_ttl: 1000000 + action: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + graphoid: + host: https://graphoid-beta.wmflabs.org + mathoid: &mathoid_options + host: https://mathoid-beta.wmflabs.org + # 10 days Varnish caching, one day client-side + cache-control: s-maxage=864000, max-age=86400 + mobileapps: + host: https://appservice.wmflabs.org + summary: + protocol: https + implementation: mcs + host: https://appservice.wmflabs.org + citoid: + host: https://citoid-beta.wmflabs.org + recommendation: + host: https://recommendation-api-beta.wmflabs.org + purged_cache_control: test_purged_cache_control + # Cache control for purged endpoints allowing short-term client caching + purged_cache_control_client_cache: test_purged_cache_control_with_client_caching + pdf: + # Cache PDF for 5 minutes since it's not purged + cache_control: s-maxage=600, max-age=600 + uri: https://proton-beta.wmflabs.org + transform: + cx_host: https://cxserver-beta.wmflabs.org + skip_updates: false + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: &default_sys + x-modules: + - path: projects/sys/default.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: false + - spec: + paths: + /legacy/key_value: + x-modules: + - path: sys/key_value.js + /legacy/page_revisions: + x-modules: + - path: sys/page_revisions.js + /table: &table_spec + x-modules: + - path: sys/table.js + options: + conf: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: test.group.local + domains: /./ + dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing + +wikimedia_project: &wikimedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikimedia.wmf.yaml + options: + <<: *default_options + pageviews: + host: https://wikimedia.org/api/rest_v1/metrics + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: + x-modules: + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: false + - spec: + paths: + /legacy/key_value: + x-modules: + - path: sys/key_value.js + /table: *table_spec + /mathoid: + x-modules: + - path: sys/mathoid.js + options: *mathoid_options + /post_data: + x-modules: + - path: sys/post_data.js + /events: + x-modules: + - path: sys/events.js + +wikipedia_project: &wikipedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikipedia.wmf.yaml + options: *default_options + - path: test/test_module.yaml + options: + events: + uri: http://127.0.0.1:8085/v1/events + topic: resource_change + transcludes_topic: change-prop.transcludes.resource-change + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: *default_sys + +enwiki_project: &enwiki_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/enwiki.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: *default_sys + +wiktionary_project: &wiktionary_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wiktionary.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: *default_sys + +# Hacky way to parametrize RESTBase tests. TODO: Move to config? +test: + content_types: + html: '/^text\/html; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/HTML\/[\d.]+"$/' + data-parsoid: '/^application\/json; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/data-parsoid/[\d.]+"$/' + wikitext: '/^text\/plain; charset=utf-8; profile="https:\/\/www.mediawiki.org\/wiki\/Specs\/wikitext\/[\d.]+"$/' + + +# The root of the spec tree. Domains tend to share specs by referencing them +# using YAML references. +spec_root: &spec_root + title: "The RESTBase root" + # Some more general RESTBase info + x-request-filters: + - path: lib/security_response_header_filter.js + - path: lib/normalize_headers_filter.js + + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true + - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ + forward_headers: true + # Need to forward cookie to backend RESTBase + - pattern: /^https?:\/\/localhost:7232.+/ + forward_headers: true + - pattern: /^https?:\/\// + paths: + /{domain:test.wikipedia.org}: *wikipedia_project + # The order is important for tests. + # Redirect tests require en.wiki being not the first wiki in the list. + /{domain:en.wikipedia.org}: *enwiki_project + /{domain:ru.wikipedia.org}: *wikipedia_project + /{domain:de.wikipedia.org}: *wikipedia_project + /{domain:test2.wikipedia.org}: *wikipedia_project + /{domain:commons.wikimedia.org}: *default_project + + # labs, used for most tests + /{domain:en.wikipedia.beta.wmflabs.org}: *wikipedia_project + # Serbian wiki in beta for language variants tests + /{domain:sr.wikipedia.beta.wmflabs.org}: *wikipedia_project + + # For security testing we rely on mocks, so it's OK to use French wiki. + /{domain:fr.wikipedia.org}: + <<: *wikipedia_project + x-route-filters: + - path: ./lib/mediawiki_auth_filter.js + options: + permissions: + - read + # global domain + /{domain:wikimedia.org}: *wikimedia_project + + # Wiktionary has some specific endpoints + /{domain:en.wiktionary.org}: *wiktionary_project + + + +# Finally, a standard service-runner config. +info: + name: restbase + +services: + - name: restbase + module: hyperswitch + conf: + port: 7231 + spec: *spec_root + salt: secret + default_page_size: 1 + user_agent: RESTBase-testsuite + ui_name: RESTBase + ui_url: https://www.mediawiki.org/wiki/RESTBase + ui_title: RESTBase docs + +logging: + name: restbase-test + level: info + streams: + - type: stdout + +num_workers: 0 diff --git a/config.storage.test.yaml b/config.storage.test.yaml new file mode 100644 index 000000000..6516ac78b --- /dev/null +++ b/config.storage.test.yaml @@ -0,0 +1,89 @@ +# RESTBase wikimedia example config +default_project: &default_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - spec: + paths: + /key_value: + x-modules: + - path: sys/key_value.js + /page_revisions: + # We need to forward cookie to the API + x-route-filters: + - path: ./lib/mediawiki_auth_filter.js + x-modules: + - path: sys/page_revisions.js + /{api:sys}: + x-modules: + - spec: + paths: + /table: + x-modules: + - path: sys/table.js + options: + conf: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: test.group.local + domains: /./ + dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing + /action: + x-modules: + - path: sys/action.js + options: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + + +# The root of the spec tree. Domains tend to share specs by referencing them +# using YAML references. +spec_root: &spec_root + title: "The RESTBase Storage Service" + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true + paths: + /{domain}: *default_project + +# Finally, a standard service-runner config. +info: + name: restbase + +services: + - name: restbase + module: hyperswitch + conf: + port: 7231 + spec: *spec_root + salt: secret + default_page_size: 125 + user_agent: RESTBase + ui_name: RESTBase + ui_url: https://www.mediawiki.org/wiki/RESTBase + ui_title: RESTBase docs + +logging: + name: restbase + level: error + streams: + - type: stdout + +#metrics: +# type: statsd +# host: localhost +# port: 8125 +# batch: true + +num_workers: 0 diff --git a/config.test.yaml b/config.test.yaml deleted file mode 100644 index 4d882e857..000000000 --- a/config.test.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# RESTBase test config, used in integration tests. - -# Load some project templates. These are referenced / shared between domains -# in the root_spec further down. -default_project: &default_project - x-modules: - - path: test/test_module.yaml - - path: projects/wmf_default.yaml - options: &default_options - service_type: '{env(RB_SERVICE_TYPE, fullstack)}' - proxy: - backend_host_template: '{env(RB_TEST_BACKEND_HOST_TEMPLATE, {{"/{domain}/sys"}})}' - table: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: test.group.local - domains: /./ - dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing - parsoid: - host: https://parsoid-beta.wmflabs.org - grace_ttl: 1000000 - action: - apiUriTemplate: "{{'https://{domain}/w/api.php'}}" - baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" - graphoid: - host: https://graphoid-beta.wmflabs.org - mathoid: - host: https://mathoid-beta.wmflabs.org - # 10 days Varnish caching, one day client-side - cache-control: s-maxage=864000, max-age=86400 - mobileapps: - host: https://appservice.wmflabs.org - summary: - protocol: https - implementation: mcs - host: https://appservice.wmflabs.org - citoid: - host: https://citoid-beta.wmflabs.org - recommendation: - host: https://recommendation-api-beta.wmflabs.org - events: {} - purged_cache_control: test_purged_cache_control - # Cache control for purged endpoints allowing short-term client caching - purged_cache_control_client_cache: test_purged_cache_control_with_client_caching - pdf: - # Cache PDF for 5 minutes since it's not purged - cache_control: s-maxage=600, max-age=600 - uri: https://proton-beta.wmflabs.org - transform: - cx_host: https://cxserver-beta.wmflabs.org - skip_updates: false - -# A separate project for en.wikipedia because it is more feature-rich -en.wikipedia.org: &en.wikipedia.org - x-modules: - - path: test/test_module.yaml - options: - events: - uri: http://127.0.0.1:8085/v1/events - topic: resource_change - transcludes_topic: change-prop.transcludes.resource-change - - path: projects/wmf_enwiki.yaml - options: *default_options - -labs_project: &labs_project - x-modules: - - path: test/test_module.yaml - - path: projects/wmf_enwiki.yaml - options: *default_options - -# A different project template, sharing configuration options. -wikimedia.org: &wikimedia.org - x-modules: - - path: test/test_module.yaml - - path: projects/wikimedia.org.yaml - options: - <<: *default_options - pageviews: - host: https://wikimedia.org/api/rest_v1/metrics - -wiktionary_project: &wiktionary_project - x-modules: - - path: projects/wmf_wiktionary.yaml - options: *default_options - -wikipedia_project: &wikipedia_project - x-modules: - - path: test/test_module.yaml - - path: projects/wmf_wikipedia.yaml - options: *default_options - -# Hacky way to parametrize RESTBase tests. TODO: Move to config? -test: - content_types: - html: '/^text\/html; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/HTML\/[\d.]+"$/' - data-parsoid: '/^application\/json; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/data-parsoid/[\d.]+"$/' - wikitext: '/^text\/plain; charset=utf-8; profile="https:\/\/www.mediawiki.org\/wiki\/Specs\/wikitext\/[\d.]+"$/' - - -# The root of the spec tree. Domains tend to share specs by referencing them -# using YAML references. -spec_root: &spec_root - title: "The RESTBase root" - # Some more general RESTBase info - x-request-filters: - - path: lib/security_response_header_filter.js - - path: lib/normalize_headers_filter.js - - x-sub-request-filters: - - type: default - name: http - options: - allow: - - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ - forward_headers: true - - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ - forward_headers: true - # Need to forward cookie to backend RESTBase - - pattern: /^https?:\/\/localhost:7232.+/ - forward_headers: true - - pattern: /^https?:\/\// - paths: - /{domain:test.wikipedia.org}: *wikipedia_project - # The order is important for tests. - # Redirect tests require en.wiki being not the first wiki in the list. - /{domain:en.wikipedia.org}: *en.wikipedia.org - /{domain:ru.wikipedia.org}: *wikipedia_project - /{domain:de.wikipedia.org}: *wikipedia_project - /{domain:test2.wikipedia.org}: *wikipedia_project - /{domain:commons.wikimedia.org}: *default_project - - # labs, used for most tests - /{domain:en.wikipedia.beta.wmflabs.org}: *labs_project - # Serbian wiki in beta for language variants tests - /{domain:sr.wikipedia.beta.wmflabs.org}: *labs_project - - # For security testing we rely on mocks, so it's OK to use French wiki. - /{domain:fr.wikipedia.org}: - <<: *wikipedia_project - x-route-filters: - - path: ./lib/mediawiki_auth_filter.js - options: - permissions: - - read - # global domain - /{domain:wikimedia.org}: *wikimedia.org - - # Wiktionary has some specific endpoints - /{domain:en.wiktionary.org}: *wiktionary_project - -# Finally, a standard service-runner config. -info: - name: restbase - -services: - - name: restbase - module: hyperswitch - conf: - port: 7231 - spec: *spec_root - salt: secret - default_page_size: 1 - user_agent: RESTBase-testsuite - ui_name: RESTBase - ui_url: https://www.mediawiki.org/wiki/RESTBase - ui_title: RESTBase docs - -logging: - name: restbase-test - level: error - streams: - - type: stdout - -num_workers: 0 diff --git a/lib/loader_module.js b/lib/loader_module.js deleted file mode 100644 index 1f36811bb..000000000 --- a/lib/loader_module.js +++ /dev/null @@ -1,23 +0,0 @@ -'const strict'; - -module.exports = (spec, options) => { - options = options || {}; - spec = spec || {}; - spec.paths = spec.paths || {}; - if (options.exclude) { - if (!Array.isArray(options.exclude)) { - options.exclude = options.exclude.split(','); - } - options.exclude.forEach((module) => { - module = /^\//.test(module) ? module : `/${module}`; - if (spec.paths[module]) { - delete spec.paths[module]; - } - }); - } - return { - spec, - operations: {}, - globals: { options } - }; -}; diff --git a/projects/backend.yaml b/projects/backend.yaml deleted file mode 100644 index 825ebee53..000000000 --- a/projects/backend.yaml +++ /dev/null @@ -1,9 +0,0 @@ -paths: - /{api:v1}: - x-modules: - - path: v1/proxy.js - options: '{{options}}' - /{api:sys}: - x-modules: - - path: projects/sys/backend.js - options: '{{options}}' diff --git a/projects/sys/backend-proxy.yaml b/projects/proxy.yaml similarity index 65% rename from projects/sys/backend-proxy.yaml rename to projects/proxy.yaml index 3358b31e2..b88d93f85 100644 --- a/projects/sys/backend-proxy.yaml +++ b/projects/proxy.yaml @@ -2,9 +2,8 @@ paths: /key_value: x-modules: - path: sys/backend_proxy.js - options: '{{options.proxy}}' + options: '{{options}}' /page_revisions: x-modules: - path: sys/backend_proxy.js - options: '{{options.proxy}}' - + options: '{{options}}' diff --git a/projects/sys/backend-storage.yaml b/projects/sys/backend-storage.yaml deleted file mode 100644 index 858b71777..000000000 --- a/projects/sys/backend-storage.yaml +++ /dev/null @@ -1,16 +0,0 @@ -paths: - /table: - x-modules: - - path: sys/table.js - options: - conf: '{{options.table}}' - /action: - x-modules: - - path: sys/action.js - options: "{{options.action}}" - /key_value: - x-modules: - - path: sys/key_value.js - /page_revisions: - x-modules: - - path: sys/page_revisions.js diff --git a/projects/sys/backend.js b/projects/sys/backend.js deleted file mode 100644 index 607336dd1..000000000 --- a/projects/sys/backend.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict'; - -const HyperSwitch = require('hyperswitch'); -const loader = require('../../lib/loader_module.js'); - -const specStorage = HyperSwitch.utils.loadSpec(`${__dirname}/backend-storage.yaml`); -const specProxy = HyperSwitch.utils.loadSpec(`${__dirname}/backend-proxy.yaml`); - -module.exports = (options) => { - let spec; - options = options || {}; - options.service_type = options.service_type || 'fullstack'; - switch (options.service_type.toLowerCase()) { - case 'frontend': - spec = specProxy; - break; - case 'backend': - case 'fullstack': - spec = specStorage; - break; - default: - throw new Error('Invalid value for option service_type supplied!'); - } - return loader(spec, options); -}; diff --git a/projects/sys/business.js b/projects/sys/business.js deleted file mode 100644 index 3380611da..000000000 --- a/projects/sys/business.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const HyperSwitch = require('hyperswitch'); -const loader = require('../../lib/loader_module.js'); - -const spec = HyperSwitch.utils.loadSpec(`${__dirname}/business.yaml`); - -module.exports = (options) => { - return loader(spec, options); -}; diff --git a/projects/sys/business.yaml b/projects/sys/default.wmf.yaml similarity index 99% rename from projects/sys/business.yaml rename to projects/sys/default.wmf.yaml index 0562b7d89..ae6fa89d3 100644 --- a/projects/sys/business.yaml +++ b/projects/sys/default.wmf.yaml @@ -120,9 +120,8 @@ paths: x-modules: - path: sys/mobileapps.js options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' + options.mobileapps)}}' /events: x-modules: - path: sys/events.js options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' - diff --git a/projects/v1/default.wmf.yaml b/projects/v1/default.wmf.yaml new file mode 100644 index 000000000..336b44686 --- /dev/null +++ b/projects/v1/default.wmf.yaml @@ -0,0 +1,99 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' diff --git a/projects/v1/enwiki.wmf.yaml b/projects/v1/enwiki.wmf.yaml new file mode 100644 index 000000000..458b1abc5 --- /dev/null +++ b/projects/v1/enwiki.wmf.yaml @@ -0,0 +1,126 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /feed: + x-modules: + - path: v1/feed.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + - path: v1/announcements.yaml + options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' + - path: v1/onthisday.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' diff --git a/projects/v1/wikidata.wmf.yaml b/projects/v1/wikidata.wmf.yaml new file mode 100644 index 000000000..73f7f3e0d --- /dev/null +++ b/projects/v1/wikidata.wmf.yaml @@ -0,0 +1,82 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, options.mobileapps)}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/lists.js + options: '{{options.lists}}' + diff --git a/projects/v1/wikimedia.wmf.yaml b/projects/v1/wikimedia.wmf.yaml new file mode 100644 index 000000000..034dff621 --- /dev/null +++ b/projects/v1/wikimedia.wmf.yaml @@ -0,0 +1,75 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Software available under the Apache 2 license + url: http://www.apache.org/licenses/LICENSE-2.0 + +securityDefinitions: &wp/content-security/1.0.0 + mediawiki_auth: + description: Checks permissions using MW api + type: apiKey + in: header + name: cookie + x-internal-request-whitelist: + - /http:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +paths: + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /metrics: + x-modules: + - path: v1/metrics.yaml + options: '{{options.pageviews}}' + /transform: + x-modules: + - path: v1/transform-global.yaml + options: '{{options.transform}}' + /feed: + x-modules: + - path: v1/availability.yaml + options: '{{options.mobileapps}}' diff --git a/projects/v1/wikipedia.wmf.yaml b/projects/v1/wikipedia.wmf.yaml new file mode 100644 index 000000000..cdcaaa408 --- /dev/null +++ b/projects/v1/wikipedia.wmf.yaml @@ -0,0 +1,127 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /feed: + x-modules: + - path: v1/feed.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + - path: v1/announcements.yaml + options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' + - path: v1/onthisday.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + diff --git a/projects/v1/wikivoyage.wmf.yaml b/projects/v1/wikivoyage.wmf.yaml new file mode 100644 index 000000000..4a12d2efa --- /dev/null +++ b/projects/v1/wikivoyage.wmf.yaml @@ -0,0 +1,122 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + # NOTE: We're setting the `no-store` cache-control header so that we don't + # store MCS results - we don't have pre-generation for wikivoyage. + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps, + {"cache-control": "no-store"})}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + diff --git a/projects/v1/wiktionary.wmf.yaml b/projects/v1/wiktionary.wmf.yaml new file mode 100644 index 000000000..be3d01fae --- /dev/null +++ b/projects/v1/wiktionary.wmf.yaml @@ -0,0 +1,114 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /: + x-modules: + - path: v1/common_schemas.yaml + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/definition.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml deleted file mode 100644 index e713feac9..000000000 --- a/projects/wikimedia.org.yaml +++ /dev/null @@ -1,101 +0,0 @@ -paths: - /{api:v1}: - x-modules: - # swagger options, overriding the shared ones from the merged specs (?) - - spec: - info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Software available under the Apache 2 license - url: http://www.apache.org/licenses/LICENSE-2.0 - - securityDefinitions: &wp/content-security/1.0.0 - mediawiki_auth: - description: Checks permissions using MW api - type: apiKey - in: header - name: cookie - x-internal-request-whitelist: - - /http:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ - # Override the base path for host-based (proxied) requests. In our case, - # we proxy https://{domain}/api/rest_v1/ to the API. - x-host-basePath: /api/rest_v1 - paths: - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /metrics: - x-modules: - - path: v1/metrics.yaml - options: '{{options.pageviews}}' - /transform: - x-modules: - - path: v1/transform-global.yaml - options: '{{options.transform}}' - /feed: - x-modules: - - path: v1/availability.yaml - options: '{{options.mobileapps}}' - options: '{{options}}' - - path: v1/proxy.js - options: '{{options}}' - - /{api:sys}: - x-modules: - - path: projects/sys/backend.js - options: '{{options}}' - - spec: - paths: - /mathoid: - x-modules: - - path: sys/mathoid.js - options: '{{options.mathoid}}' - /post_data: - x-modules: - - path: sys/post_data.js - /events: - x-modules: - - path: sys/events.js - options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' - options: '{{options}}' diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml deleted file mode 100644 index 877bf757e..000000000 --- a/projects/wmf_default.yaml +++ /dev/null @@ -1,113 +0,0 @@ -paths: - /{api:v1}: - x-modules: - # swagger options, overriding the shared ones from the merged specs (?) - - spec: - info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 - # Override the base path for host-based (proxied) requests. In our case, - # we proxy https://{domain}/api/rest_v1/ to the API. - x-host-basePath: /api/rest_v1 - x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' - paths: - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/summary.js - options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, - options.summary)}}' - - path: v1/related.js - options: '{{options.related}}' - - path: v1/random.yaml - options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, - options.mobileapps)}}' - - path: v1/pdf.js - options: '{{options.pdf}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/citoid.js - options: '{{options.citoid}}' - - path: v1/lists.js - options: '{{options.lists}}' - - path: v1/recommend.yaml - options: '{{options.recommendation}}' - - path: v1/css.yaml - options: - host: '{{options.mobileapps.host}}' - - path: v1/javascript.yaml - options: - host: '{{options.mobileapps.host}}' - options: '{{options}}' - - path: v1/proxy.js - options: '{{options}}' - - /{api:sys}: - x-modules: - - path: projects/sys/business.js - options: '{{merge(options, {"exclude": "mobileapps"})}}' - - path: projects/sys/backend.js - options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml deleted file mode 100644 index 560c6388c..000000000 --- a/projects/wmf_enwiki.yaml +++ /dev/null @@ -1,140 +0,0 @@ -paths: - /{api:v1}: - x-modules: - # swagger options, overriding the shared ones from the merged specs (?) - - spec: - info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 - # Override the base path for host-based (proxied) requests. In our case, - # we proxy https://{domain}/api/rest_v1/ to the API. - x-host-basePath: /api/rest_v1 - x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' - paths: - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/mobileapps.yaml - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/summary.js - options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, - options.summary)}}' - - path: v1/media.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/metadata.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/references.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/mobile-html.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/related.js - options: '{{options.related}}' - - path: v1/random.yaml - options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, - options.mobileapps)}}' - - path: v1/pdf.js - options: '{{options.pdf}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /feed: - x-modules: - - path: v1/feed.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - - path: v1/announcements.yaml - options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' - - path: v1/onthisday.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/citoid.js - options: '{{options.citoid}}' - - path: v1/lists.js - options: '{{options.lists}}' - - path: v1/recommend.yaml - options: '{{options.recommendation}}' - - path: v1/css.yaml - options: - host: '{{options.mobileapps.host}}' - - path: v1/javascript.yaml - options: - host: '{{options.mobileapps.host}}' - options: '{{options}}' - - path: v1/proxy.js - options: '{{options}}' - - /{api:sys}: - x-modules: - - path: projects/sys/business.js - options: '{{options}}' - - path: projects/sys/backend.js - options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml deleted file mode 100644 index 554f5238c..000000000 --- a/projects/wmf_wikidata.yaml +++ /dev/null @@ -1,95 +0,0 @@ -paths: - /{api:v1}: - x-modules: - # swagger options, overriding the shared ones from the merged specs (?) - - spec: - info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 - # Override the base path for host-based (proxied) requests. In our case, - # we proxy https://{domain}/api/rest_v1/ to the API. - x-host-basePath: /api/rest_v1 - x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' - paths: - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/random.yaml - options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, options.mobileapps)}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/lists.js - options: '{{options.lists}}' - options: '{{options}}' - - path: v1/proxy.js - options: '{{options}}' - - /{api:sys}: - x-modules: - - path: projects/sys/business.js - options: '{{merge(options, {"exclude": "mobileapps"})}}' - - path: projects/sys/backend.js - options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wikipedia.yaml b/projects/wmf_wikipedia.yaml deleted file mode 100644 index 560c6388c..000000000 --- a/projects/wmf_wikipedia.yaml +++ /dev/null @@ -1,140 +0,0 @@ -paths: - /{api:v1}: - x-modules: - # swagger options, overriding the shared ones from the merged specs (?) - - spec: - info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 - # Override the base path for host-based (proxied) requests. In our case, - # we proxy https://{domain}/api/rest_v1/ to the API. - x-host-basePath: /api/rest_v1 - x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' - paths: - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/mobileapps.yaml - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/summary.js - options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, - options.summary)}}' - - path: v1/media.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/metadata.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/references.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/mobile-html.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/related.js - options: '{{options.related}}' - - path: v1/random.yaml - options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, - options.mobileapps)}}' - - path: v1/pdf.js - options: '{{options.pdf}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /feed: - x-modules: - - path: v1/feed.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - - path: v1/announcements.yaml - options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' - - path: v1/onthisday.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/citoid.js - options: '{{options.citoid}}' - - path: v1/lists.js - options: '{{options.lists}}' - - path: v1/recommend.yaml - options: '{{options.recommendation}}' - - path: v1/css.yaml - options: - host: '{{options.mobileapps.host}}' - - path: v1/javascript.yaml - options: - host: '{{options.mobileapps.host}}' - options: '{{options}}' - - path: v1/proxy.js - options: '{{options}}' - - /{api:sys}: - x-modules: - - path: projects/sys/business.js - options: '{{options}}' - - path: projects/sys/backend.js - options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wikivoyage.yaml b/projects/wmf_wikivoyage.yaml deleted file mode 100644 index 5f5dc7c19..000000000 --- a/projects/wmf_wikivoyage.yaml +++ /dev/null @@ -1,135 +0,0 @@ -paths: - /{api:v1}: - x-modules: - # swagger options, overriding the shared ones from the merged specs (?) - - spec: - info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 - # Override the base path for host-based (proxied) requests. In our case, - # we proxy https://{domain}/api/rest_v1/ to the API. - x-host-basePath: /api/rest_v1 - x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' - paths: - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/mobileapps.yaml - # NOTE: We're setting the `no-store` cache-control header so that we don't - # store MCS results - we don't have pre-generation for wikivoyage. - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps, - {"cache-control": "no-store"})}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/summary.js - options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, - options.summary)}}' - - path: v1/media.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/metadata.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/references.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/mobile-html.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/related.js - options: '{{options.related}}' - - path: v1/random.yaml - options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, - options.mobileapps)}}' - - path: v1/pdf.js - options: '{{options.pdf}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/citoid.js - options: '{{options.citoid}}' - - path: v1/lists.js - options: '{{options.lists}}' - - path: v1/recommend.yaml - options: '{{options.recommendation}}' - - path: v1/css.yaml - options: - host: '{{options.mobileapps.host}}' - - path: v1/javascript.yaml - options: - host: '{{options.mobileapps.host}}' - options: '{{options}}' - - path: v1/proxy.js - options: '{{options}}' - - /{api:sys}: - x-modules: - - path: projects/sys/business.js - options: '{{options}}' - - path: projects/sys/backend.js - options: '{{merge(options, {"exclude": "action"})}}' diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml deleted file mode 100644 index 27a3c5b16..000000000 --- a/projects/wmf_wiktionary.yaml +++ /dev/null @@ -1,129 +0,0 @@ -openapi: 3.0.1 -paths: - /{api:v1}: &default_project_paths_v1 - x-modules: - # swagger options, overriding the shared ones from the merged specs (?) - - spec: - info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 - # Override the base path for host-based (proxied) requests. In our case, - # we proxy https://{domain}/api/rest_v1/ to the API. - x-host-basePath: /api/rest_v1 - x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' - paths: - /: - x-modules: - - path: v1/common_schemas.yaml - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/mobileapps.yaml - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - - path: v1/media.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/metadata.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/references.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/mobile-html.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/definition.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/pdf.js - options: '{{options.pdf}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/citoid.js - options: '{{options.citoid}}' - - path: v1/lists.js - options: '{{options.lists}}' - - path: v1/css.yaml - options: - host: '{{options.mobileapps.host}}' - - path: v1/javascript.yaml - options: - host: '{{options.mobileapps.host}}' - options: '{{options}}' - - path: v1/proxy.js - options: '{{options}}' - - /{api:sys}: - x-modules: - - path: projects/sys/business.js - options: '{{options}}' - - path: projects/sys/backend.js - options: '{{merge(options, {"exclude": "action"})}}' diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js index a96e3987e..e0d7424de 100644 --- a/sys/backend_proxy.js +++ b/sys/backend_proxy.js @@ -24,6 +24,7 @@ module.exports = (options) => { spec: { paths: { '/{+path}': { + 'x-hidden': true, all: { operationId: 'proxy', 'x-monitor': false diff --git a/test/features/buckets/key_value_bucket.js b/test/features/buckets/key_value_bucket.js index e791a3ed3..77bb255bf 100644 --- a/test/features/buckets/key_value_bucket.js +++ b/test/features/buckets/key_value_bucket.js @@ -8,7 +8,6 @@ const P = require('bluebird'); const parallel = require('mocha.parallel'); describe('Key value buckets', () => { - function randomString(length) { let result = ''; for (let i = 0; i < length / 10; i++) { @@ -18,16 +17,16 @@ describe('Key value buckets', () => { } function runTests(bucketName) { - const server = new Server(`${__dirname}/../../../config.example.storage.wikimedia.yaml`, true); + const server = new Server(); let bucketBaseURI; let stringBaseURI; before(() => server.start() .then(() => { - bucketBaseURI = `${server.config.baseURL()}/${bucketName}/${bucketName}TestingBucket`; + bucketBaseURI = `${server.config.backendURL()}/${bucketName}/${bucketName}TestingBucket`; return preq.put({uri: bucketBaseURI}); }) .then(() => { - stringBaseURI = `${server.config.baseURL()}/${bucketName}/${bucketName}StringBucket`; + stringBaseURI = `${server.config.backendURL()}/${bucketName}/${bucketName}StringBucket`; return preq.put({ uri: stringBaseURI, headers: { @@ -41,7 +40,6 @@ describe('Key value buckets', () => { after(() => server.stop()); it('stores a content in a bucket and gets it back', () => { - console.log(bucketBaseURI); const testData = randomString(100); return preq.put({ uri: `${bucketBaseURI}/${testData}`, diff --git a/test/features/events/events.js b/test/features/events/events.js index da9258b0c..19671bd00 100644 --- a/test/features/events/events.js +++ b/test/features/events/events.js @@ -14,9 +14,9 @@ describe('Change event emitting', () => { it('should not explode if events config is not provided', () => { return preq.post({ - uri: `${server.config.baseURL()}/events_no_config/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events_no_config/`, body: [ - { uri: '//en.wikipedia.org' } + { uri: '//en.wikipedia.beta.wmflabs.org' } ] }); }); @@ -34,7 +34,7 @@ describe('Change event emitting', () => { const events = JSON.parse(postData.toString()); assert.deepEqual(events.length, 1); const event = events[0]; - assert.deepEqual(event.meta.domain, 'en.wikipedia.org'); + assert.deepEqual(event.meta.domain, 'en.wikipedia.beta.wmflabs.org'); assert.deepEqual(!!new Date(event.meta.dt), true); assert.deepEqual(uuid.test(event.meta.id), true); assert.deepEqual(!!event.meta.request_id, true); @@ -71,11 +71,11 @@ describe('Change event emitting', () => { eventLogging = createEventLogging(really_done, { topic: 'resource_change', - uri: 'http://en.wikipedia.org/wiki/User:Pchelolo' + uri: 'http://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }); preq.post({ - uri: `${server.config.baseURL()}/events/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events/`, headers: { 'content-type': 'application/json', connection: 'close', @@ -83,7 +83,7 @@ describe('Change event emitting', () => { body: [ { meta: { - uri: '//en.wikipedia.org/wiki/User:Pchelolo' + uri: '//en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }, tags: ['test'] }, @@ -110,21 +110,21 @@ describe('Change event emitting', () => { eventLogging = createEventLogging(really_done, { topic: 'change-prop.transcludes.resource-change', - uri: 'http://en.wikipedia.org/api/rest_v1/page/html/User:Pchelolo', - trigger: 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.org/wiki/User:Pchelolo' + uri: 'http://en.wikipedia.beta.wmflabs.org/api/rest_v1/page/html/User:Pchelolo', + trigger: 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }); preq.post({ - uri: `${server.config.baseURL()}/events/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events/`, headers: { 'content-type': 'application/json', connection: 'close', - 'x-triggered-by': 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.org/wiki/User:Pchelolo' + 'x-triggered-by': 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }, body: [ { meta: { - uri: '//en.wikipedia.org/api/rest_v1/page/html/User:Pchelolo' + uri: '//en.wikipedia.beta.wmflabs.org/api/rest_v1/page/html/User:Pchelolo' }, tags: ['test'] } @@ -149,26 +149,26 @@ describe('Change event emitting', () => { eventLogging = createEventLogging(really_done, { topic: 'resource_change', - uri: 'http://en.wikipedia.org/wiki/User:Pchelolo', - trigger: 'resource_change:https://en.wikipedia.org/wiki/Prohibited' + uri: 'http://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo', + trigger: 'resource_change:https://en.wikipedia.beta.wmflabs.org/wiki/Prohibited' }); preq.post({ - uri: `${server.config.baseURL()}/events/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events/`, headers: { 'content-type': 'application/json', - 'x-triggered-by': 'resource_change:https://en.wikipedia.org/wiki/Prohibited' + 'x-triggered-by': 'resource_change:https://en.wikipedia.beta.wmflabs.org/wiki/Prohibited' }, body: [ { meta: { - uri: '//en.wikipedia.org/wiki/Prohibited' + uri: '//en.wikipedia.beta.wmflabs.org/wiki/Prohibited' }, tags: ['test'] }, { meta: { - uri: '//en.wikipedia.org/wiki/User:Pchelolo' + uri: '//en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }, tags: ['test'] } diff --git a/test/features/post_data.js b/test/features/post_data.js index de84fd283..87d38ae4a 100644 --- a/test/features/post_data.js +++ b/test/features/post_data.js @@ -14,7 +14,7 @@ describe('post_data', function() { it('should store post request by hash', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/`, body: { key: 'value' } @@ -24,7 +24,7 @@ describe('post_data', function() { assert.deepEqual(res.status, 201); assert.deepEqual(hash, '228458095a9502070fc113d99504226a6ff90a9a'); return preq.get({ - uri: `${server.config.baseURL()}/post_data/${res.body}` + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/${res.body}` }); }) .then((res) => { @@ -35,7 +35,7 @@ describe('post_data', function() { it('should not explode on empty body', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/` + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/` }) .then((res) => { assert.deepEqual(res.status, 201); @@ -44,7 +44,7 @@ describe('post_data', function() { it('should not store identical request', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/`, body: { key: 'value' } @@ -57,7 +57,7 @@ describe('post_data', function() { it('should allow read on remote request', () => { return preq.get({ - uri: `${server.config.baseURL()}/post_data/${hash}` + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/${hash}` }) .then((res) => { assert.deepEqual(res.status, 200); @@ -67,7 +67,7 @@ describe('post_data', function() { it('should deny write on remote requests', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/`, headers: { 'x-client-ip': '123.123.123.123' }, diff --git a/test/features/schema_tests.js b/test/features/schema_tests.js index 02f4840d0..f1541071a 100644 --- a/test/features/schema_tests.js +++ b/test/features/schema_tests.js @@ -13,7 +13,7 @@ const validator = new OpenAPISchemaValidator({ version: 3 }); parallel('Responses should conform to the provided JSON schema of the response', () => { const ajv = new Ajv({}); - const server = new Server(`${__dirname}/../../config.example.wikimedia.yaml`); + const server = new Server(); function getToday() { function zeroPad(num) { if (num < 10) { diff --git a/test/test_module.yaml b/test/test_module.yaml index 9210a8f6d..7a2803cfa 100644 --- a/test/test_module.yaml +++ b/test/test_module.yaml @@ -1,91 +1,84 @@ -# Simple test spec - openapi: 3.0.1 +info: + version: 1.0.0-beta + title: Wikimedia testing APIs + x-is-api-root: true paths: - /{api:v1}: - x-modules: - - spec: - info: - version: 1.0.0-beta - title: Wikimedia testing APIs - x-is-api-root: true - paths: - /service/test_parallel/{key1}/{key2}: - get: - x-request-handler: - - get_from_api1: - request: - uri: http://en.wikipedia.org/wiki/{+key1} - get_from_api2: - request: - uri: http://en.wikipedia.org/wiki/{+key2} - - return_response: - return: - status: 200 - body: - first: '{{global.get_from_api1}}' - second: '{{global.get_from_api2}}' - x-monitor: true - x-amples: - - title: Retreives parallel content with simple service - request: - params: - domain: en.wikipedia.org - key1: User:GWicke/Date - key2: User:GWicke/Date - response: - status: 200 - body: - first: - status: 200 - headers: - content-type: /^text\/html.+/ - second: - status: 200 - headers: - content-type: /^text\/html.+/ + /service/test_parallel/{key1}/{key2}: + get: + x-request-handler: + - get_from_api1: + request: + uri: http://en.wikipedia.org/wiki/{+key1} + get_from_api2: + request: + uri: http://en.wikipedia.org/wiki/{+key2} + - return_response: + return: + status: 200 + body: + first: '{{global.get_from_api1}}' + second: '{{global.get_from_api2}}' + x-monitor: true + x-amples: + - title: Retreives parallel content with simple service + request: + params: + domain: en.wikipedia.org + key1: User:GWicke/Date + key2: User:GWicke/Date + response: + status: 200 + body: + first: + status: 200 + headers: + content-type: /^text\/html.+/ + second: + status: 200 + headers: + content-type: /^text\/html.+/ - /post_data/: - post: - x-route-filters: - - type: 'default' - name: 'header_match' - options: - whitelist: - x-client-ip: - - /^(?:::ffff:)?(?:10|127)\./ - x-request-handler: - - put_to_storage: - request: - method: put - uri: /{domain}/sys/post_data/post.test/ - headers: '{{request.headers}}' - body: '{{request.body}}' - x-monitor: false + /post_data/: + post: + x-route-filters: + - type: 'default' + name: 'header_match' + options: + whitelist: + x-client-ip: + - /^(?:::ffff:)?(?:10|127)\./ + x-request-handler: + - put_to_storage: + request: + method: put + uri: /{domain}/sys/post_data/post.test/ + headers: '{{request.headers}}' + body: '{{request.body}}' + x-monitor: false - /post_data/{hash}: - get: - x-setup-handler: - - init_storage: - uri: /{domain}/sys/post_data/post.test - x-request-handler: - - get_from_storage: - request: - uri: /{domain}/sys/post_data/post.test/{hash} - x-monitor: false + /post_data/{hash}: + get: + x-setup-handler: + - init_storage: + uri: /{domain}/sys/post_data/post.test + x-request-handler: + - get_from_storage: + request: + uri: /{domain}/sys/post_data/post.test/{hash} + x-monitor: false - /events_no_config: - x-modules: - - path: sys/events.js - /events: - x-modules: - - path: sys/events.js - options: '{{options.events}}' + /events_no_config: + x-modules: + - path: sys/events.js + /events: + x-modules: + - path: sys/events.js + options: '{{options.events}}' - /http/{uri}: - get: - x-request-handler: - - request_url: - request: - uri: '{{uri}}' - options: '{{options}}' + /http/{uri}: + get: + x-request-handler: + - request_url: + request: + uri: '{{uri}}' diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index 8ddbfda3e..cafe2c9c1 100644 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -1,15 +1,15 @@ #!/bin/sh mod_dir=$( cd "$( dirname "$0" )"/../.. && pwd )/node_modules -mocha="$mod_dir"/mocha/bin/mocha -nyc="$mod_dir"/.bin/nyc +mocha="${mod_dir}"/mocha/bin/mocha +nyc="${mod_dir}"/.bin/nyc test_target=${TEST_TARGET:-$2} -test_mode=${TEST_MODE:-$3} +test_mode=${TEST_MODE:=$3} if [ "$1" = "test" ]; then test_command="${mocha}" elif [ "$1" = "coverage" ]; then - test_command="${nyc}" --reporter=lcov node_modules/.bin/_mocha + test_command="${nyc} --reporter=lcov node_modules/.bin/_mocha" else echo "Invalid test command. Must be 'test' or 'coverage" exit 1 @@ -34,15 +34,6 @@ else exit 1 fi -if [ "x$test_mode" = "x" ] || [ "$test_mode" = "single_process" ]; then - echo "Running single process mode" -elif [ "$test_mode" = "multi_process" ]; then - echo "Running multi process mode" - export RB_TEST_BACKEND_HOST_TEMPLATE="{{\"http://localhost:7232/{domain}/v1\"}}" -else - echo "Invalid TEST_MODE ${test_mode}. Must be 'single_process' or 'multi_process' if specified" - exit 1 -fi - +echo "Running ${test_mode} mode" ${test_command}; exit $?; diff --git a/test/utils/server.js b/test/utils/server.js index d88172dd2..ce633051c 100644 --- a/test/utils/server.js +++ b/test/utils/server.js @@ -4,24 +4,37 @@ const P = require('bluebird'); const TestRunner = require('service-runner/test/TestServer'); const DEFAULT_DOMAIN = 'en.wikipedia.org'; -class TestRestbase extends TestRunner { - constructor(configPath = `${__dirname}/../../config.test.yaml`, forceSkipBackend) { - super(configPath); - this._spinBackend = !forceSkipBackend && !!process.env.RB_TEST_BACKEND_HOST_TEMPLATE; - if (this._spinBackend) { - this._backendServer = new TestRestbase( - `${__dirname}/../../config.example.storage.wikimedia.yaml`, - true); +class TestRestbase { + constructor() { + const testMode = process.env.TEST_MODE || 'fs'; + switch (testMode) { + case 'fs': + this._frontendServer = new TestRunner(`${__dirname}/../../config.fullstack.test.yaml`); + this._backendServer = undefined; + break; + case 'ftfs': + this._frontendServer = new TestRunner(`${__dirname}/../../config.frontend.test.yaml`); + this._backendServer = new TestRunner(`${__dirname}/../../config.fullstack.test.yaml`); + break; + case 'ftbe': + this._frontendServer = new TestRunner(`${__dirname}/../../config.frontend.test.yaml`); + this._backendServer = new TestRunner(`${__dirname}/../../config.storage.test.yaml`); + break; + default: + throw new Error(`Invalid test mode ${testMode}`); } } get config() { - if (!this._running) { + if (!this._frontendServer._running) { throw new Error('Accessing test service config before starting the service'); } - const conf = this._runner._impl.config; + const conf = this._frontendServer._runner._impl.config; + const backendConf = this._backendServer ? this._backendServer._runner._impl.config : conf; const hostPort = `http://localhost:${conf.services[0].conf.port}`; + const backendHostPort = `http://localhost:${backendConf.services[0].conf.port}`; const baseURL = (domain = DEFAULT_DOMAIN) => `${hostPort}/${domain}/v1`; + const backendURL = (domain = DEFAULT_DOMAIN) => `${backendHostPort}/${domain}/v1`; const bucketURL = (domain) => `${baseURL(domain)}/page`; const apiPath = '/w/api.php'; const apiBase = (domain = DEFAULT_DOMAIN) => `https://${domain}`; @@ -30,6 +43,7 @@ class TestRestbase extends TestRunner { defaultDomain: DEFAULT_DOMAIN, hostPort, baseURL, + backendURL, bucketURL, apiBase, apiPath, @@ -40,13 +54,13 @@ class TestRestbase extends TestRunner { } start() { - const startPromise = this._spinBackend ? this._backendServer.start() : P.resolve(); - return startPromise.then(() => super.start()); + const startPromise = this._backendServer ? this._backendServer.start() : P.resolve(); + return startPromise.then(() => this._frontendServer.start()); } stop() { - const stopPromise = this._spinBackend ? this._backendServer.stop() : P.resolve(); - return stopPromise.then(() => super.stop()); + const stopPromise = this._backendServer ? this._backendServer.stop() : P.resolve(); + return stopPromise.then(() => this._frontendServer.stop()); } } diff --git a/v1/proxy.js b/v1/proxy.js deleted file mode 100644 index bac85d118..000000000 --- a/v1/proxy.js +++ /dev/null @@ -1,55 +0,0 @@ -'use strict'; - -const HyperSwitch = require('hyperswitch'); - -const specProxy = HyperSwitch.utils.loadSpec(`${__dirname}/../projects/sys/backend-proxy.yaml`); - -function addAuthFilter(spec) { - if (!spec.paths['/page_revisions']) { - return; - } - const pageRev = spec.paths['/page_revisions']; - pageRev['x-route-filters'] = pageRev['x-route-filters'] || []; - pageRev['x-route-filters'].push({ - path: './lib/mediawiki_auth_filter.js' - }); -} - -function hidePaths(spec) { - const paths = spec.paths; - Object.keys(paths).forEach((path) => { - paths[path]['x-hidden'] = true; - }); -} - -module.exports = (options) => { - let spec = {}; - options = options || {}; - options.service_type = options.service_type || 'fullstack'; - options.proxy = options.proxy || {}; - if (!Object.prototype.hasOwnProperty.call(options, 'expose_proxy')) { - options.proxy.expose_proxy = true; - } - if (options.proxy.expose_proxy) { - switch (options.service_type.toLowerCase()) { - case 'frontend': - break; - case 'backend': - spec = specProxy; - addAuthFilter(spec); - break; - case 'fullstack': - spec = specProxy; - addAuthFilter(spec); - hidePaths(spec); - break; - default: - throw new Error('Invalid value for option service_type supplied!'); - } - } - return { - spec, - operations: {}, - globals: { options } - }; -}; From 9625be0ff0b5fab6decae4a3cffa7e155df777c2 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Tue, 4 Jun 2019 17:28:54 +0300 Subject: [PATCH 9/9] Fix up proxy hiding and protecting Change-Id: Ice3f1ee70a8fbe358835d60b8b3d20b59e063ec7 --- .travis.yml | 5 +- config.frontend.test.yaml | 16 +- config.fullstack.test.yaml | 7 +- config.storage.test.yaml | 2 +- projects/v1/enwiki.wmf.yaml | 126 -------------- projects/wikimedia.org.yaml | 105 ++++++++++++ projects/wmf_default.yaml | 156 +++++++++++++++++ projects/wmf_enwiki.yaml | 210 +++++++++++++++++++++++ projects/wmf_wikidata.yaml | 124 ++++++++++++++ projects/wmf_wikipedia.yaml | 231 ++++++++++++++++++++++++++ projects/wmf_wikivoyage.yaml | 168 +++++++++++++++++++ projects/wmf_wiktionary.yaml | 163 ++++++++++++++++++ sys/backend_proxy.js | 49 ++---- test/features/pagecontent/save_api.js | 4 +- test/features/router/misc.js | 2 +- test/features/security/security.js | 3 + test/utils/run_tests.sh | 3 +- 17 files changed, 1194 insertions(+), 180 deletions(-) delete mode 100644 projects/v1/enwiki.wmf.yaml create mode 100644 projects/wikimedia.org.yaml create mode 100644 projects/wmf_default.yaml create mode 100644 projects/wmf_enwiki.yaml create mode 100644 projects/wmf_wikidata.yaml create mode 100644 projects/wmf_wikipedia.yaml create mode 100644 projects/wmf_wikivoyage.yaml create mode 100644 projects/wmf_wiktionary.yaml diff --git a/.travis.yml b/.travis.yml index ffd923d12..1a8ffede8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,9 @@ env: - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=fs - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftfs - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftfs - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftbe - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftbe +# Disabled until a new feature in restbase-mod-table-* is delivered. +# - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftbe +# - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftbe before_install: - wget https://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz -P ../ diff --git a/config.frontend.test.yaml b/config.frontend.test.yaml index 1bbcad983..7428a3c05 100644 --- a/config.frontend.test.yaml +++ b/config.frontend.test.yaml @@ -100,7 +100,7 @@ enwiki_project: &enwiki_project paths: /{api:v1}: x-modules: - - path: projects/v1/enwiki.wmf.yaml + - path: projects/v1/wikipedia.wmf.yaml options: *default_options /{api:sys}: *default_sys @@ -149,7 +149,7 @@ spec_root: &spec_root /{domain:test.wikipedia.org}: *wikipedia_project # The order is important for tests. # Redirect tests require en.wiki being not the first wiki in the list. - /{domain:en.wikipedia.org}: *enwiki_project + /{domain:en.wikipedia.org}: *wikipedia_project /{domain:ru.wikipedia.org}: *wikipedia_project /{domain:de.wikipedia.org}: *wikipedia_project /{domain:test2.wikipedia.org}: *wikipedia_project @@ -178,24 +178,24 @@ spec_root: &spec_root # Finally, a standard service-runner config. info: - name: restbase + name: restrouter services: - - name: restbase + - name: restrouter module: hyperswitch conf: - port: 7232 + port: 7233 spec: *spec_root salt: secret default_page_size: 1 - user_agent: RESTBase-testsuite + user_agent: RESTRouer-testsuite ui_name: RESTBase ui_url: https://www.mediawiki.org/wiki/RESTBase ui_title: RESTBase docs logging: - name: restbase-test - level: info + name: restrouter + level: warn streams: - type: stdout diff --git a/config.fullstack.test.yaml b/config.fullstack.test.yaml index 673f93b17..1f28ab1d5 100644 --- a/config.fullstack.test.yaml +++ b/config.fullstack.test.yaml @@ -53,7 +53,6 @@ default_project: &default_project - path: projects/proxy.yaml options: backend_host_template: '{{"/{domain}/sys/legacy"}}' - block_external_reqs: false - spec: paths: /legacy/key_value: @@ -98,7 +97,6 @@ wikimedia_project: &wikimedia_project - path: projects/proxy.yaml options: backend_host_template: '{{"/{domain}/sys/legacy"}}' - block_external_reqs: false - spec: paths: /legacy/key_value: @@ -142,7 +140,7 @@ enwiki_project: &enwiki_project paths: /{api:v1}: x-modules: - - path: projects/v1/enwiki.wmf.yaml + - path: projects/v1/wikipedia.wmf.yaml options: *default_options - path: projects/proxy.yaml options: @@ -179,7 +177,6 @@ spec_root: &spec_root # Some more general RESTBase info x-request-filters: - path: lib/security_response_header_filter.js - - path: lib/normalize_headers_filter.js x-sub-request-filters: - type: default @@ -244,7 +241,7 @@ services: logging: name: restbase-test - level: info + level: warn streams: - type: stdout diff --git a/config.storage.test.yaml b/config.storage.test.yaml index 6516ac78b..edf55aca6 100644 --- a/config.storage.test.yaml +++ b/config.storage.test.yaml @@ -76,7 +76,7 @@ services: logging: name: restbase - level: error + level: warn streams: - type: stdout diff --git a/projects/v1/enwiki.wmf.yaml b/projects/v1/enwiki.wmf.yaml deleted file mode 100644 index 458b1abc5..000000000 --- a/projects/v1/enwiki.wmf.yaml +++ /dev/null @@ -1,126 +0,0 @@ -openapi: 3.0.1 -info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 -# Override the base path for host-based (proxied) requests. In our case, -# we proxy https://{domain}/api/rest_v1/ to the API. -x-host-basePath: /api/rest_v1 -x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' -paths: - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/mobileapps.yaml - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/summary.js - options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, - options.summary)}}' - - path: v1/media.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/metadata.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/references.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/mobile-html.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/related.js - options: '{{options.related}}' - - path: v1/random.yaml - options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, - options.mobileapps)}}' - - path: v1/pdf.js - options: '{{options.pdf}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /feed: - x-modules: - - path: v1/feed.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - - path: v1/announcements.yaml - options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' - - path: v1/onthisday.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/citoid.js - options: '{{options.citoid}}' - - path: v1/lists.js - options: '{{options.lists}}' - - path: v1/recommend.yaml - options: '{{options.recommendation}}' - - path: v1/css.yaml - options: - host: '{{options.mobileapps.host}}' - - path: v1/javascript.yaml - options: - host: '{{options.mobileapps.host}}' diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml new file mode 100644 index 000000000..99cae3ee3 --- /dev/null +++ b/projects/wikimedia.org.yaml @@ -0,0 +1,105 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Software available under the Apache 2 license + url: http://www.apache.org/licenses/LICENSE-2.0 + + securityDefinitions: &wp/content-security/1.0.0 + mediawiki_auth: + description: Checks permissions using MW api + type: apiKey + in: header + name: cookie + x-internal-request-whitelist: + - /http:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + paths: + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /metrics: + x-modules: + - path: v1/metrics.yaml + options: '{{options.pageviews}}' + /transform: + x-modules: + - path: v1/transform-global.yaml + options: '{{options.transform}}' + /feed: + x-modules: + - path: v1/availability.yaml + options: '{{options.mobileapps}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /mathoid: + x-modules: + - path: sys/mathoid.js + options: '{{options.mathoid}}' + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml new file mode 100644 index 000000000..b1c21ee04 --- /dev/null +++ b/projects/wmf_default.yaml @@ -0,0 +1,156 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + # Wikisource + pl.wikisource.org: + - '/^Wśród_czarnych\//' + # Wikimedia + commons.wikimedia.org: + - '/Commons:Featured_picture_candidates\//' + - 'Commons:Quality_images/Subject/Places/Natural_structures' + - '/Commons:Undeletion_requests\//' + - '/Commons:WikiProject_Aviation\/recent_uploads\//' + - '/^(?:User|Benutzer):/' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml new file mode 100644 index 000000000..242655145 --- /dev/null +++ b/projects/wmf_enwiki.yaml @@ -0,0 +1,210 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /feed: + x-modules: + - path: v1/feed.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + - path: v1/announcements.yaml + options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' + - path: v1/onthisday.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + en.wikipedia.org: + - 'User:B-bot/Event_log' + - 'User:DeltaQuad/UAA/Wait' + - 'User:JamesR/AdminStats' + - 'User:Kudpung/Dashboard' + # Various dashboards + - 'User:Breawycker/Wikipedia' + - 'User:Sonia/dashboard' + - 'User:Ocaasi/dashboard' + - 'User:Nolelover' + - 'User:Calmer_Waters' + - '/User%3ARedwolf24\//' + - 'User:Technical_13/dashboard' + - 'Template:Cratstats' + # Cyberbot is creating 90% of null edits + - '/^User:Cyberbot_I\//' + - '/^User:Cyberbot_II\//' + - '/^User:Cyberpower678\//' + - '/^User:Darts170Darts170\//' + - 'صارف:Cyberbot_I/Run/Adminstats' + - 'Defnyddiwr:Cyberbot_I/Run/Adminstats' + - 'User:Pentjuuu!.!/sandbox' + - 'User:AllyD/CSDlog' + - 'User:Peter_I._Vardy/sandbox-13' + - 'User:I_dream_of_horses/CSD_log' + - 'User:MJ180MJ180/sandbox' + - 'Talk:United_States_presidential_election,_2016' + - 'Wikipedia:Reference_desk/Humanities' + - 'Wikipedia:WikiProject_Deletion_sorting/People' + - 'Wikipedia:WikiProject_Deletion_sorting/United_States_of_America' + - 'Wikipedia:Articles_for_creation/Redirects' + - 'Wikipedia:Administrators%27_noticeboard/Incidents' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml new file mode 100644 index 000000000..471969351 --- /dev/null +++ b/projects/wmf_wikidata.yaml @@ -0,0 +1,124 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, options.mobileapps)}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/lists.js + options: '{{options.lists}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wikipedia.yaml b/projects/wmf_wikipedia.yaml new file mode 100644 index 000000000..090f8c0f6 --- /dev/null +++ b/projects/wmf_wikipedia.yaml @@ -0,0 +1,231 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /feed: + x-modules: + - path: v1/feed.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + - path: v1/announcements.yaml + options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' + - path: v1/onthisday.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + # Wikipedia + ca.wikipedia.org: + - 'Usuari:TronaBot/log:Activitat_reversors_per_hores' + ceb.wikipedia.org: + - 'Gumagamit:Lsjbot/Anomalier-PRIVAT' + - 'Gumagamit:Lsjbot/Kartrutor2' + de.wikipedia.org: + - '/The_Big_Bang_Theory\/Staffel/' + - 'Wikipedia:Café' + - 'Wikipedia:Defekte_Weblinks/Bot2015-Problem' + - 'Wikipedia_Diskussion:Hauptseite/Schon_gewusst' + - 'Benutzer:Anglo-Araneophilus/Almigdad_Mojalli' + - 'Benutzer:Wartungsstube/Berlin' + - 'Benutzer:Wartungsstube/Musik' + - 'Benutzer:Wartungsstube/Unternehmen' + - 'Benutzer:Wartungsstube/Schifffahrt' + - 'Benutzer:Verum/ege' + - 'Benutzer:Septembermorgen/Bottabelle/Französische_Kantone_N–Z' + - 'Wikipedia:WikiProjekt_Planen_und_Bauen/Zu_überarbeitende_Artikel' + es.wikipedia.org: + - 'Wikipedia:Café/Archivo/Miscelánea/Actual' + fr.wikipedia.org: + - 'Utilisateur:ZéroBot/Log/Erreurs' + - 'Utilisateur:SyntaxTerror/Ajouts_du_modèle_Autorité' + - '/^Utilisateur:[\s\S]+[Bb]rouillon' + - 'Discussion_utilisateur:NaggoBot/CommonsDR' + - 'Projet:France/Annonces/Admissibilité' + - '/Wikipédia:Le_saviez-vous_.+/Anecdotes_proposées/' + hy.wikipedia.org: + - "/Մասնակից:Omicroñ\\'R/" + it.wikipedia.org: + - 'Utente:Effems/Sandbox7' + nl.wikipedia.org: + - 'Gebruiker:Eg-T2g/Kladblok' + pt.wikipedia.org: + - 'Wikipédia:Pedidos/Bloqueio' + ru.wikipedia.org: + - 'Википедия:Форум/Технический' + - 'Портал:Герпетология' + sv.wikipedia.org: + - 'Användare:Lsjbot/Anomalier-PRIVAT' + - 'Användare:Lsjbot/Namnkonflikter-PRIVAT' + ur.wikipedia.org: + - 'نام_مقامات_ایل' + - 'نام_مقامات_ڈی' + - 'نام_مقامات_جے' + - 'نام_مقامات_جی' + - 'نام_مقامات_ایچ' + - 'نام_مقامات_ایم' + - 'نام_مقامات_ایس' + zh.wikipedia.org: + - 'Wikipedia:互助客栈/条目探讨' + - 'Draft:日本人工湖列表' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wikivoyage.yaml b/projects/wmf_wikivoyage.yaml new file mode 100644 index 000000000..6e02b5da6 --- /dev/null +++ b/projects/wmf_wikivoyage.yaml @@ -0,0 +1,168 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + # NOTE: We're setting the `no-store` cache-control header so that we don't + # store MCS results - we don't have pre-generation for wikivoyage. + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps, + {"cache-control": "no-store"})}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml new file mode 100644 index 000000000..194a99acb --- /dev/null +++ b/projects/wmf_wiktionary.yaml @@ -0,0 +1,163 @@ +openapi: 3.0.1 +paths: + /{api:v1}: &default_project_paths_v1 + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /: + x-modules: + - path: v1/common_schemas.yaml + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/definition.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js index e0d7424de..5f0e69b17 100644 --- a/sys/backend_proxy.js +++ b/sys/backend_proxy.js @@ -1,59 +1,42 @@ 'use strict'; -const P = require('bluebird'); const HyperSwitch = require('hyperswitch'); - const Template = HyperSwitch.Template; -const HTTPError = HyperSwitch.HTTPError; module.exports = (options) => { options = options || {}; options.backend_host_template = options.backend_host_template || '/{domain}/sys'; - if (!Object.prototype.hasOwnProperty.call(options, 'use_path_segment')) { - options.use_path_segment = true; - } - if (!Object.prototype.hasOwnProperty.call(options, 'block_external_reqs')) { - options.block_external_reqs = true; - } const backendURITemplate = new Template({ uri: `${options.backend_host_template}/{{path}}` }); - const usePathSegment = options.use_path_segment; - const blockExternalReqs = options.block_external_reqs; return { spec: { paths: { '/{+path}': { - 'x-hidden': true, + 'x-route-filters': options.block_external_reqs ? + [{ + type: 'default', + name: 'header_match', + options: { + whitelist: { + 'x-client-ip': ['/^(?:::ffff:)?(?:10|127)\\./'] + } + } + }] : [], all: { operationId: 'proxy', - 'x-monitor': false + 'x-monitor': false, + 'x-hidden': true } } } }, operations: { proxy: (hyper, req) => { - if (blockExternalReqs && !hyper._isSysRequest(req) && - req.headers['x-request-class'] === 'external') { - return P.reject(new HTTPError({ - status: 403, - body: { - type: 'forbidden', - title: 'Forbidden', - description: 'You are not allowed to access this URI' - } - })); - } - if (usePathSegment) { - // if usePathSegment is set (true by default), then the proxy module - // will include the path segment preceding the specified path to - // construct the full request URI - const uri = req.uri.toString(); - const uriPrefix = uri.substring(0, uri.indexOf(`/${req.params.path}`)); - const segment = uriPrefix.split('/').pop(); - req.params.path = `${segment}/${req.params.path}`; - } + const uri = req.uri.toString(); + const uriPrefix = uri.substring(0, uri.indexOf(`/${req.params.path}`)); + const segment = uriPrefix.split('/').pop(); + req.params.path = `${segment}/${req.params.path}`; return hyper.request({ method: req.method, uri: backendURITemplate.expand({ request: req }).uri, diff --git a/test/features/pagecontent/save_api.js b/test/features/pagecontent/save_api.js index 8985eb32a..75b0785f3 100644 --- a/test/features/pagecontent/save_api.js +++ b/test/features/pagecontent/save_api.js @@ -376,7 +376,7 @@ describe('page save api', function() { } }); - it('save HTML', () => { + it('save HTML', () => { const test = () => { return preq.get({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/${pageTitle}/${lastRev}` @@ -385,7 +385,6 @@ describe('page save api', function() { return preq.post({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/${pageTitle}`, headers: { - 'x-client-ip': '123.123.123.123', cookie: 'test' }, body: { @@ -403,7 +402,6 @@ describe('page save api', function() { if (NOCK_TESTS) { const api = nock(server.config.apiURL('en.wikipedia.beta.wmflabs.org'), { reqheaders: { - 'x-client-ip': '123.123.123.123', 'x-forwarded-for'(headerValue) { return headerValue.indexOf('127.0.0.1') >= 0; }, diff --git a/test/features/router/misc.js b/test/features/router/misc.js index b15eab902..e5c2fcfc6 100644 --- a/test/features/router/misc.js +++ b/test/features/router/misc.js @@ -18,7 +18,7 @@ describe('router - misc', function() { after(() => server.stop()); it('should deny access to /{domain}/sys', () => { - return preq.get({uri: `${server.config.hostPort}/en.wikipedia.org/sys/table`}) + return preq.get({uri: `${server.config.hostPort}/en.wikipedia.org/sys/action/query`}) .catch((err) => { assert.deepEqual(err.status, 403); }); diff --git a/test/features/security/security.js b/test/features/security/security.js index 942358f9d..86c74ce0a 100644 --- a/test/features/security/security.js +++ b/test/features/security/security.js @@ -59,6 +59,9 @@ describe('router - security', function() { .post('', (body) => { return body && body.generator === 'allpages'; }) .reply(200, sampleApiResponse) .post('', (body) => { return body && body.meta === 'userinfo'; }) + .reply(200, sampleRightsResponse) + .post('', (body) => { return body && body.meta === 'userinfo'; }) + .optionally() .reply(200, sampleRightsResponse); return preq.get({ diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index cafe2c9c1..ffb87dbd5 100644 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -4,7 +4,8 @@ mod_dir=$( cd "$( dirname "$0" )"/../.. && pwd )/node_modules mocha="${mod_dir}"/mocha/bin/mocha nyc="${mod_dir}"/.bin/nyc test_target=${TEST_TARGET:-$2} -test_mode=${TEST_MODE:=$3} +test_mode=${TEST_MODE:-$3} +export TEST_MODE=${test_mode} if [ "$1" = "test" ]; then test_command="${mocha}"