diff --git a/.travis.yml b/.travis.yml index ffd923d12..1a8ffede8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,9 @@ env: - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=fs - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftfs - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftfs - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftbe - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftbe +# Disabled until a new feature in restbase-mod-table-* is delivered. +# - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftbe +# - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftbe before_install: - wget https://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz -P ../ diff --git a/config.frontend.test.yaml b/config.frontend.test.yaml index 1bbcad983..7428a3c05 100644 --- a/config.frontend.test.yaml +++ b/config.frontend.test.yaml @@ -100,7 +100,7 @@ enwiki_project: &enwiki_project paths: /{api:v1}: x-modules: - - path: projects/v1/enwiki.wmf.yaml + - path: projects/v1/wikipedia.wmf.yaml options: *default_options /{api:sys}: *default_sys @@ -149,7 +149,7 @@ spec_root: &spec_root /{domain:test.wikipedia.org}: *wikipedia_project # The order is important for tests. # Redirect tests require en.wiki being not the first wiki in the list. - /{domain:en.wikipedia.org}: *enwiki_project + /{domain:en.wikipedia.org}: *wikipedia_project /{domain:ru.wikipedia.org}: *wikipedia_project /{domain:de.wikipedia.org}: *wikipedia_project /{domain:test2.wikipedia.org}: *wikipedia_project @@ -178,24 +178,24 @@ spec_root: &spec_root # Finally, a standard service-runner config. info: - name: restbase + name: restrouter services: - - name: restbase + - name: restrouter module: hyperswitch conf: - port: 7232 + port: 7233 spec: *spec_root salt: secret default_page_size: 1 - user_agent: RESTBase-testsuite + user_agent: RESTRouer-testsuite ui_name: RESTBase ui_url: https://www.mediawiki.org/wiki/RESTBase ui_title: RESTBase docs logging: - name: restbase-test - level: info + name: restrouter + level: warn streams: - type: stdout diff --git a/config.fullstack.test.yaml b/config.fullstack.test.yaml index 673f93b17..1f28ab1d5 100644 --- a/config.fullstack.test.yaml +++ b/config.fullstack.test.yaml @@ -53,7 +53,6 @@ default_project: &default_project - path: projects/proxy.yaml options: backend_host_template: '{{"/{domain}/sys/legacy"}}' - block_external_reqs: false - spec: paths: /legacy/key_value: @@ -98,7 +97,6 @@ wikimedia_project: &wikimedia_project - path: projects/proxy.yaml options: backend_host_template: '{{"/{domain}/sys/legacy"}}' - block_external_reqs: false - spec: paths: /legacy/key_value: @@ -142,7 +140,7 @@ enwiki_project: &enwiki_project paths: /{api:v1}: x-modules: - - path: projects/v1/enwiki.wmf.yaml + - path: projects/v1/wikipedia.wmf.yaml options: *default_options - path: projects/proxy.yaml options: @@ -179,7 +177,6 @@ spec_root: &spec_root # Some more general RESTBase info x-request-filters: - path: lib/security_response_header_filter.js - - path: lib/normalize_headers_filter.js x-sub-request-filters: - type: default @@ -244,7 +241,7 @@ services: logging: name: restbase-test - level: info + level: warn streams: - type: stdout diff --git a/config.storage.test.yaml b/config.storage.test.yaml index 6516ac78b..edf55aca6 100644 --- a/config.storage.test.yaml +++ b/config.storage.test.yaml @@ -76,7 +76,7 @@ services: logging: name: restbase - level: error + level: warn streams: - type: stdout diff --git a/projects/v1/enwiki.wmf.yaml b/projects/v1/enwiki.wmf.yaml deleted file mode 100644 index 458b1abc5..000000000 --- a/projects/v1/enwiki.wmf.yaml +++ /dev/null @@ -1,126 +0,0 @@ -openapi: 3.0.1 -info: - version: 1.0.0 - title: Wikimedia REST API - description: > - This API provides cacheable and straightforward access to - Wikimedia content and data, in machine-readable formats. - - ### Global Rules - - - Limit your clients to no more than 200 requests/s to this API. - Each API endpoint's documentation may detail more specific usage limits. - - Set a unique `User-Agent` or `Api-User-Agent` header that - allows us to contact you quickly. Email addresses or URLs - of contact pages work well. - - By using this API, you agree to Wikimedia's - [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and - [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). - Unless otherwise specified in the endpoint documentation - below, content accessed via this API is licensed under the - [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) - and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, - and you irrevocably agree to release modifications or - additions made through this API under these licenses. - See https://www.mediawiki.org/wiki/REST_API for background and details. - - ### Endpoint documentation - - Please consult each endpoint's documentation for details on: - - - Licensing information for the specific type of content - and data served via the endpoint. - - Stability markers to inform you about development status and - change policy, according to - [our API version policy](https://www.mediawiki.org/wiki/API_versioning). - - Endpoint specific usage limits. - - termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use - contact: - name: the Wikimedia Services team - url: http://mediawiki.org/wiki/REST_API - license: - name: Apache2 - url: http://www.apache.org/licenses/LICENSE-2.0 -# Override the base path for host-based (proxied) requests. In our case, -# we proxy https://{domain}/api/rest_v1/ to the API. -x-host-basePath: /api/rest_v1 -x-route-filters: - - path: lib/content_location_filter.js - - path: ./lib/normalize_title_filter.js - options: - redirect_cache_control: '{{options.purged_cache_control}}' -paths: - /page: - x-modules: - - path: v1/content.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - - path: v1/content_segments.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - cx_host: '{{options.transform.cx_host}}' - - path: v1/mobileapps.yaml - options: '{{merge({"response_cache_control": options.purged_cache_control}, - options.mobileapps)}}' - - path: v1/graphoid.yaml - options: '{{options.graphoid}}' - - path: v1/summary.js - options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, - options.summary)}}' - - path: v1/media.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/metadata.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/references.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/mobile-html.yaml - options: - response_cache_control: '{{options.purged_cache_control}}' - host: '{{options.mobileapps.host}}' - - path: v1/related.js - options: '{{options.related}}' - - path: v1/random.yaml - options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, - options.mobileapps)}}' - - path: v1/pdf.js - options: '{{options.pdf}}' - - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. - /feed: - x-modules: - - path: v1/feed.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - - path: v1/announcements.yaml - options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' - - path: v1/onthisday.js - options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' - /transform: - x-modules: - - path: v1/transform.yaml - - path: v1/transform-lang.js - options: '{{options.transform}}' - /media: - x-modules: - - path: v1/mathoid.yaml - options: '{{options.mathoid}}' - /data: - x-modules: - - path: v1/citoid.js - options: '{{options.citoid}}' - - path: v1/lists.js - options: '{{options.lists}}' - - path: v1/recommend.yaml - options: '{{options.recommendation}}' - - path: v1/css.yaml - options: - host: '{{options.mobileapps.host}}' - - path: v1/javascript.yaml - options: - host: '{{options.mobileapps.host}}' diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml new file mode 100644 index 000000000..99cae3ee3 --- /dev/null +++ b/projects/wikimedia.org.yaml @@ -0,0 +1,105 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Software available under the Apache 2 license + url: http://www.apache.org/licenses/LICENSE-2.0 + + securityDefinitions: &wp/content-security/1.0.0 + mediawiki_auth: + description: Checks permissions using MW api + type: apiKey + in: header + name: cookie + x-internal-request-whitelist: + - /http:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + paths: + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /metrics: + x-modules: + - path: v1/metrics.yaml + options: '{{options.pageviews}}' + /transform: + x-modules: + - path: v1/transform-global.yaml + options: '{{options.transform}}' + /feed: + x-modules: + - path: v1/availability.yaml + options: '{{options.mobileapps}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /mathoid: + x-modules: + - path: sys/mathoid.js + options: '{{options.mathoid}}' + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml new file mode 100644 index 000000000..b1c21ee04 --- /dev/null +++ b/projects/wmf_default.yaml @@ -0,0 +1,156 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + # Wikisource + pl.wikisource.org: + - '/^Wśród_czarnych\//' + # Wikimedia + commons.wikimedia.org: + - '/Commons:Featured_picture_candidates\//' + - 'Commons:Quality_images/Subject/Places/Natural_structures' + - '/Commons:Undeletion_requests\//' + - '/Commons:WikiProject_Aviation\/recent_uploads\//' + - '/^(?:User|Benutzer):/' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml new file mode 100644 index 000000000..242655145 --- /dev/null +++ b/projects/wmf_enwiki.yaml @@ -0,0 +1,210 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /feed: + x-modules: + - path: v1/feed.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + - path: v1/announcements.yaml + options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' + - path: v1/onthisday.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + en.wikipedia.org: + - 'User:B-bot/Event_log' + - 'User:DeltaQuad/UAA/Wait' + - 'User:JamesR/AdminStats' + - 'User:Kudpung/Dashboard' + # Various dashboards + - 'User:Breawycker/Wikipedia' + - 'User:Sonia/dashboard' + - 'User:Ocaasi/dashboard' + - 'User:Nolelover' + - 'User:Calmer_Waters' + - '/User%3ARedwolf24\//' + - 'User:Technical_13/dashboard' + - 'Template:Cratstats' + # Cyberbot is creating 90% of null edits + - '/^User:Cyberbot_I\//' + - '/^User:Cyberbot_II\//' + - '/^User:Cyberpower678\//' + - '/^User:Darts170Darts170\//' + - 'صارف:Cyberbot_I/Run/Adminstats' + - 'Defnyddiwr:Cyberbot_I/Run/Adminstats' + - 'User:Pentjuuu!.!/sandbox' + - 'User:AllyD/CSDlog' + - 'User:Peter_I._Vardy/sandbox-13' + - 'User:I_dream_of_horses/CSD_log' + - 'User:MJ180MJ180/sandbox' + - 'Talk:United_States_presidential_election,_2016' + - 'Wikipedia:Reference_desk/Humanities' + - 'Wikipedia:WikiProject_Deletion_sorting/People' + - 'Wikipedia:WikiProject_Deletion_sorting/United_States_of_America' + - 'Wikipedia:Articles_for_creation/Redirects' + - 'Wikipedia:Administrators%27_noticeboard/Incidents' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml new file mode 100644 index 000000000..471969351 --- /dev/null +++ b/projects/wmf_wikidata.yaml @@ -0,0 +1,124 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, options.mobileapps)}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/lists.js + options: '{{options.lists}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wikipedia.yaml b/projects/wmf_wikipedia.yaml new file mode 100644 index 000000000..090f8c0f6 --- /dev/null +++ b/projects/wmf_wikipedia.yaml @@ -0,0 +1,231 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /feed: + x-modules: + - path: v1/feed.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + - path: v1/announcements.yaml + options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' + - path: v1/onthisday.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + # Wikipedia + ca.wikipedia.org: + - 'Usuari:TronaBot/log:Activitat_reversors_per_hores' + ceb.wikipedia.org: + - 'Gumagamit:Lsjbot/Anomalier-PRIVAT' + - 'Gumagamit:Lsjbot/Kartrutor2' + de.wikipedia.org: + - '/The_Big_Bang_Theory\/Staffel/' + - 'Wikipedia:Café' + - 'Wikipedia:Defekte_Weblinks/Bot2015-Problem' + - 'Wikipedia_Diskussion:Hauptseite/Schon_gewusst' + - 'Benutzer:Anglo-Araneophilus/Almigdad_Mojalli' + - 'Benutzer:Wartungsstube/Berlin' + - 'Benutzer:Wartungsstube/Musik' + - 'Benutzer:Wartungsstube/Unternehmen' + - 'Benutzer:Wartungsstube/Schifffahrt' + - 'Benutzer:Verum/ege' + - 'Benutzer:Septembermorgen/Bottabelle/Französische_Kantone_N–Z' + - 'Wikipedia:WikiProjekt_Planen_und_Bauen/Zu_überarbeitende_Artikel' + es.wikipedia.org: + - 'Wikipedia:Café/Archivo/Miscelánea/Actual' + fr.wikipedia.org: + - 'Utilisateur:ZéroBot/Log/Erreurs' + - 'Utilisateur:SyntaxTerror/Ajouts_du_modèle_Autorité' + - '/^Utilisateur:[\s\S]+[Bb]rouillon' + - 'Discussion_utilisateur:NaggoBot/CommonsDR' + - 'Projet:France/Annonces/Admissibilité' + - '/Wikipédia:Le_saviez-vous_.+/Anecdotes_proposées/' + hy.wikipedia.org: + - "/Մասնակից:Omicroñ\\'R/" + it.wikipedia.org: + - 'Utente:Effems/Sandbox7' + nl.wikipedia.org: + - 'Gebruiker:Eg-T2g/Kladblok' + pt.wikipedia.org: + - 'Wikipédia:Pedidos/Bloqueio' + ru.wikipedia.org: + - 'Википедия:Форум/Технический' + - 'Портал:Герпетология' + sv.wikipedia.org: + - 'Användare:Lsjbot/Anomalier-PRIVAT' + - 'Användare:Lsjbot/Namnkonflikter-PRIVAT' + ur.wikipedia.org: + - 'نام_مقامات_ایل' + - 'نام_مقامات_ڈی' + - 'نام_مقامات_جے' + - 'نام_مقامات_جی' + - 'نام_مقامات_ایچ' + - 'نام_مقامات_ایم' + - 'نام_مقامات_ایس' + zh.wikipedia.org: + - 'Wikipedia:互助客栈/条目探讨' + - 'Draft:日本人工湖列表' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wikivoyage.yaml b/projects/wmf_wikivoyage.yaml new file mode 100644 index 000000000..6e02b5da6 --- /dev/null +++ b/projects/wmf_wikivoyage.yaml @@ -0,0 +1,168 @@ +paths: + /{api:v1}: + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + # NOTE: We're setting the `no-store` cache-control header so that we don't + # store MCS results - we don't have pre-generation for wikivoyage. + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps, + {"cache-control": "no-store"})}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml new file mode 100644 index 000000000..194a99acb --- /dev/null +++ b/projects/wmf_wiktionary.yaml @@ -0,0 +1,163 @@ +openapi: 3.0.1 +paths: + /{api:v1}: &default_project_paths_v1 + x-modules: + # swagger options, overriding the shared ones from the merged specs (?) + - spec: + info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 + # Override the base path for host-based (proxied) requests. In our case, + # we proxy https://{domain}/api/rest_v1/ to the API. + x-host-basePath: /api/rest_v1 + x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' + paths: + /: + x-modules: + - path: v1/common_schemas.yaml + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/definition.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + options: '{{options}}' + + /{api:sys}: + x-modules: + - spec: + paths: + /table: &sys_table + x-modules: + - path: sys/table.js + options: + conf: '{{options.table}}' + /key_value: &sys_key_value + x-modules: + - path: sys/key_value.js + /page_revisions: + x-modules: + - path: sys/page_revisions.js + /post_data: &sys_post_data + x-modules: + - path: sys/post_data.js + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: &parsoid_options + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' + options: '{{options}}' diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js index e0d7424de..5f0e69b17 100644 --- a/sys/backend_proxy.js +++ b/sys/backend_proxy.js @@ -1,59 +1,42 @@ 'use strict'; -const P = require('bluebird'); const HyperSwitch = require('hyperswitch'); - const Template = HyperSwitch.Template; -const HTTPError = HyperSwitch.HTTPError; module.exports = (options) => { options = options || {}; options.backend_host_template = options.backend_host_template || '/{domain}/sys'; - if (!Object.prototype.hasOwnProperty.call(options, 'use_path_segment')) { - options.use_path_segment = true; - } - if (!Object.prototype.hasOwnProperty.call(options, 'block_external_reqs')) { - options.block_external_reqs = true; - } const backendURITemplate = new Template({ uri: `${options.backend_host_template}/{{path}}` }); - const usePathSegment = options.use_path_segment; - const blockExternalReqs = options.block_external_reqs; return { spec: { paths: { '/{+path}': { - 'x-hidden': true, + 'x-route-filters': options.block_external_reqs ? + [{ + type: 'default', + name: 'header_match', + options: { + whitelist: { + 'x-client-ip': ['/^(?:::ffff:)?(?:10|127)\\./'] + } + } + }] : [], all: { operationId: 'proxy', - 'x-monitor': false + 'x-monitor': false, + 'x-hidden': true } } } }, operations: { proxy: (hyper, req) => { - if (blockExternalReqs && !hyper._isSysRequest(req) && - req.headers['x-request-class'] === 'external') { - return P.reject(new HTTPError({ - status: 403, - body: { - type: 'forbidden', - title: 'Forbidden', - description: 'You are not allowed to access this URI' - } - })); - } - if (usePathSegment) { - // if usePathSegment is set (true by default), then the proxy module - // will include the path segment preceding the specified path to - // construct the full request URI - const uri = req.uri.toString(); - const uriPrefix = uri.substring(0, uri.indexOf(`/${req.params.path}`)); - const segment = uriPrefix.split('/').pop(); - req.params.path = `${segment}/${req.params.path}`; - } + const uri = req.uri.toString(); + const uriPrefix = uri.substring(0, uri.indexOf(`/${req.params.path}`)); + const segment = uriPrefix.split('/').pop(); + req.params.path = `${segment}/${req.params.path}`; return hyper.request({ method: req.method, uri: backendURITemplate.expand({ request: req }).uri, diff --git a/test/features/pagecontent/save_api.js b/test/features/pagecontent/save_api.js index 8985eb32a..75b0785f3 100644 --- a/test/features/pagecontent/save_api.js +++ b/test/features/pagecontent/save_api.js @@ -376,7 +376,7 @@ describe('page save api', function() { } }); - it('save HTML', () => { + it('save HTML', () => { const test = () => { return preq.get({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/${pageTitle}/${lastRev}` @@ -385,7 +385,6 @@ describe('page save api', function() { return preq.post({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/${pageTitle}`, headers: { - 'x-client-ip': '123.123.123.123', cookie: 'test' }, body: { @@ -403,7 +402,6 @@ describe('page save api', function() { if (NOCK_TESTS) { const api = nock(server.config.apiURL('en.wikipedia.beta.wmflabs.org'), { reqheaders: { - 'x-client-ip': '123.123.123.123', 'x-forwarded-for'(headerValue) { return headerValue.indexOf('127.0.0.1') >= 0; }, diff --git a/test/features/router/misc.js b/test/features/router/misc.js index b15eab902..e5c2fcfc6 100644 --- a/test/features/router/misc.js +++ b/test/features/router/misc.js @@ -18,7 +18,7 @@ describe('router - misc', function() { after(() => server.stop()); it('should deny access to /{domain}/sys', () => { - return preq.get({uri: `${server.config.hostPort}/en.wikipedia.org/sys/table`}) + return preq.get({uri: `${server.config.hostPort}/en.wikipedia.org/sys/action/query`}) .catch((err) => { assert.deepEqual(err.status, 403); }); diff --git a/test/features/security/security.js b/test/features/security/security.js index 942358f9d..86c74ce0a 100644 --- a/test/features/security/security.js +++ b/test/features/security/security.js @@ -59,6 +59,9 @@ describe('router - security', function() { .post('', (body) => { return body && body.generator === 'allpages'; }) .reply(200, sampleApiResponse) .post('', (body) => { return body && body.meta === 'userinfo'; }) + .reply(200, sampleRightsResponse) + .post('', (body) => { return body && body.meta === 'userinfo'; }) + .optionally() .reply(200, sampleRightsResponse); return preq.get({ diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index cafe2c9c1..ffb87dbd5 100644 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -4,7 +4,8 @@ mod_dir=$( cd "$( dirname "$0" )"/../.. && pwd )/node_modules mocha="${mod_dir}"/mocha/bin/mocha nyc="${mod_dir}"/.bin/nyc test_target=${TEST_TARGET:-$2} -test_mode=${TEST_MODE:=$3} +test_mode=${TEST_MODE:-$3} +export TEST_MODE=${test_mode} if [ "$1" = "test" ]; then test_command="${mocha}"