diff --git a/.travis.yml b/.travis.yml index 66ecf7e8f..1a8ffede8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,13 @@ addons: - openjdk-8-jre-headless env: - - CASSANDRA_VERSION=3.11.2 TEST_TARGET=all + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=fs + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=fs + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftfs + - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftfs +# Disabled until a new feature in restbase-mod-table-* is delivered. +# - CASSANDRA_VERSION=3.11.2 TEST_TARGET=sqlite TEST_MODE=ftbe +# - CASSANDRA_VERSION=3.11.2 TEST_TARGET=cassandra TEST_MODE=ftbe before_install: - wget https://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz -P ../ @@ -29,4 +35,4 @@ before_install: - sed -i -e 's/^-XX:+UseNUMA/#-XX:+UseNUMA/' ../apache-cassandra-${CASSANDRA_VERSION}/conf/jvm.options - bash -x ../apache-cassandra-${CASSANDRA_VERSION}/bin/cassandra -script: npm run lint && npm run coverage -- ${TEST_TARGET} && (npm run-script coveralls || exit 0) +script: npm run lint && npm run coverage -- ${TEST_TARGET} ${TEST_MODE} && (npm run-script coveralls || exit 0) diff --git a/config.example.wikimedia.yaml b/config.example.wikimedia.yaml deleted file mode 100644 index 7c959822f..000000000 --- a/config.example.wikimedia.yaml +++ /dev/null @@ -1,181 +0,0 @@ -# RESTBase wikimedia example config - -# Load some project templates. These are referenced / shared between domains -# in the root_spec further down. -default_project: &default_project - x-modules: - - path: projects/wmf_default.yaml - options: &default_options - table: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: test.group.local - domains: /./ - # ignored in cassandra, but useful in SQLite testing. only used in tests. - dbname: test.db.sqlite3 - parsoid: - host: https://parsoid-beta.wmflabs.org - grace_ttl: 1000000 - stash_ratelimit: 5 - action: - apiUriTemplate: "{{'https://{domain}/w/api.php'}}" - baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" - graphoid: - host: https://graphoid-beta.wmflabs.org - mathoid: - host: https://mathoid-beta.wmflabs.org - # 10 days Varnish caching, one day client-side - cache-control: s-maxage=864000, max-age=86400 - mobileapps: - host: https://appservice.wmflabs.org - summary: - protocol: https - implementation: mcs - host: https://appservice.wmflabs.org - related: - cache_control: s-maxage=86400, max-age=86400 - citoid: - host: https://citoid-beta.wmflabs.org - recommendation: - host: https://recommendation-api-beta.wmflabs.org - # 10 days Varnish caching, one day client-side - purged_cache_control: s-maxage=1209600, max-age=0, must-revalidate - # Cache control for purged endpoints allowing short-term client caching - purged_cache_control_client_cache: s-maxage=1209600, max-age=300 - pdf: - # Cache PDF for 5 minutes since it's not purged - cache_control: s-maxage=600, max-age=600 - uri: https://proton-beta.wmflabs.org - transform: - cx_host: https://cxserver-beta.wmflabs.org - skip_updates: false - -# A separate project for en.wikipedia because it is more feature-rich -en.wikipedia.org: &en.wikipedia.org - x-modules: - - path: projects/wmf_enwiki.yaml - options: *default_options - -# A different project template, sharing configuration options. -wikimedia.org: &wikimedia.org - x-modules: - - path: projects/wikimedia.org.yaml - options: - <<: *default_options - pageviews: - host: https://wikimedia.org/api/rest_v1/metrics - -wikidata.org: &wikidata.org - x-modules: - - path: projects/wmf_wikidata.yaml - options: *default_options - -wiktionary_project: &wiktionary_project - x-modules: - - path: projects/wmf_wiktionary.yaml - options: *default_options - -wikipedia_project: &wikipedia_project - x-modules: - - path: projects/wmf_wikipedia.yaml - options: *default_options - -wikivoyage_project: &wikivoyage_project - x-modules: - - path: projects/wmf_wikivoyage.yaml - options: *default_options - -# The root of the spec tree. Domains tend to share specs by referencing them -# using YAML references. -spec_root: &spec_root - title: "The RESTBase root" - x-request-filters: - - path: lib/security_response_header_filter.js - - path: lib/normalize_headers_filter.js - x-sub-request-filters: - - type: default - name: http - options: - allow: - - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ - forward_headers: true - - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ - forward_headers: true - - pattern: /^https?:\/\// - paths: - /{domain:en.wikipedia.org}: *en.wikipedia.org - /{domain:ru.wikipedia.org}: *wikipedia_project - /{domain:de.wikipedia.org}: *wikipedia_project - /{domain:es.wikipedia.org}: *wikipedia_project - /{domain:it.wikipedia.org}: *wikipedia_project - /{domain:da.wikipedia.org}: *wikipedia_project - /{domain:sv.wikipedia.org}: *wikipedia_project - - # Example for wiktionary - /{domain:en.wiktionary.org}: *wiktionary_project - - # global domain - /{domain:wikimedia.org}: *wikimedia.org - - # special case for wikidata - /{domain:www.wikidata.org}: *wikidata.org - - # example for non-wikipedia - /{domain:en.wikivoyage.beta.wmflabs.org}: *wikivoyage_project - - # A robots.txt to make sure that the content isn't indexed. - /robots.txt: - get: - x-request-handler: - - static: - return: - status: 200 - headers: - content-type: text/plain - body: | - User-agent: * - Allow: /*/v1/?doc - Disallow: / - -# Finally, a standard service-runner config. -info: - name: restbase - -services: - - name: restbase - module: hyperswitch - conf: - port: 7231 - spec: *spec_root - salt: secret - default_page_size: 125 - user_agent: RESTBase - ui_name: RESTBase - ui_url: https://www.mediawiki.org/wiki/RESTBase - ui_title: RESTBase docs - -logging: - name: restbase - level: warn - streams: - - type: stdout - -#metrics: -# type: statsd -# host: localhost -# port: 8125 -# batch: true - -ratelimiter: - type: kademlia - # Cluster nodes - seeds: - # Port 3050 used by default - - 127.0.0.1 - -num_workers: 0 diff --git a/config.frontend.test.yaml b/config.frontend.test.yaml new file mode 100644 index 000000000..7428a3c05 --- /dev/null +++ b/config.frontend.test.yaml @@ -0,0 +1,202 @@ +default_project: &default_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: test/test_module.yaml + - path: projects/v1/default.wmf.yaml + options: &default_options + parsoid: + host: https://parsoid-beta.wmflabs.org + grace_ttl: 1000000 + action: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + graphoid: + host: https://graphoid-beta.wmflabs.org + mathoid: &mathoid_options + host: https://mathoid-beta.wmflabs.org + # 10 days Varnish caching, one day client-side + cache-control: s-maxage=864000, max-age=86400 + mobileapps: + host: https://appservice.wmflabs.org + summary: + protocol: https + implementation: mcs + host: https://appservice.wmflabs.org + citoid: + host: https://citoid-beta.wmflabs.org + recommendation: + host: https://recommendation-api-beta.wmflabs.org + purged_cache_control: test_purged_cache_control + # Cache control for purged endpoints allowing short-term client caching + purged_cache_control_client_cache: test_purged_cache_control_with_client_caching + pdf: + # Cache PDF for 5 minutes since it's not purged + cache_control: s-maxage=600, max-age=600 + uri: https://proton-beta.wmflabs.org + transform: + cx_host: https://cxserver-beta.wmflabs.org + skip_updates: false + /{api:sys}: &default_sys + x-modules: + - path: projects/sys/default.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"http://localhost:7231/{domain}/v1"}}' + block_external_reqs: false + +wikimedia_project: &wikimedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikimedia.wmf.yaml + options: + <<: *default_options + pageviews: + host: https://wikimedia.org/api/rest_v1/metrics + /{api:sys}: + x-modules: + - path: projects/proxy.yaml + options: + backend_host_template: '{{"http://localhost:7231/{domain}/v1"}}' + block_external_reqs: false + - spec: + paths: + /mathoid: + x-modules: + - path: sys/mathoid.js + options: *mathoid_options + /post_data: + x-modules: + - path: sys/post_data.js + /events: + x-modules: + - path: sys/events.js + +wikipedia_project: &wikipedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikipedia.wmf.yaml + options: *default_options + - path: test/test_module.yaml + options: + events: + uri: http://127.0.0.1:8085/v1/events + topic: resource_change + transcludes_topic: change-prop.transcludes.resource-change + /{api:sys}: *default_sys + +enwiki_project: &enwiki_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikipedia.wmf.yaml + options: *default_options + /{api:sys}: *default_sys + +wiktionary_project: &wiktionary_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: test/test_module.yaml + - path: projects/v1/wiktionary.wmf.yaml + options: *default_options + /{api:sys}: *default_sys + +# Hacky way to parametrize RESTBase tests. TODO: Move to config? +test: + content_types: + html: '/^text\/html; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/HTML\/[\d.]+"$/' + data-parsoid: '/^application\/json; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/data-parsoid/[\d.]+"$/' + wikitext: '/^text\/plain; charset=utf-8; profile="https:\/\/www.mediawiki.org\/wiki\/Specs\/wikitext\/[\d.]+"$/' + + +# The root of the spec tree. Domains tend to share specs by referencing them +# using YAML references. +spec_root: &spec_root + title: "The RESTBase root" + # Some more general RESTBase info + x-request-filters: + - path: lib/security_response_header_filter.js + - path: lib/normalize_headers_filter.js + + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true + - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ + forward_headers: true + # Need to forward cookie to backend RESTBase + - pattern: /^https?:\/\/localhost:7231.+/ + forward_headers: true + - pattern: /^https?:\/\// + paths: + /{domain:test.wikipedia.org}: *wikipedia_project + # The order is important for tests. + # Redirect tests require en.wiki being not the first wiki in the list. + /{domain:en.wikipedia.org}: *wikipedia_project + /{domain:ru.wikipedia.org}: *wikipedia_project + /{domain:de.wikipedia.org}: *wikipedia_project + /{domain:test2.wikipedia.org}: *wikipedia_project + /{domain:commons.wikimedia.org}: *default_project + + # labs, used for most tests + /{domain:en.wikipedia.beta.wmflabs.org}: *wikipedia_project + # Serbian wiki in beta for language variants tests + /{domain:sr.wikipedia.beta.wmflabs.org}: *wikipedia_project + + # For security testing we rely on mocks, so it's OK to use French wiki. + /{domain:fr.wikipedia.org}: + <<: *wikipedia_project + x-route-filters: + - path: ./lib/mediawiki_auth_filter.js + options: + permissions: + - read + # global domain + /{domain:wikimedia.org}: *wikimedia_project + + # Wiktionary has some specific endpoints + /{domain:en.wiktionary.org}: *wiktionary_project + + + +# Finally, a standard service-runner config. +info: + name: restrouter + +services: + - name: restrouter + module: hyperswitch + conf: + port: 7233 + spec: *spec_root + salt: secret + default_page_size: 1 + user_agent: RESTRouer-testsuite + ui_name: RESTBase + ui_url: https://www.mediawiki.org/wiki/RESTBase + ui_title: RESTBase docs + +logging: + name: restrouter + level: warn + streams: + - type: stdout + +num_workers: 0 diff --git a/config.fullstack.test.yaml b/config.fullstack.test.yaml new file mode 100644 index 000000000..1f28ab1d5 --- /dev/null +++ b/config.fullstack.test.yaml @@ -0,0 +1,248 @@ +# RESTBase test config, used in integration tests. + +# Load some project templates. These are referenced / shared between domains +# in the root_spec further down. +default_project: &default_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/default.wmf.yaml + options: &default_options + parsoid: + host: https://parsoid-beta.wmflabs.org + grace_ttl: 1000000 + action: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + graphoid: + host: https://graphoid-beta.wmflabs.org + mathoid: &mathoid_options + host: https://mathoid-beta.wmflabs.org + # 10 days Varnish caching, one day client-side + cache-control: s-maxage=864000, max-age=86400 + mobileapps: + host: https://appservice.wmflabs.org + summary: + protocol: https + implementation: mcs + host: https://appservice.wmflabs.org + citoid: + host: https://citoid-beta.wmflabs.org + recommendation: + host: https://recommendation-api-beta.wmflabs.org + purged_cache_control: test_purged_cache_control + # Cache control for purged endpoints allowing short-term client caching + purged_cache_control_client_cache: test_purged_cache_control_with_client_caching + pdf: + # Cache PDF for 5 minutes since it's not purged + cache_control: s-maxage=600, max-age=600 + uri: https://proton-beta.wmflabs.org + transform: + cx_host: https://cxserver-beta.wmflabs.org + skip_updates: false + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: &default_sys + x-modules: + - path: projects/sys/default.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + - spec: + paths: + /legacy/key_value: + x-modules: + - path: sys/key_value.js + /legacy/page_revisions: + x-modules: + - path: sys/page_revisions.js + /table: &table_spec + x-modules: + - path: sys/table.js + options: + conf: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: test.group.local + domains: /./ + dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing + +wikimedia_project: &wikimedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikimedia.wmf.yaml + options: + <<: *default_options + pageviews: + host: https://wikimedia.org/api/rest_v1/metrics + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: + x-modules: + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + - spec: + paths: + /legacy/key_value: + x-modules: + - path: sys/key_value.js + /table: *table_spec + /mathoid: + x-modules: + - path: sys/mathoid.js + options: *mathoid_options + /post_data: + x-modules: + - path: sys/post_data.js + /events: + x-modules: + - path: sys/events.js + +wikipedia_project: &wikipedia_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikipedia.wmf.yaml + options: *default_options + - path: test/test_module.yaml + options: + events: + uri: http://127.0.0.1:8085/v1/events + topic: resource_change + transcludes_topic: change-prop.transcludes.resource-change + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: *default_sys + +enwiki_project: &enwiki_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wikipedia.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: *default_sys + +wiktionary_project: &wiktionary_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - path: projects/v1/wiktionary.wmf.yaml + options: *default_options + - path: projects/proxy.yaml + options: + backend_host_template: '{{"/{domain}/sys/legacy"}}' + block_external_reqs: true + /{api:sys}: *default_sys + +# Hacky way to parametrize RESTBase tests. TODO: Move to config? +test: + content_types: + html: '/^text\/html; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/HTML\/[\d.]+"$/' + data-parsoid: '/^application\/json; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/data-parsoid/[\d.]+"$/' + wikitext: '/^text\/plain; charset=utf-8; profile="https:\/\/www.mediawiki.org\/wiki\/Specs\/wikitext\/[\d.]+"$/' + + +# The root of the spec tree. Domains tend to share specs by referencing them +# using YAML references. +spec_root: &spec_root + title: "The RESTBase root" + # Some more general RESTBase info + x-request-filters: + - path: lib/security_response_header_filter.js + + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true + - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ + forward_headers: true + # Need to forward cookie to backend RESTBase + - pattern: /^https?:\/\/localhost:7232.+/ + forward_headers: true + - pattern: /^https?:\/\// + paths: + /{domain:test.wikipedia.org}: *wikipedia_project + # The order is important for tests. + # Redirect tests require en.wiki being not the first wiki in the list. + /{domain:en.wikipedia.org}: *enwiki_project + /{domain:ru.wikipedia.org}: *wikipedia_project + /{domain:de.wikipedia.org}: *wikipedia_project + /{domain:test2.wikipedia.org}: *wikipedia_project + /{domain:commons.wikimedia.org}: *default_project + + # labs, used for most tests + /{domain:en.wikipedia.beta.wmflabs.org}: *wikipedia_project + # Serbian wiki in beta for language variants tests + /{domain:sr.wikipedia.beta.wmflabs.org}: *wikipedia_project + + # For security testing we rely on mocks, so it's OK to use French wiki. + /{domain:fr.wikipedia.org}: + <<: *wikipedia_project + x-route-filters: + - path: ./lib/mediawiki_auth_filter.js + options: + permissions: + - read + # global domain + /{domain:wikimedia.org}: *wikimedia_project + + # Wiktionary has some specific endpoints + /{domain:en.wiktionary.org}: *wiktionary_project + + + +# Finally, a standard service-runner config. +info: + name: restbase + +services: + - name: restbase + module: hyperswitch + conf: + port: 7231 + spec: *spec_root + salt: secret + default_page_size: 1 + user_agent: RESTBase-testsuite + ui_name: RESTBase + ui_url: https://www.mediawiki.org/wiki/RESTBase + ui_title: RESTBase docs + +logging: + name: restbase-test + level: warn + streams: + - type: stdout + +num_workers: 0 diff --git a/config.storage.test.yaml b/config.storage.test.yaml new file mode 100644 index 000000000..edf55aca6 --- /dev/null +++ b/config.storage.test.yaml @@ -0,0 +1,89 @@ +# RESTBase wikimedia example config +default_project: &default_project + x-modules: + - spec: + paths: + /{api:v1}: + x-modules: + - spec: + paths: + /key_value: + x-modules: + - path: sys/key_value.js + /page_revisions: + # We need to forward cookie to the API + x-route-filters: + - path: ./lib/mediawiki_auth_filter.js + x-modules: + - path: sys/page_revisions.js + /{api:sys}: + x-modules: + - spec: + paths: + /table: + x-modules: + - path: sys/table.js + options: + conf: + backend: '{env(RB_TEST_BACKEND, sqlite)}' + hosts: [localhost] + keyspace: system + username: cassandra + password: cassandra + defaultConsistency: one # or 'localQuorum' for production + storage_groups: + - name: test.group.local + domains: /./ + dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing + /action: + x-modules: + - path: sys/action.js + options: + apiUriTemplate: "{{'https://{domain}/w/api.php'}}" + baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" + + +# The root of the spec tree. Domains tend to share specs by referencing them +# using YAML references. +spec_root: &spec_root + title: "The RESTBase Storage Service" + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ + forward_headers: true + paths: + /{domain}: *default_project + +# Finally, a standard service-runner config. +info: + name: restbase + +services: + - name: restbase + module: hyperswitch + conf: + port: 7231 + spec: *spec_root + salt: secret + default_page_size: 125 + user_agent: RESTBase + ui_name: RESTBase + ui_url: https://www.mediawiki.org/wiki/RESTBase + ui_title: RESTBase docs + +logging: + name: restbase + level: warn + streams: + - type: stdout + +#metrics: +# type: statsd +# host: localhost +# port: 8125 +# batch: true + +num_workers: 0 diff --git a/config.test.yaml b/config.test.yaml deleted file mode 100644 index 815af4341..000000000 --- a/config.test.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# RESTBase test config, used in integration tests. - -# Load some project templates. These are referenced / shared between domains -# in the root_spec further down. -default_project: &default_project - x-modules: - - path: test/test_module.yaml - - path: projects/wmf_default.yaml - options: &default_options - table: - backend: '{env(RB_TEST_BACKEND, sqlite)}' - hosts: [localhost] - keyspace: system - username: cassandra - password: cassandra - defaultConsistency: one # or 'localQuorum' for production - storage_groups: - - name: test.group.local - domains: /./ - dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite testing - parsoid: - host: https://parsoid-beta.wmflabs.org - grace_ttl: 1000000 - action: - apiUriTemplate: "{{'https://{domain}/w/api.php'}}" - baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}" - graphoid: - host: https://graphoid-beta.wmflabs.org - mathoid: - host: https://mathoid-beta.wmflabs.org - # 10 days Varnish caching, one day client-side - cache-control: s-maxage=864000, max-age=86400 - mobileapps: - host: https://appservice.wmflabs.org - summary: - protocol: https - implementation: mcs - host: https://appservice.wmflabs.org - citoid: - host: https://citoid-beta.wmflabs.org - recommendation: - host: https://recommendation-api-beta.wmflabs.org - events: {} - purged_cache_control: test_purged_cache_control - # Cache control for purged endpoints allowing short-term client caching - purged_cache_control_client_cache: test_purged_cache_control_with_client_caching - pdf: - # Cache PDF for 5 minutes since it's not purged - cache_control: s-maxage=600, max-age=600 - uri: https://proton-beta.wmflabs.org - transform: - cx_host: https://cxserver-beta.wmflabs.org - skip_updates: false - -# A separate project for en.wikipedia because it is more feature-rich -en.wikipedia.org: &en.wikipedia.org - x-modules: - - path: test/test_module.yaml - options: - events: - uri: http://127.0.0.1:8085/v1/events - topic: resource_change - transcludes_topic: change-prop.transcludes.resource-change - - path: projects/wmf_enwiki.yaml - options: *default_options - -labs_project: &labs_project - x-modules: - - path: test/test_module.yaml - - path: projects/wmf_enwiki.yaml - options: *default_options - -# A different project template, sharing configuration options. -wikimedia.org: &wikimedia.org - x-modules: - - path: test/test_module.yaml - - path: projects/wikimedia.org.yaml - options: - <<: *default_options - pageviews: - host: https://wikimedia.org/api/rest_v1/metrics - -wiktionary_project: &wiktionary_project - x-modules: - - path: projects/wmf_wiktionary.yaml - options: *default_options - -wikipedia_project: &wikipedia_project - x-modules: - - path: test/test_module.yaml - - path: projects/wmf_wikipedia.yaml - options: *default_options - -# Hacky way to parametrize RESTBase tests. TODO: Move to config? -test: - content_types: - html: '/^text\/html; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/HTML\/[\d.]+"$/' - data-parsoid: '/^application\/json; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/data-parsoid/[\d.]+"$/' - wikitext: '/^text\/plain; charset=utf-8; profile="https:\/\/www.mediawiki.org\/wiki\/Specs\/wikitext\/[\d.]+"$/' - - -# The root of the spec tree. Domains tend to share specs by referencing them -# using YAML references. -spec_root: &spec_root - title: "The RESTBase root" - # Some more general RESTBase info - x-request-filters: - - path: lib/security_response_header_filter.js - - path: lib/normalize_headers_filter.js - - x-sub-request-filters: - - type: default - name: http - options: - allow: - - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ - forward_headers: true - - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/ - forward_headers: true - - pattern: /^https?:\/\// - paths: - /{domain:test.wikipedia.org}: *wikipedia_project - # The order is important for tests. - # Redirect tests require en.wiki being not the first wiki in the list. - /{domain:en.wikipedia.org}: *en.wikipedia.org - /{domain:ru.wikipedia.org}: *wikipedia_project - /{domain:de.wikipedia.org}: *wikipedia_project - /{domain:test2.wikipedia.org}: *wikipedia_project - /{domain:commons.wikimedia.org}: *default_project - - # labs, used for most tests - /{domain:en.wikipedia.beta.wmflabs.org}: *labs_project - # Serbian wiki in beta for language variants tests - /{domain:sr.wikipedia.beta.wmflabs.org}: *labs_project - - # For security testing we rely on mocks, so it's OK to use French wiki. - /{domain:fr.wikipedia.org}: - <<: *wikipedia_project - x-route-filters: - - path: ./lib/mediawiki_auth_filter.js - options: - permissions: - - read - # global domain - /{domain:wikimedia.org}: *wikimedia.org - - # Wiktionary has some specific endpoints - /{domain:en.wiktionary.org}: *wiktionary_project - -# Finally, a standard service-runner config. -info: - name: restbase - -services: - - name: restbase - module: hyperswitch - conf: - port: 7231 - spec: *spec_root - salt: secret - default_page_size: 1 - user_agent: RESTBase-testsuite - ui_name: RESTBase - ui_url: https://www.mediawiki.org/wiki/RESTBase - ui_title: RESTBase docs - -logging: - name: restbase-test - level: error - streams: - - type: stdout - -num_workers: 0 diff --git a/projects/proxy.yaml b/projects/proxy.yaml new file mode 100644 index 000000000..b88d93f85 --- /dev/null +++ b/projects/proxy.yaml @@ -0,0 +1,9 @@ +paths: + /key_value: + x-modules: + - path: sys/backend_proxy.js + options: '{{options}}' + /page_revisions: + x-modules: + - path: sys/backend_proxy.js + options: '{{options}}' diff --git a/projects/sys/default.wmf.yaml b/projects/sys/default.wmf.yaml new file mode 100644 index 000000000..ae6fa89d3 --- /dev/null +++ b/projects/sys/default.wmf.yaml @@ -0,0 +1,127 @@ +paths: + /action: + x-modules: + - path: sys/action.js + options: "{{options.action}}" + /post_data: + x-modules: + - path: sys/post_data.js + /page_save: + x-modules: + - path: sys/page_save.js + /parsoid: + x-modules: + - path: sys/parsoid.js + options: + parsoidHost: '{{options.parsoid.host}}' + response_cache_control: '{{options.purged_cache_control}}' + grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}' + # A list of pages that we don't currently want to re-render on + # each edit. Most of these are huge bot-edited pages, which are + # rarely viewed in any case. + rerenderBlacklist: + # en wiki + en.wikipedia.org: + - 'User:B-bot/Event_log' + - 'User:DeltaQuad/UAA/Wait' + - 'User:JamesR/AdminStats' + - 'User:Kudpung/Dashboard' + # Various dashboards + - 'User:Breawycker/Wikipedia' + - 'User:Sonia/dashboard' + - 'User:Ocaasi/dashboard' + - 'User:Nolelover' + - 'User:Calmer_Waters' + - '/User%3ARedwolf24\//' + - 'User:Technical_13/dashboard' + - 'Template:Cratstats' + # Cyberbot is creating 90% of null edits + - '/^User:Cyberbot_I\//' + - '/^User:Cyberbot_II\//' + - '/^User:Cyberpower678\//' + - '/^User:Darts170Darts170\//' + - 'صارف:Cyberbot_I/Run/Adminstats' + - 'Defnyddiwr:Cyberbot_I/Run/Adminstats' + - 'User:Pentjuuu!.!/sandbox' + - 'User:AllyD/CSDlog' + - 'User:Peter_I._Vardy/sandbox-13' + - 'User:I_dream_of_horses/CSD_log' + - 'User:MJ180MJ180/sandbox' + - 'Talk:United_States_presidential_election,_2016' + - 'Wikipedia:Reference_desk/Humanities' + - 'Wikipedia:WikiProject_Deletion_sorting/People' + - 'Wikipedia:WikiProject_Deletion_sorting/United_States_of_America' + - 'Wikipedia:Articles_for_creation/Redirects' + - 'Wikipedia:Administrators%27_noticeboard/Incidents' + # Wikipedia + ca.wikipedia.org: + - 'Usuari:TronaBot/log:Activitat_reversors_per_hores' + ceb.wikipedia.org: + - 'Gumagamit:Lsjbot/Anomalier-PRIVAT' + - 'Gumagamit:Lsjbot/Kartrutor2' + de.wikipedia.org: + - '/The_Big_Bang_Theory\/Staffel/' + - 'Wikipedia:Café' + - 'Wikipedia:Defekte_Weblinks/Bot2015-Problem' + - 'Wikipedia_Diskussion:Hauptseite/Schon_gewusst' + - 'Benutzer:Anglo-Araneophilus/Almigdad_Mojalli' + - 'Benutzer:Wartungsstube/Berlin' + - 'Benutzer:Wartungsstube/Musik' + - 'Benutzer:Wartungsstube/Unternehmen' + - 'Benutzer:Wartungsstube/Schifffahrt' + - 'Benutzer:Verum/ege' + - 'Benutzer:Septembermorgen/Bottabelle/Französische_Kantone_N–Z' + - 'Wikipedia:WikiProjekt_Planen_und_Bauen/Zu_überarbeitende_Artikel' + es.wikipedia.org: + - 'Wikipedia:Café/Archivo/Miscelánea/Actual' + fr.wikipedia.org: + - 'Utilisateur:ZéroBot/Log/Erreurs' + - 'Utilisateur:SyntaxTerror/Ajouts_du_modèle_Autorité' + - '/^Utilisateur:[\s\S]+[Bb]rouillon' + - 'Discussion_utilisateur:NaggoBot/CommonsDR' + - 'Projet:France/Annonces/Admissibilité' + - '/Wikipédia:Le_saviez-vous_.+/Anecdotes_proposées/' + hy.wikipedia.org: + - "/Մասնակից:Omicroñ\\'R/" + it.wikipedia.org: + - 'Utente:Effems/Sandbox7' + nl.wikipedia.org: + - 'Gebruiker:Eg-T2g/Kladblok' + pt.wikipedia.org: + - 'Wikipédia:Pedidos/Bloqueio' + ru.wikipedia.org: + - 'Википедия:Форум/Технический' + - 'Портал:Герпетология' + sv.wikipedia.org: + - 'Användare:Lsjbot/Anomalier-PRIVAT' + - 'Användare:Lsjbot/Namnkonflikter-PRIVAT' + ur.wikipedia.org: + - 'نام_مقامات_ایل' + - 'نام_مقامات_ڈی' + - 'نام_مقامات_جے' + - 'نام_مقامات_جی' + - 'نام_مقامات_ایچ' + - 'نام_مقامات_ایم' + - 'نام_مقامات_ایس' + zh.wikipedia.org: + - 'Wikipedia:互助客栈/条目探讨' + - 'Draft:日本人工湖列表' + # Wikisource + pl.wikisource.org: + - '/^Wśród_czarnych\//' + # Wikimedia + commons.wikimedia.org: + - '/Commons:Featured_picture_candidates\//' + - 'Commons:Quality_images/Subject/Places/Natural_structures' + - '/Commons:Undeletion_requests\//' + - '/Commons:WikiProject_Aviation\/recent_uploads\//' + - '/^(?:User|Benutzer):/' + /mobileapps: + x-modules: + - path: sys/mobileapps.js + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + /events: + x-modules: + - path: sys/events.js + options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}' diff --git a/projects/v1/default.wmf.yaml b/projects/v1/default.wmf.yaml new file mode 100644 index 000000000..336b44686 --- /dev/null +++ b/projects/v1/default.wmf.yaml @@ -0,0 +1,99 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' diff --git a/projects/v1/wikidata.wmf.yaml b/projects/v1/wikidata.wmf.yaml new file mode 100644 index 000000000..73f7f3e0d --- /dev/null +++ b/projects/v1/wikidata.wmf.yaml @@ -0,0 +1,82 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, options.mobileapps)}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/lists.js + options: '{{options.lists}}' + diff --git a/projects/v1/wikimedia.wmf.yaml b/projects/v1/wikimedia.wmf.yaml new file mode 100644 index 000000000..034dff621 --- /dev/null +++ b/projects/v1/wikimedia.wmf.yaml @@ -0,0 +1,75 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Software available under the Apache 2 license + url: http://www.apache.org/licenses/LICENSE-2.0 + +securityDefinitions: &wp/content-security/1.0.0 + mediawiki_auth: + description: Checks permissions using MW api + type: apiKey + in: header + name: cookie + x-internal-request-whitelist: + - /http:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/ +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +paths: + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /metrics: + x-modules: + - path: v1/metrics.yaml + options: '{{options.pageviews}}' + /transform: + x-modules: + - path: v1/transform-global.yaml + options: '{{options.transform}}' + /feed: + x-modules: + - path: v1/availability.yaml + options: '{{options.mobileapps}}' diff --git a/projects/v1/wikipedia.wmf.yaml b/projects/v1/wikipedia.wmf.yaml new file mode 100644 index 000000000..cdcaaa408 --- /dev/null +++ b/projects/v1/wikipedia.wmf.yaml @@ -0,0 +1,127 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /feed: + x-modules: + - path: v1/feed.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + - path: v1/announcements.yaml + options: '{{merge({"announcement_cache_control": "s-maxage=86400, max-age=86400"}, options.mobileapps)}}' + - path: v1/onthisday.js + options: '{{merge({"feed_cache_control": "s-maxage=300, max-age=60"}, options.mobileapps)}}' + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + diff --git a/projects/v1/wikivoyage.wmf.yaml b/projects/v1/wikivoyage.wmf.yaml new file mode 100644 index 000000000..4a12d2efa --- /dev/null +++ b/projects/v1/wikivoyage.wmf.yaml @@ -0,0 +1,122 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + # NOTE: We're setting the `no-store` cache-control header so that we don't + # store MCS results - we don't have pre-generation for wikivoyage. + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps, + {"cache-control": "no-store"})}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/summary.js + options: '{{merge({"response_cache_control": options.purged_cache_control_client_cache}, + options.summary)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/related.js + options: '{{options.related}}' + - path: v1/random.yaml + options: '{{merge({"random_cache_control": "s-maxage=2, max-age=1"}, + options.mobileapps)}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/recommend.yaml + options: '{{options.recommendation}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' + diff --git a/projects/v1/wiktionary.wmf.yaml b/projects/v1/wiktionary.wmf.yaml new file mode 100644 index 000000000..be3d01fae --- /dev/null +++ b/projects/v1/wiktionary.wmf.yaml @@ -0,0 +1,114 @@ +openapi: 3.0.1 +info: + version: 1.0.0 + title: Wikimedia REST API + description: > + This API provides cacheable and straightforward access to + Wikimedia content and data, in machine-readable formats. + + ### Global Rules + + - Limit your clients to no more than 200 requests/s to this API. + Each API endpoint's documentation may detail more specific usage limits. + - Set a unique `User-Agent` or `Api-User-Agent` header that + allows us to contact you quickly. Email addresses or URLs + of contact pages work well. + + By using this API, you agree to Wikimedia's + [Terms of Use](https://wikimediafoundation.org/wiki/Terms_of_Use) and + [Privacy Policy](https://wikimediafoundation.org/wiki/Privacy_policy). + Unless otherwise specified in the endpoint documentation + below, content accessed via this API is licensed under the + [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) + and [GFDL](https://www.gnu.org/copyleft/fdl.html) licenses, + and you irrevocably agree to release modifications or + additions made through this API under these licenses. + See https://www.mediawiki.org/wiki/REST_API for background and details. + + ### Endpoint documentation + + Please consult each endpoint's documentation for details on: + - Licensing information for the specific type of content + and data served via the endpoint. + - Stability markers to inform you about development status and + change policy, according to + [our API version policy](https://www.mediawiki.org/wiki/API_versioning). + - Endpoint specific usage limits. + + termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use + contact: + name: the Wikimedia Services team + url: http://mediawiki.org/wiki/REST_API + license: + name: Apache2 + url: http://www.apache.org/licenses/LICENSE-2.0 +# Override the base path for host-based (proxied) requests. In our case, +# we proxy https://{domain}/api/rest_v1/ to the API. +x-host-basePath: /api/rest_v1 +x-route-filters: + - path: lib/content_location_filter.js + - path: ./lib/normalize_title_filter.js + options: + redirect_cache_control: '{{options.purged_cache_control}}' +paths: + /: + x-modules: + - path: v1/common_schemas.yaml + /page: + x-modules: + - path: v1/content.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + - path: v1/content_segments.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + cx_host: '{{options.transform.cx_host}}' + - path: v1/mobileapps.yaml + options: '{{merge({"response_cache_control": options.purged_cache_control}, + options.mobileapps)}}' + - path: v1/media.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/metadata.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/references.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/mobile-html.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/graphoid.yaml + options: '{{options.graphoid}}' + - path: v1/definition.yaml + options: + response_cache_control: '{{options.purged_cache_control}}' + host: '{{options.mobileapps.host}}' + - path: v1/pdf.js + options: '{{options.pdf}}' + - path: v1/common_schemas.yaml # Doesn't really matter where to mount it. + /transform: + x-modules: + - path: v1/transform.yaml + - path: v1/transform-lang.js + options: '{{options.transform}}' + /media: + x-modules: + - path: v1/mathoid.yaml + options: '{{options.mathoid}}' + /data: + x-modules: + - path: v1/citoid.js + options: '{{options.citoid}}' + - path: v1/lists.js + options: '{{options.lists}}' + - path: v1/css.yaml + options: + host: '{{options.mobileapps.host}}' + - path: v1/javascript.yaml + options: + host: '{{options.mobileapps.host}}' diff --git a/sys/backend_proxy.js b/sys/backend_proxy.js new file mode 100644 index 000000000..5f0e69b17 --- /dev/null +++ b/sys/backend_proxy.js @@ -0,0 +1,67 @@ +'use strict'; + +const HyperSwitch = require('hyperswitch'); +const Template = HyperSwitch.Template; + +module.exports = (options) => { + options = options || {}; + options.backend_host_template = options.backend_host_template || '/{domain}/sys'; + const backendURITemplate = new Template({ + uri: `${options.backend_host_template}/{{path}}` + }); + return { + spec: { + paths: { + '/{+path}': { + 'x-route-filters': options.block_external_reqs ? + [{ + type: 'default', + name: 'header_match', + options: { + whitelist: { + 'x-client-ip': ['/^(?:::ffff:)?(?:10|127)\\./'] + } + } + }] : [], + all: { + operationId: 'proxy', + 'x-monitor': false, + 'x-hidden': true + } + } + } + }, + operations: { + proxy: (hyper, req) => { + const uri = req.uri.toString(); + const uriPrefix = uri.substring(0, uri.indexOf(`/${req.params.path}`)); + const segment = uriPrefix.split('/').pop(); + req.params.path = `${segment}/${req.params.path}`; + return hyper.request({ + method: req.method, + uri: backendURITemplate.expand({ request: req }).uri, + headers: req.headers, + body: req.body, + query: req.query + }) + .then((res) => { + // We are slowly switching to storing data as binary blobs + // in the storage component. When running multi-process, + // the storage component will return the appropriate + // content-type and deserializing the response will + // happen internally in preq. However, when running + // single process, preq is not involved, so for consistency + // we need to manually deserialize the returned blob. + if (res && + res.headers && + res.headers['content-type'] && + res.headers['content-type'].startsWith('application/json') && + Buffer.isBuffer(res.body)) { + res.body = JSON.parse(res.body.toString('utf8')); + } + return res; + }); + } + } + }; +}; diff --git a/sys/key_value.js b/sys/key_value.js index da489e2c9..64270e189 100644 --- a/sys/key_value.js +++ b/sys/key_value.js @@ -133,9 +133,7 @@ class KVBucket { if (req.headers['if-none-hash-match']) { delete req.headers['if-none-hash-match']; - return hyper.get({ - uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket, rp.key]) - }) + return this.getRevision(hyper, req) .then((oldContent) => { // TODO: proper etag-based compare. if (stringify(req.body) === stringify(oldContent.body) && diff --git a/sys/mathoid.js b/sys/mathoid.js index e8087baf7..329cf4b3f 100644 --- a/sys/mathoid.js +++ b/sys/mathoid.js @@ -82,7 +82,10 @@ class MathoidService { indirectionP = hyper.put({ uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid_ng.hash_table', origHash]), - headers: { 'x-store-content-type': 'text/plain' }, + headers: { + 'content-type': 'text/plain', + 'x-store-content-type': 'text/plain' + }, body: hash }); } @@ -95,7 +98,9 @@ class MathoidService { return P.join( hyper.put({ uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid_ng.check', hash]), - headers: prefixHeaders(checkRes.headers), + headers: Object.assign({ + 'content-type': 'application/json' + }, prefixHeaders(checkRes.headers)), body: checkRes.body }), indirectionP, @@ -131,7 +136,9 @@ class MathoidService { }); const reqObj = { uri: new URI([domain, 'sys', 'key_value', `mathoid_ng.${format}`, hash]), - headers: prefixHeaders(completeBody[format].headers), + headers: Object.assign({ + 'content-type': completeBody[format].headers['content-type'] + }, prefixHeaders(completeBody[format].headers)), body: completeBody[format].body }; if (format === 'png' && reqObj.body && reqObj.body.type === 'Buffer') { @@ -251,11 +258,19 @@ module.exports = (options) => { resources: [ { uri: '/{domain}/sys/post_data/mathoid_ng.input' - }, { + }, + { uri: '/{domain}/sys/key_value/mathoid_ng.hash_table', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'string' } - }, { + }, + { uri: '/{domain}/sys/key_value/mathoid_ng.check', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'json' } } ] diff --git a/sys/mobileapps.js b/sys/mobileapps.js index beccad7eb..76c0fad7d 100644 --- a/sys/mobileapps.js +++ b/sys/mobileapps.js @@ -23,7 +23,6 @@ class MobileApps { uri: new URI([rp.domain, 'sys', 'key_value', BUCKET_NAME, rp.title]) }) .then((res) => { - res.body = JSON.parse(res.body.toString('utf8')); if (!rp.revision || `${mwUtils.parseETag(res.headers.etag).rev}` === `${rp.revision}`) { return res; diff --git a/sys/parsoid.js b/sys/parsoid.js index dde8e86d6..2b45bbce6 100644 --- a/sys/parsoid.js +++ b/sys/parsoid.js @@ -204,7 +204,7 @@ class ParsoidService { uri: this.getStashBucketURI(domain, title, revision, tid) }) .then((res) => { - res = JSON.parse(res.body.toString('utf8')); + res = res.body; res.revid = revision; return res; }); @@ -328,11 +328,7 @@ class ParsoidService { */ _getContentWithFallback(hyper, domain, title, revision, tid) { if (!revision && !tid) { - return hyper.get({ uri: this.getLatestBucketURI(domain, title) }) - .then((res) => { - res.body = JSON.parse(res.body.toString('utf8')); - return res; - }); + return hyper.get({ uri: this.getLatestBucketURI(domain, title) }); } else if (!tid) { return hyper.get({ uri: this.getLatestBucketURI(domain, title) }) .then((res) => { @@ -340,17 +336,12 @@ class ParsoidService { if (revision !== resEtag.rev) { throw new HTTPError({ status: 404 }); } - res.body = JSON.parse(res.body.toString('utf8')); return res; }); } else { return hyper.get({ uri: this.getStashBucketURI(domain, title, revision, tid) }) - .then((res) => { - res.body = JSON.parse(res.body.toString('utf8')); - return res; - }) .catch({ status: 404 }, () => hyper.get({ uri: this.getLatestBucketURI(domain, title) }) .then((res) => { @@ -358,7 +349,6 @@ class ParsoidService { if (revision !== resEtag.rev || tid !== resEtag.tid) { throw new HTTPError({ status: 404 }); } - res.body = JSON.parse(res.body.toString('utf8')); return res; }) ); @@ -807,12 +797,18 @@ module.exports = (options) => { resources: [ { uri: '/{domain}/sys/key_value/parsoid', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'blob' } }, { uri: '/{domain}/sys/key_value/parsoid-stash', + headers: { + 'content-type': 'application/json' + }, body: { valueType: 'blob', default_time_to_live: options.grace_ttl diff --git a/sys/post_data.js b/sys/post_data.js index a671ed383..827a2f6d8 100644 --- a/sys/post_data.js +++ b/sys/post_data.js @@ -57,6 +57,9 @@ class PostDataBucket { const rp = req.params; return hyper.put({ uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket]), + headers: { + 'content-type': 'application/json' + }, body: { keyType: 'string', valueType: 'json' diff --git a/test/features/buckets/key_value_bucket.js b/test/features/buckets/key_value_bucket.js index cf4799365..77bb255bf 100644 --- a/test/features/buckets/key_value_bucket.js +++ b/test/features/buckets/key_value_bucket.js @@ -8,7 +8,6 @@ const P = require('bluebird'); const parallel = require('mocha.parallel'); describe('Key value buckets', () => { - function randomString(length) { let result = ''; for (let i = 0; i < length / 10; i++) { @@ -20,11 +19,23 @@ describe('Key value buckets', () => { function runTests(bucketName) { const server = new Server(); let bucketBaseURI; + let stringBaseURI; before(() => server.start() .then(() => { - bucketBaseURI = - `${server.config.baseURL()}/buckets/${bucketName}/${bucketName}TestingBucket`; - return preq.put({ uri: bucketBaseURI} ); + bucketBaseURI = `${server.config.backendURL()}/${bucketName}/${bucketName}TestingBucket`; + return preq.put({uri: bucketBaseURI}); + }) + .then(() => { + stringBaseURI = `${server.config.backendURL()}/${bucketName}/${bucketName}StringBucket`; + return preq.put({ + uri: stringBaseURI, + headers: { + 'content-type': 'application/json' + }, + body: { + valueType: 'string' + } + }); })); after(() => server.stop()); @@ -45,6 +56,28 @@ describe('Key value buckets', () => { assert.deepEqual(res.body, new Buffer(testData)); }); }); + it('Supports text/plain', () => { + const testData = randomString(100); + return preq.put({ + uri: `${stringBaseURI}/${testData}`, + headers: { + 'content-type': 'text/plain', + 'x-store-content-type': 'text/plain' + }, + body: testData + }) + .then((res) => { + assert.deepEqual(res.status, 201); + return preq.get({ + uri: `${stringBaseURI}/${testData}` + }); + }) + .then((res) => { + assert.deepEqual(res.status, 200); + assert.deepEqual(res.body, testData); + assert.deepEqual(res.headers['content-type'], 'text/plain'); + }); + }); it('throws 404 error if key not found', () => { return preq.get({ diff --git a/test/features/events/events.js b/test/features/events/events.js index da9258b0c..19671bd00 100644 --- a/test/features/events/events.js +++ b/test/features/events/events.js @@ -14,9 +14,9 @@ describe('Change event emitting', () => { it('should not explode if events config is not provided', () => { return preq.post({ - uri: `${server.config.baseURL()}/events_no_config/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events_no_config/`, body: [ - { uri: '//en.wikipedia.org' } + { uri: '//en.wikipedia.beta.wmflabs.org' } ] }); }); @@ -34,7 +34,7 @@ describe('Change event emitting', () => { const events = JSON.parse(postData.toString()); assert.deepEqual(events.length, 1); const event = events[0]; - assert.deepEqual(event.meta.domain, 'en.wikipedia.org'); + assert.deepEqual(event.meta.domain, 'en.wikipedia.beta.wmflabs.org'); assert.deepEqual(!!new Date(event.meta.dt), true); assert.deepEqual(uuid.test(event.meta.id), true); assert.deepEqual(!!event.meta.request_id, true); @@ -71,11 +71,11 @@ describe('Change event emitting', () => { eventLogging = createEventLogging(really_done, { topic: 'resource_change', - uri: 'http://en.wikipedia.org/wiki/User:Pchelolo' + uri: 'http://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }); preq.post({ - uri: `${server.config.baseURL()}/events/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events/`, headers: { 'content-type': 'application/json', connection: 'close', @@ -83,7 +83,7 @@ describe('Change event emitting', () => { body: [ { meta: { - uri: '//en.wikipedia.org/wiki/User:Pchelolo' + uri: '//en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }, tags: ['test'] }, @@ -110,21 +110,21 @@ describe('Change event emitting', () => { eventLogging = createEventLogging(really_done, { topic: 'change-prop.transcludes.resource-change', - uri: 'http://en.wikipedia.org/api/rest_v1/page/html/User:Pchelolo', - trigger: 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.org/wiki/User:Pchelolo' + uri: 'http://en.wikipedia.beta.wmflabs.org/api/rest_v1/page/html/User:Pchelolo', + trigger: 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }); preq.post({ - uri: `${server.config.baseURL()}/events/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events/`, headers: { 'content-type': 'application/json', connection: 'close', - 'x-triggered-by': 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.org/wiki/User:Pchelolo' + 'x-triggered-by': 'mediawiki.revision-create:https://en.wikimedia.org/wiki/Template:One,change-prop.transcludes.resource-change:https://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }, body: [ { meta: { - uri: '//en.wikipedia.org/api/rest_v1/page/html/User:Pchelolo' + uri: '//en.wikipedia.beta.wmflabs.org/api/rest_v1/page/html/User:Pchelolo' }, tags: ['test'] } @@ -149,26 +149,26 @@ describe('Change event emitting', () => { eventLogging = createEventLogging(really_done, { topic: 'resource_change', - uri: 'http://en.wikipedia.org/wiki/User:Pchelolo', - trigger: 'resource_change:https://en.wikipedia.org/wiki/Prohibited' + uri: 'http://en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo', + trigger: 'resource_change:https://en.wikipedia.beta.wmflabs.org/wiki/Prohibited' }); preq.post({ - uri: `${server.config.baseURL()}/events/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/events/`, headers: { 'content-type': 'application/json', - 'x-triggered-by': 'resource_change:https://en.wikipedia.org/wiki/Prohibited' + 'x-triggered-by': 'resource_change:https://en.wikipedia.beta.wmflabs.org/wiki/Prohibited' }, body: [ { meta: { - uri: '//en.wikipedia.org/wiki/Prohibited' + uri: '//en.wikipedia.beta.wmflabs.org/wiki/Prohibited' }, tags: ['test'] }, { meta: { - uri: '//en.wikipedia.org/wiki/User:Pchelolo' + uri: '//en.wikipedia.beta.wmflabs.org/wiki/User:Pchelolo' }, tags: ['test'] } diff --git a/test/features/pagecontent/access_checks.js b/test/features/pagecontent/access_checks.js index a3f6a00e2..64dbdbdaa 100644 --- a/test/features/pagecontent/access_checks.js +++ b/test/features/pagecontent/access_checks.js @@ -53,8 +53,18 @@ describe('Access checks', () => { return server.start() // Do a preparation request to force siteinfo fetch so that we don't need to mock it .then(() => P.join( - preq.get({ uri: `${server.config.bucketURL()}/html/Main_Page` }), - preq.get({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/Main_Page` }) + preq.get({ + uri: `${server.config.bucketURL()}/html/Main_Page`, + headers: { + 'cache-control': 'no-cache' + } + }), + preq.get({ + uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/Main_Page`, + headers: { + 'cache-control': 'no-cache' + } + }) )) // Load in the revisions .then(() => { diff --git a/test/features/pagecontent/save_api.js b/test/features/pagecontent/save_api.js index 8985eb32a..75b0785f3 100644 --- a/test/features/pagecontent/save_api.js +++ b/test/features/pagecontent/save_api.js @@ -376,7 +376,7 @@ describe('page save api', function() { } }); - it('save HTML', () => { + it('save HTML', () => { const test = () => { return preq.get({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/${pageTitle}/${lastRev}` @@ -385,7 +385,6 @@ describe('page save api', function() { return preq.post({ uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/${pageTitle}`, headers: { - 'x-client-ip': '123.123.123.123', cookie: 'test' }, body: { @@ -403,7 +402,6 @@ describe('page save api', function() { if (NOCK_TESTS) { const api = nock(server.config.apiURL('en.wikipedia.beta.wmflabs.org'), { reqheaders: { - 'x-client-ip': '123.123.123.123', 'x-forwarded-for'(headerValue) { return headerValue.indexOf('127.0.0.1') >= 0; }, diff --git a/test/features/post_data.js b/test/features/post_data.js index de84fd283..87d38ae4a 100644 --- a/test/features/post_data.js +++ b/test/features/post_data.js @@ -14,7 +14,7 @@ describe('post_data', function() { it('should store post request by hash', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/`, body: { key: 'value' } @@ -24,7 +24,7 @@ describe('post_data', function() { assert.deepEqual(res.status, 201); assert.deepEqual(hash, '228458095a9502070fc113d99504226a6ff90a9a'); return preq.get({ - uri: `${server.config.baseURL()}/post_data/${res.body}` + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/${res.body}` }); }) .then((res) => { @@ -35,7 +35,7 @@ describe('post_data', function() { it('should not explode on empty body', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/` + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/` }) .then((res) => { assert.deepEqual(res.status, 201); @@ -44,7 +44,7 @@ describe('post_data', function() { it('should not store identical request', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/`, body: { key: 'value' } @@ -57,7 +57,7 @@ describe('post_data', function() { it('should allow read on remote request', () => { return preq.get({ - uri: `${server.config.baseURL()}/post_data/${hash}` + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/${hash}` }) .then((res) => { assert.deepEqual(res.status, 200); @@ -67,7 +67,7 @@ describe('post_data', function() { it('should deny write on remote requests', () => { return preq.post({ - uri: `${server.config.baseURL()}/post_data/`, + uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/post_data/`, headers: { 'x-client-ip': '123.123.123.123' }, diff --git a/test/features/router/misc.js b/test/features/router/misc.js index b15eab902..e5c2fcfc6 100644 --- a/test/features/router/misc.js +++ b/test/features/router/misc.js @@ -18,7 +18,7 @@ describe('router - misc', function() { after(() => server.stop()); it('should deny access to /{domain}/sys', () => { - return preq.get({uri: `${server.config.hostPort}/en.wikipedia.org/sys/table`}) + return preq.get({uri: `${server.config.hostPort}/en.wikipedia.org/sys/action/query`}) .catch((err) => { assert.deepEqual(err.status, 403); }); diff --git a/test/features/schema_tests.js b/test/features/schema_tests.js index 02f4840d0..f1541071a 100644 --- a/test/features/schema_tests.js +++ b/test/features/schema_tests.js @@ -13,7 +13,7 @@ const validator = new OpenAPISchemaValidator({ version: 3 }); parallel('Responses should conform to the provided JSON schema of the response', () => { const ajv = new Ajv({}); - const server = new Server(`${__dirname}/../../config.example.wikimedia.yaml`); + const server = new Server(); function getToday() { function zeroPad(num) { if (num < 10) { diff --git a/test/features/security/security.js b/test/features/security/security.js index 942358f9d..86c74ce0a 100644 --- a/test/features/security/security.js +++ b/test/features/security/security.js @@ -59,6 +59,9 @@ describe('router - security', function() { .post('', (body) => { return body && body.generator === 'allpages'; }) .reply(200, sampleApiResponse) .post('', (body) => { return body && body.meta === 'userinfo'; }) + .reply(200, sampleRightsResponse) + .post('', (body) => { return body && body.meta === 'userinfo'; }) + .optionally() .reply(200, sampleRightsResponse); return preq.get({ diff --git a/test/test_module.yaml b/test/test_module.yaml index cf40952ae..7a2803cfa 100644 --- a/test/test_module.yaml +++ b/test/test_module.yaml @@ -1,95 +1,84 @@ -# Simple test spec - openapi: 3.0.1 +info: + version: 1.0.0-beta + title: Wikimedia testing APIs + x-is-api-root: true paths: - /{api:v1}: - x-modules: - - spec: - info: - version: 1.0.0-beta - title: Wikimedia testing APIs - x-is-api-root: true - paths: - /service/test_parallel/{key1}/{key2}: - get: - x-request-handler: - - get_from_api1: - request: - uri: http://en.wikipedia.org/wiki/{+key1} - get_from_api2: - request: - uri: http://en.wikipedia.org/wiki/{+key2} - - return_response: - return: - status: 200 - body: - first: '{{global.get_from_api1}}' - second: '{{global.get_from_api2}}' - x-monitor: true - x-amples: - - title: Retreives parallel content with simple service - request: - params: - domain: en.wikipedia.org - key1: User:GWicke/Date - key2: User:GWicke/Date - response: - status: 200 - body: - first: - status: 200 - headers: - content-type: /^text\/html.+/ - second: - status: 200 - headers: - content-type: /^text\/html.+/ - - /post_data/: - post: - x-route-filters: - - type: 'default' - name: 'header_match' - options: - whitelist: - x-client-ip: - - /^(?:::ffff:)?(?:10|127)\./ - x-request-handler: - - put_to_storage: - request: - method: put - uri: /{domain}/sys/post_data/post.test/ - headers: '{{request.headers}}' - body: '{{request.body}}' - x-monitor: false + /service/test_parallel/{key1}/{key2}: + get: + x-request-handler: + - get_from_api1: + request: + uri: http://en.wikipedia.org/wiki/{+key1} + get_from_api2: + request: + uri: http://en.wikipedia.org/wiki/{+key2} + - return_response: + return: + status: 200 + body: + first: '{{global.get_from_api1}}' + second: '{{global.get_from_api2}}' + x-monitor: true + x-amples: + - title: Retreives parallel content with simple service + request: + params: + domain: en.wikipedia.org + key1: User:GWicke/Date + key2: User:GWicke/Date + response: + status: 200 + body: + first: + status: 200 + headers: + content-type: /^text\/html.+/ + second: + status: 200 + headers: + content-type: /^text\/html.+/ - /post_data/{hash}: - get: - x-setup-handler: - - init_storage: - uri: /{domain}/sys/post_data/post.test - x-request-handler: - - get_from_storage: - request: - uri: /{domain}/sys/post_data/post.test/{hash} - x-monitor: false + /post_data/: + post: + x-route-filters: + - type: 'default' + name: 'header_match' + options: + whitelist: + x-client-ip: + - /^(?:::ffff:)?(?:10|127)\./ + x-request-handler: + - put_to_storage: + request: + method: put + uri: /{domain}/sys/post_data/post.test/ + headers: '{{request.headers}}' + body: '{{request.body}}' + x-monitor: false - /buckets/key_value: - x-modules: - - path: sys/key_value.js + /post_data/{hash}: + get: + x-setup-handler: + - init_storage: + uri: /{domain}/sys/post_data/post.test + x-request-handler: + - get_from_storage: + request: + uri: /{domain}/sys/post_data/post.test/{hash} + x-monitor: false - /events_no_config: - x-modules: - - path: sys/events.js - /events: - x-modules: - - path: sys/events.js - options: '{{options.events}}' + /events_no_config: + x-modules: + - path: sys/events.js + /events: + x-modules: + - path: sys/events.js + options: '{{options.events}}' - /http/{uri}: - get: - x-request-handler: - - request_url: - request: - uri: '{{uri}}' - options: '{{options}}' + /http/{uri}: + get: + x-request-handler: + - request_url: + request: + uri: '{{uri}}' diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index 899beeb0a..ffb87dbd5 100644 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -1,54 +1,40 @@ #!/bin/sh mod_dir=$( cd "$( dirname "$0" )"/../.. && pwd )/node_modules -mocha="$mod_dir"/mocha/bin/mocha -nyc="$mod_dir"/.bin/nyc +mocha="${mod_dir}"/mocha/bin/mocha +nyc="${mod_dir}"/.bin/nyc +test_target=${TEST_TARGET:-$2} +test_mode=${TEST_MODE:-$3} +export TEST_MODE=${test_mode} -runTest ( ) { - if [ "$1" = "cassandra" ]; then - echo "Running with Cassandra backend" - if [ `nc -z localhost 9042 < /dev/null; echo $?` != 0 ]; then - echo "Waiting for Cassandra to start..." - while [ `nc -z localhost 9042; echo $?` != 0 ]; do - sleep 1 - done - echo "Cassandra is ready." - fi - export RB_TEST_BACKEND=cassandra - sh ./test/utils/cleandb.sh local_group_test - else - echo "Running with SQLite backend" - export RB_TEST_BACKEND=sqlite - rm -f test.db.sqlite3 - fi - - if [ "$2" = "test" ]; then - "${mocha}" - elif [ "$2" = "coverage" ]; then - "${nyc}" --reporter=lcov node_modules/.bin/_mocha - fi -} +if [ "$1" = "test" ]; then + test_command="${mocha}" +elif [ "$1" = "coverage" ]; then + test_command="${nyc} --reporter=lcov node_modules/.bin/_mocha" +else + echo "Invalid test command. Must be 'test' or 'coverage" + exit 1 +fi -if [ "x$2" = "x" ]; then - # no concrete backend is provided, check for cassandra - `echo exit;` | cqlsh 2> /dev/null - if [ "$?" -eq 0 ]; then - runTest "cassandra" $1 - else - echo "Cassandra not available. Using SQLite backend for tests" - runTest "sqlite" $1 +if [ "x$test_target" = "x" ] || [ "$test_target" = "sqlite" ]; then + echo "Running with SQLite backend" + rm -f test.db.sqlite3 +elif [ "$test_target" = "cassandra" ]; then + echo "Running with Cassandra backend" + if [ `nc -z localhost 9042 < /dev/null; echo $?` != 0 ]; then + echo "Waiting for Cassandra to start..." + while [ `nc -z localhost 9042; echo $?` != 0 ]; do + sleep 1 + done + echo "Cassandra is ready." fi -elif [ "$2" = "sqlite" ]; then - runTest "sqlite" $1 -elif [ "$2" = "cassandra" ]; then - runTest "cassandra" $1 -elif [ "$2" = "all" ]; then - runTest "cassandra" $1 - cassandra_result=$? - runTest "sqlite" $1 - sqlite_result=$? - exit $(($cassandra_result + $sqlite_result)) + export RB_TEST_BACKEND=cassandra + sh ./test/utils/cleandb.sh local_group_test else - echo "Invalid testing mode" + echo "Invalid TEST_TARGET ${test_target}. Must me 'sqlite' or 'cassandra' if specified" exit 1 fi + +echo "Running ${test_mode} mode" +${test_command}; +exit $?; diff --git a/test/utils/server.js b/test/utils/server.js index 1b0055ab8..ce633051c 100644 --- a/test/utils/server.js +++ b/test/utils/server.js @@ -1,19 +1,40 @@ 'use strict'; +const P = require('bluebird'); const TestRunner = require('service-runner/test/TestServer'); const DEFAULT_DOMAIN = 'en.wikipedia.org'; -class TestRestbase extends TestRunner { - constructor(configPath = `${__dirname}/../../config.test.yaml`) { - super(configPath); +class TestRestbase { + constructor() { + const testMode = process.env.TEST_MODE || 'fs'; + switch (testMode) { + case 'fs': + this._frontendServer = new TestRunner(`${__dirname}/../../config.fullstack.test.yaml`); + this._backendServer = undefined; + break; + case 'ftfs': + this._frontendServer = new TestRunner(`${__dirname}/../../config.frontend.test.yaml`); + this._backendServer = new TestRunner(`${__dirname}/../../config.fullstack.test.yaml`); + break; + case 'ftbe': + this._frontendServer = new TestRunner(`${__dirname}/../../config.frontend.test.yaml`); + this._backendServer = new TestRunner(`${__dirname}/../../config.storage.test.yaml`); + break; + default: + throw new Error(`Invalid test mode ${testMode}`); + } } get config() { - if (!this._running) { + if (!this._frontendServer._running) { throw new Error('Accessing test service config before starting the service'); } - const hostPort = 'http://localhost:7231'; + const conf = this._frontendServer._runner._impl.config; + const backendConf = this._backendServer ? this._backendServer._runner._impl.config : conf; + const hostPort = `http://localhost:${conf.services[0].conf.port}`; + const backendHostPort = `http://localhost:${backendConf.services[0].conf.port}`; const baseURL = (domain = DEFAULT_DOMAIN) => `${hostPort}/${domain}/v1`; + const backendURL = (domain = DEFAULT_DOMAIN) => `${backendHostPort}/${domain}/v1`; const bucketURL = (domain) => `${baseURL(domain)}/page`; const apiPath = '/w/api.php'; const apiBase = (domain = DEFAULT_DOMAIN) => `https://${domain}`; @@ -22,14 +43,25 @@ class TestRestbase extends TestRunner { defaultDomain: DEFAULT_DOMAIN, hostPort, baseURL, + backendURL, bucketURL, apiBase, apiPath, apiURL, parsoidURI: 'https://parsoid-beta.wmflabs.org', - conf: this._runner._impl.config + conf } } + + start() { + const startPromise = this._backendServer ? this._backendServer.start() : P.resolve(); + return startPromise.then(() => this._frontendServer.start()); + } + + stop() { + const stopPromise = this._backendServer ? this._backendServer.stop() : P.resolve(); + return stopPromise.then(() => this._frontendServer.stop()); + } } module.exports = TestRestbase; diff --git a/v1/definition.yaml b/v1/definition.yaml index 3b28726cb..232f2a04a 100644 --- a/v1/definition.yaml +++ b/v1/definition.yaml @@ -91,6 +91,8 @@ paths: - init: method: put uri: /{domain}/sys/key_value/term.definition-ng + headers: + content-type: application/json body: valueType: json x-request-handler: @@ -124,7 +126,8 @@ paths: method: put uri: /{domain}/sys/key_value/term.definition-ng/{request.params.term} headers: - if-none-hash-match": '*' + content-type: 'application/json' + if-none-hash-match: '*' cache-control: '{{request.headers.cache-control}}' x-store-etag: '{{extract.headers.etag}}' x-store-content-language: '{{extract.headers.content-language}}' diff --git a/v1/mathoid.yaml b/v1/mathoid.yaml index 28622aa87..29a759fa7 100644 --- a/v1/mathoid.yaml +++ b/v1/mathoid.yaml @@ -191,16 +191,22 @@ paths: x-setup-handler: - init_svg: uri: /wikimedia.org/sys/key_value/mathoid_ng.svg + headers: + content-type: 'application/json' body: keyType: string valueType: string - init_mml: uri: /wikimedia.org/sys/key_value/mathoid_ng.mml + headers: + content-type: 'application/json' body: keyType: string valueType: string - init_png: uri: /wikimedia.org/sys/key_value/mathoid_ng.png + headers: + content-type: 'application/json' body: keyType: string valueType: blob @@ -208,7 +214,7 @@ paths: - check_storage: request: method: get - uri: /wikimedia.org/sys/key_value/mathoid_ng.{$.request.params.format}/{$.request.params.hash} + uri: /wikimedia.org/sys/key_value/mathoid_ng.{format}/{hash} headers: cache-control: '{{ cache-control }}' catch: diff --git a/v1/summary_new.yaml b/v1/summary_new.yaml index 6379ab879..fa17e3fb0 100644 --- a/v1/summary_new.yaml +++ b/v1/summary_new.yaml @@ -93,8 +93,10 @@ paths: x-setup-handler: # Set up a simple key-value bucket. - init: - method: put + method: 'put' uri: /{domain}/sys/key_value/page_summary + headers: + content-type: application/json body: valueType: json x-request-handler: @@ -135,7 +137,8 @@ paths: method: put uri: /{domain}/sys/key_value/page_summary/{request.params.title} headers: - if-none-hash-match": '*' + if-none-hash-match: '*' + content-type: 'application/json' cache-control: '{{request.headers.cache-control}}' x-store-etag: '{{extract.headers.etag}}' x-store-content-language: '{{extract.headers.content-language}}'