Merge 4142764 into d08ad0e

config.example.yaml

@@ -52,8 +52,10 @@ services:
                     # XXX Check the base RESTBase URI
                     baseUriTemplate: "{{'http://{domain}:7231/{domain}/v1'}}"
                   parsoid:
-                    # XXX Check Parsoid URL!
+                    # XXX Check Parsoid/JS URL!
                     host: http://localhost:8142
+                    # XXX Check Parsoid/PHP URL!
+                    php_host: http://localhost:8142
                   table:
                     backend: sqlite
                     dbname: db.sqlite3

config.frontend.test.yaml

@@ -9,6 +9,8 @@ default_project: &default_project
                 options: &default_options
                   parsoid:
                     host: https://parsoid-beta.wmflabs.org
+                    # TODO: replace with actual Parsoid/PHP host when available
+                    php_host: https://parsoid-beta.wmflabs.org
                     grace_ttl: 1000000
                   action:
                     apiUriTemplate: "{{'https://{domain}/w/api.php'}}"

config.fullstack.test.yaml

@@ -8,6 +8,8 @@ default_project: &default_project
                 options: &default_options
                   parsoid:
                     host: https://parsoid-beta.wmflabs.org
+                    # TODO: replace with actual Parsoid/PHP host when available
+                    php_host: https://parsoid-beta.wmflabs.org
                     grace_ttl: 1000000
                   action:
                     apiUriTemplate: "{{'https://{domain}/w/api.php'}}"

config.fullstack_no_php_host.test.yaml

@@ -0,0 +1,273 @@
+default_project: &default_project
+  x-modules:
+    - spec:
+        paths:
+          /{api:v1}:
+            x-modules:
+              - path: projects/v1/default.wmf.yaml
+                options: &default_options
+                  parsoid:
+                    host: https://parsoid-beta.wmflabs.org
+                    grace_ttl: 1000000
+                  action:
+                    apiUriTemplate: "{{'https://{domain}/w/api.php'}}"
+                    baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}"
+                  graphoid:
+                    host: https://graphoid-beta.wmflabs.org
+                  mathoid: &mathoid_options
+                    host: https://mathoid-beta.wmflabs.org
+                    # 10 days Varnish caching, one day client-side
+                    cache-control: s-maxage=864000, max-age=86400
+                  mobileapps:
+                    host: https://appservice.wmflabs.org
+                  summary:
+                    protocol: https
+                    implementation: mcs
+                    host: https://appservice.wmflabs.org
+                  citoid:
+                    host: https://citoid-beta.wmflabs.org
+                  recommendation:
+                    host: https://recommendation-api-beta.wmflabs.org
+                  purged_cache_control: test_purged_cache_control
+                  # Cache control for purged endpoints allowing short-term client caching
+                  purged_cache_control_client_cache: test_purged_cache_control_with_client_caching
+                  pdf:
+                    # Cache PDF for 5 minutes since it's not purged
+                    cache_control: s-maxage=600, max-age=600
+                    uri: https://proton-beta.wmflabs.org
+                  transform:
+                    cx_host: https://cxserver-beta.wmflabs.org
+                  skip_updates: false
+              - path: projects/proxy.yaml
+                options: &proxy_options
+                  backend_host_template: '{{"/{domain}/sys/legacy"}}'
+                  block_external_reqs: true
+          /{api:sys}: &default_sys
+            x-modules:
+              - path: projects/sys/default.wmf.yaml
+                options: *default_options
+              - path: projects/proxy.yaml
+                options:
+                  <<: *proxy_options
+                  block_external_reqs: false
+              - spec:
+                  paths:
+                    /legacy/key_value:
+                      x-modules:
+                        - path: sys/key_value.js
+                    /legacy/page_revisions:
+                      x-modules:
+                        - path: sys/page_revisions.js
+                    /table: &table_spec
+                      x-modules:
+                        - path: sys/table.js
+                          options:
+                            conf:
+                              backend: '{env(RB_TEST_BACKEND, sqlite)}'
+                              hosts: [localhost]
+                              keyspace: system
+                              username: cassandra
+                              password: cassandra
+                              defaultConsistency: one # or 'localQuorum' for production
+                              storage_groups:
+                                - name: test.group.local
+                                  domains: /./
+                              # ignored in cassandra, but useful in SQLite testing
+                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}'
+
+wikimedia_project: &wikimedia_project
+  x-modules:
+    - spec:
+        paths:
+          /{api:v1}:
+            x-modules:
+              - path: projects/v1/wikimedia.wmf.yaml
+                options:
+                  <<: *default_options
+                  pageviews:
+                    host: https://wikimedia.org/api/rest_v1/metrics
+              - path: projects/proxy.yaml
+                options: *proxy_options
+          /{api:sys}:
+            x-modules:
+              - path: projects/proxy.yaml
+                options:
+                  <<: *proxy_options
+                  block_external_reqs: false
+              - spec:
+                  paths:
+                    /legacy/key_value:
+                      x-modules:
+                        - path: sys/key_value.js
+                    /table: *table_spec
+                    /mathoid:
+                      x-modules:
+                        - path: sys/mathoid.js
+                          options: *mathoid_options
+                    /post_data:
+                      x-modules:
+                        - path: sys/post_data.js
+                    /events:
+                      x-modules:
+                        - path: sys/events.js
+                          # No options provided deliberately to avoid log noise upon testing
+
+wikipedia_project: &wikipedia_project
+  x-modules:
+    - spec:
+        paths:
+          /{api:v1}:
+            x-modules:
+              - path: projects/v1/wikipedia.wmf.yaml
+                options: *default_options
+              - path: projects/proxy.yaml
+                options: *proxy_options
+          /{api:sys}: *default_sys
+
+wiktionary_project: &wiktionary_project
+  x-modules:
+    - spec:
+        paths:
+          /{api:v1}:
+            x-modules:
+              - path: projects/v1/wiktionary.wmf.yaml
+                options: *default_options
+              - path: projects/proxy.yaml
+                options: *proxy_options
+          /{api:sys}: *default_sys
+
+commons_project: &commons_project
+  x-modules:
+    - spec:
+        paths:
+          /{api:v1}:
+            x-modules:
+              - path: projects/v1/default.wmf.yaml
+                options: *default_options
+              - spec:
+                  paths:
+                    /data:
+                      x-modules:
+                        - path: v1/recommend-caption.yaml
+                          options: '{{options.recommendation}}'
+                options: *default_options
+              - path: projects/proxy.yaml
+                options: *proxy_options
+          /{api:sys}: *default_sys
+
+wikidata_project: &wikidata_project
+  x-modules:
+    - spec:
+        paths:
+          /{api:v1}:
+            x-modules:
+              - path: projects/v1/wikidata.wmf.yaml
+                options: *default_options
+              - path: projects/proxy.yaml
+                options: *proxy_options
+          /{api:sys}: *default_sys
+
+testing_project: &testing_project
+  x-modules:
+    - spec:
+        paths:
+          /{api:v1}:
+            x-modules:
+              - path: test/test_module.yaml
+                options:
+                  events:
+                    uri: http://127.0.0.1:8085/v1/events
+                    stream: resource_change
+                    transcludes_stream: change-prop.transcludes.resource-change
+              - path: projects/proxy.yaml
+                options: *proxy_options
+          /{api:sys}: *default_sys
+
+# Hacky way to parametrize RESTBase tests. TODO: Move to config?
+test:
+  content_types:
+    html: '/^text\/html; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/HTML\/[\d.]+"$/'
+    data-parsoid: '/^application\/json; charset=utf-8; profile="https:\/\/www\.mediawiki\.org\/wiki\/Specs\/data-parsoid/[\d.]+"$/'
+    wikitext: '/^text\/plain; charset=utf-8; profile="https:\/\/www.mediawiki.org\/wiki\/Specs\/wikitext\/[\d.]+"$/'
+
+
+# The root of the spec tree. Domains tend to share specs by referencing them
+# using YAML references.
+spec_root: &spec_root
+  title: "The RESTBase root"
+  # Some more general RESTBase info
+  x-request-filters:
+    - path: lib/security_response_header_filter.js
+
+  x-sub-request-filters:
+    - type: default
+      name: http
+      options:
+        allow:
+          - pattern: /^https?:\/\/[a-zA-Z0-9\.]+\/w\/api\.php/
+            forward_headers: true
+          - pattern: /^https?:\/\/parsoid-beta.wmflabs.org.+/
+            forward_headers: true
+          - pattern: /^https?:\/\//
+  paths:
+    /{domain:test.wikipedia.org}: *wikipedia_project
+    # The order is important for tests.
+    # Redirect tests require en.wiki being not the first wiki in the list.
+    /{domain:en.wikipedia.org}: *wikipedia_project
+    /{domain:ru.wikipedia.org}: *wikipedia_project
+    /{domain:de.wikipedia.org}: *wikipedia_project
+    /{domain:test2.wikipedia.org}: *wikipedia_project
+    /{domain:commons.wikimedia.org}: *commons_project
+    /{domain:www.wikidata.org}: *wikidata_project
+
+    # labs, used for most tests
+    /{domain:en.wikipedia.beta.wmflabs.org}: *wikipedia_project
+    /{domain:commons.wikimedia.beta.wmflabs.org}: *commons_project
+    /{domain:wikidata.beta.wmflabs.org}: *wikidata_project
+
+    # Serbian wiki in beta for language variants tests
+    /{domain:sr.wikipedia.beta.wmflabs.org}: *wikipedia_project
+
+    # For security testing we rely on mocks, so it's OK to use French wiki.
+    /{domain:fr.wikipedia.org}:
+      <<: *wikipedia_project
+      x-route-filters:
+        - path: ./lib/mediawiki_auth_filter.js
+          options:
+            permissions:
+              - read
+
+    # global domain
+    /{domain:wikimedia.org}: *wikimedia_project
+
+    # Wiktionary has some specific endpoints
+    /{domain:en.wiktionary.org}: *wiktionary_project
+
+    # Fake domain to run tests relying on test_module
+    /{domain:fake.wikipedia.org}: *testing_project
+
+# Finally, a standard service-runner config.
+info:
+  name: restbase
+
+services:
+  - name: restbase
+    module: hyperswitch
+    conf: &rb_conf
+      # In production we expose both 7231 and 7233 ports from fullstack RB.
+      port: 7233
+      spec: *spec_root
+      salt: secret
+      default_page_size: 1
+      user_agent: RESTBase-testsuite
+      ui_name: RESTBase
+      ui_url: https://www.mediawiki.org/wiki/RESTBase
+      ui_title: RESTBase docs
+
+logging:
+  name: restbase-test
+  level: warn
+  streams:
+    - type: stdout
+
+num_workers: 0

projects/example.yaml

@@ -56,6 +56,7 @@ paths:
                 - path: sys/parsoid.js
                   options:
                     parsoidHost: '{{options.parsoid.host}}'
+                    parsoidPhpHost:  '{{options.parsoid.php_host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
         options: '{{options}}'

projects/sys/default.wmf.yaml

@@ -14,6 +14,7 @@ paths:
       - path: sys/parsoid.js
         options:
           parsoidHost: '{{options.parsoid.host}}'
+          parsoidPhpHost: '{{options.parsoid.php_host}}'
           response_cache_control: '{{options.purged_cache_control}}'
           grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
           # A list of pages that we don't currently want to re-render on