Merge 98ed298 into da7255c

wikimedia · Aug 8, 2019 · 2dcdd6a · 2dcdd6a
2 parents da7255c + 98ed298
commit 2dcdd6a
Show file tree

Hide file tree

Showing 7 changed files with 213 additions and 1 deletion.
diff --git a/projects/v1/wikipedia.wmf.yaml b/projects/v1/wikipedia.wmf.yaml
@@ -115,6 +115,9 @@ paths:
   /transform:
     x-modules:
       - path: v1/transform.yaml
+      - path: v1/pcs/transform.js
+        options:
+          mobileapps_host: '{{options.mobileapps.host}}'
       - path: v1/transform-lang.js
         options: '{{options.transform}}'
   /media:

diff --git a/projects/v1/wikivoyage.wmf.yaml b/projects/v1/wikivoyage.wmf.yaml
@@ -107,6 +107,9 @@ paths:
   /transform:
     x-modules:
       - path: v1/transform.yaml
+      - path: v1/pcs/transform.js
+        options:
+          mobileapps_host: '{{options.mobileapps.host}}'
       - path: v1/transform-lang.js
         options: '{{options.transform}}'
   /media:

diff --git a/projects/v1/wiktionary.wmf.yaml b/projects/v1/wiktionary.wmf.yaml
@@ -102,6 +102,9 @@ paths:
   /transform:
     x-modules:
       - path: v1/transform.yaml
+      - path: v1/pcs/transform.js
+        options:
+          mobileapps_host: '{{options.mobileapps.host}}'
       - path: v1/transform-lang.js
         options: '{{options.transform}}'
   /media:

diff --git a/sys/parsoid.js b/sys/parsoid.js
@@ -659,7 +659,8 @@ class ParsoidService {
                 'content-type': 'application/json',
                 'user-agent': req['user-agent'],
                 'content-language': req.headers['content-language'],
-                accept: req.headers.accept
+                accept: req.headers.accept,
+                'accept-language': req.headers['accept-language']
             },
             body: req.body
         };

diff --git a/test/features/pcs.js b/test/features/pcs.js
@@ -76,3 +76,72 @@ const preq   = require('preq');
     });
 });
 
+describe('Page Content Service: transforms', () => {
+    const server = new Server();
+    before(() => server.start());
+    after(() => server.stop());
+
+    it('should transform wikitext to mobile-html', () => {
+        return preq.post({
+            uri: `${server.config.baseURL()}/transform/wikitext/to/mobile-html/Main_Page`,
+            body: {
+                wikitext: '== Heading =='
+            }
+        })
+        .then((res) => {
+            assert.deepEqual(res.status, 200);
+            assert.deepEqual(res.headers['content-language'], 'en');
+            assert.checkString(res.headers['cache-control'], /private/, 'Must not be cached');
+            assert.checkString(res.body, /<h2 id="Heading" class="(:?[^"]+)">Heading<\/h2>/);
+        })
+    });
+
+    it('should transform wikitext to mobile-html, language variants, no variant', () => {
+        return preq.post({
+            uri: `${server.config.baseURL('sr.wikipedia.beta.wmflabs.org')}/transform/wikitext/to/mobile-html/RESTBase_Testing_Page`,
+            body: { wikitext: 'Ово је тестна страница - 1\n\nOvo je testna stranica - 2' }
+        })
+        .then((res) => {
+            assert.deepEqual(res.status, 200);
+            assert.deepEqual(res.headers['content-language'], 'sr');
+            assert.checkString(res.headers['cache-control'], /private/, 'Must not be cached');
+            assert.checkString(res.body, /Ово је тестна страница - 1/, 'Must not convert cyrillic with no variant');
+            assert.checkString(res.body, /Ovo je testna stranica - 2/, 'Must not convert latin with no variant');
+        });
+    });
+
+    it('should transform wikitext to mobile-html, language variants, cyrillic variant', () => {
+        return preq.post({
+            uri: `${server.config.baseURL('sr.wikipedia.beta.wmflabs.org')}/transform/wikitext/to/mobile-html/RESTBase_Testing_Page`,
+            headers: {
+                'accept-language': 'sr-ec'
+            },
+            body: { wikitext: 'Ово је тестна страница - 1\n\nOvo je testna stranica - 2' }
+        })
+        .then((res) => {
+            assert.deepEqual(res.status, 200);
+            assert.deepEqual(res.headers['content-language'], 'sr-ec');
+            assert.checkString(res.headers['cache-control'], /private/, 'Must not be cached');
+            assert.checkString(res.body, /Ово је тестна страница - 1/, 'Must not convert cyrillic with cyrillic variant');
+            assert.checkString(res.body, /Ово је тестна страница - 2/, 'Must convert latin with cyrillic variant');
+        });
+    });
+
+    it('should transform wikitext to mobile-html, language variants, latin variant', () => {
+        return preq.post({
+            uri: `${server.config.baseURL('sr.wikipedia.beta.wmflabs.org')}/transform/wikitext/to/mobile-html/RESTBase_Testing_Page`,
+            headers: {
+                'accept-language': 'sr-el'
+            },
+            body: { wikitext: 'Ово је тестна страница - 1\n\nOvo je testna stranica - 2' }
+        })
+        .then((res) => {
+            assert.deepEqual(res.status, 200);
+            assert.deepEqual(res.headers['content-language'], 'sr-el');
+            assert.checkString(res.headers['cache-control'], /private/, 'Must not be cached');
+            assert.checkString(res.body, /Ovo je testna stranica - 1/, 'Must convert cyrillic with latin variant');
+            assert.checkString(res.body, /Ovo je testna stranica - 2/, 'Must not convert latin with latin variant');
+        });
+    });
+});
+
diff --git a/v1/pcs/transform.js b/v1/pcs/transform.js
@@ -0,0 +1,52 @@
+'use strict';
+
+const HyperSwitch = require('hyperswitch');
+const URI = HyperSwitch.URI;
+
+class TransformService {
+    constructor(options) {
+        if (!options && !options.mobileapps_host) {
+            throw new Error('mobileapps_host not specified for PCS transform configuration');
+        }
+        this._options = options;
+    }
+
+    transformWikitextToMobileHtml(hyper, req) {
+        const rp = req.params;
+        return hyper.post({
+            uri: new URI(
+                [rp.domain, 'sys', 'parsoid', 'transform', 'wikitext', 'to', 'html', rp.title]
+            ),
+            headers: {
+                'accept-language': req.headers && req.headers['accept-language']
+            },
+            body: {
+                wikitext: req.body.wikitext
+            }
+        })
+        .then((res) => {
+            return hyper.post({
+                uri: new URI(`${this._options.mobileapps_host}/${rp.domain}` +
+                    `/v1/transform/html/to/mobile-html/${rp.title}`),
+                headers: {
+                    'content-type': res.headers['content-type']
+                },
+                body: res.body
+            })
+            .tap((mobileRes) => {
+                mobileRes.headers['content-language'] = res.headers['content-language'];
+            });
+        });
+    }
+}
+
+module.exports = (options) => {
+    const service = new TransformService(options);
+    const spec = HyperSwitch.utils.loadSpec(`${__dirname}/transform.yaml`);
+    return {
+        spec,
+        operations: {
+            transformWikitextToMobileHtml: service.transformWikitextToMobileHtml.bind(service)
+        }
+    };
+};
diff --git a/v1/pcs/transform.yaml b/v1/pcs/transform.yaml
@@ -0,0 +1,81 @@
+openapi: 3.0.1
+info:
+  version: 1.0.0-beta
+  title: MediaWiki page mobile-html API
+  description: API for retrieving page content for reading purposes
+  termsOfService: https://www.mediawiki.org/wiki/REST_API#Terms_and_conditions
+  contact:
+    name: Reading Infrastructure
+    url: https://www.mediawiki.org/wiki/Wikimedia_Reading_Infrastructure_team
+  license:
+    name: Apache licence, v2
+    url: https://www.apache.org/licenses/LICENSE-2.0
+tags:
+  - name: Transforms
+    description: convert content between HTML and Wikitext
+paths:
+  /wikitext/to/mobile-html/{title}:
+    post:
+      tags:
+        - Transforms
+      summary: Transform Wikitext to Mobile HTML
+      description: |
+        Transform wikitext to Mobile HTML.
+
+        - Stability: [stable](https://www.mediawiki.org/wiki/API_versioning#Stable)
+        - Rate limit: 25 req/s (5 req/s when `stash: true`)
+      parameters:
+        - name: title
+          in: path
+          description: 'Page title. Use underscores instead of spaces. Example: `Main_Page`.'
+          required: true
+          schema:
+            type: string
+        - name: Accept-Language
+          in: header
+          description: |
+            The desired language variant code for wikis where LanguageConverter is enabled. Example: `sr-el` for Latin transcription of the Serbian language.
+          schema:
+            type: string
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              required:
+                - wikitext
+              properties:
+                wikitext:
+                  type: string
+                  description: The Wikitext to transform to HTML
+                  x-textarea: true
+        required: true
+      responses:
+        200:
+          description: See wikipage https://www.mediawiki.org/wiki/Parsoid/MediaWiki_DOM_spec
+          content:
+            text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/Mobile-HTML/1.0.0":
+              schema:
+                type: string
+        404:
+          description: Unknown page title
+          content:
+            application/problem+json:
+              schema:
+                $ref: '#/components/schemas/problem'
+        default:
+          description: Error
+          content:
+            application/problem+json:
+              schema:
+                $ref: '#/components/schemas/problem'
+      x-route-filters:
+        - type: default
+          name: ratelimit_route
+          options:
+            limits:
+              internal: 10
+              external: 25
+      operationId: transformWikitextToMobileHtml
+      x-monitor: false
+
+