Merge 93c8d9a into 59e1721

lib/language_variants_filter.js

@@ -0,0 +1,60 @@
+"use strict";
+
+const HyperSwitch = require('hyperswitch');
+const URI = HyperSwitch.URI;
+const mwUtil = require('./mwUtil');
+
+module.exports = (hyper, req, next, options, specInfo) => {
+    const rp = req.params;
+    const acceptLanguage = req.headers && req.headers['accept-language'];
+    if (!acceptLanguage || mwUtil.isNoCacheRequest(req)) {
+        delete req.headers['accept-language'];
+        return next(hyper, req);
+    }
+
+    const revTableURI = [rp.domain, 'sys', 'page_revisions', 'page', rp.title];
+    if (rp.revision) {
+        revTableURI.push(`${rp.revision}`);
+    }
+    return hyper.get({ uri: new URI(revTableURI) })
+    .then((res) => {
+        const revision = res.body.items[0];
+        return mwUtil.shouldConvertLangVariant(hyper, req, revision.page_language, acceptLanguage)
+        .then((shouldConvert) => {
+            if (!shouldConvert) {
+                delete req.headers['accept-language'];
+                return next(hyper, req);
+            }
+
+            if (/\/page\/html\//.test(specInfo.path)) {
+                // It's HTML, hit Parsoid for conversion
+                return next(hyper, req)
+                .then(html => hyper.post({
+                    uri: new URI([rp.domain, 'sys', 'parsoid', 'transform', 'html', 'to', 'html']),
+                    headers: {
+                        'content-type': 'application/json',
+                        'content-language': revision.page_language
+                    },
+                    body: {
+                        original: {
+                            html: {
+                                headers: html.headers,
+                                body: html.body.toString()
+                            }
+                        },
+                        updates: {
+                            variant: {
+                                source: null,
+                                target: acceptLanguage
+                            }
+                        }
+                    }
+                }));
+                // We can skip setting Vary: accept-language as Parsoid sets it.
+            } else {
+                // TODO: It's something else, so just forward to MCS
+                return next(hyper, req);
+            }
+        });
+    });
+};

lib/mwUtil.js

@@ -66,6 +66,31 @@ mwUtil.parseETag = (etag) => {
     }
 };
 
+/**
+ * Checks whether a specialized requested language variant exist for
+ * a page in a requested language on a particular wiki.
+ * @param {HyperSwitch} hyper
+ * @param {Object} req
+ * @param {string} pageLanguage the language of the requested page.
+ * @param {string} acceptLanguage the language variant accepted by the client.
+ * @return {Promise<boolean>}
+ */
+mwUtil.shouldConvertLangVariant = (hyper, req, pageLanguage, acceptLanguage) => {
+    if (!acceptLanguage) {
+        return P.resolve(false);
+    }
+    return mwUtil.getSiteInfo(hyper, req)
+    .then((siteInfo) => {
+        // First, whether specialized variants exist for the page lang on a wiki
+        const pageLangVariants = siteInfo.languagevariants[pageLanguage];
+        if (!pageLangVariants) {
+            return false;
+        }
+        // Second, if the requested language variant exist for page language
+        return pageLangVariants.indexOf(acceptLanguage) !== -1;
+    });
+};
+
 /**
  * Extract the date from an `etag` header of the form
  * "<revision>/<tid>/<suffix>"
@@ -257,7 +282,7 @@ mwUtil.decodeBody = (contentResponse) => {
  *
  * @param {!HyperSwitch} hyper
  * @param {!Object} req
- * @param {?String} domain Wiki domain to get siteinfo from (defaults to the request domain).
+ * @param {string} [domain] Wiki domain to get siteinfo from (defaults to the request domain).
  */
 mwUtil.getSiteInfo = (hyper, req, domain) => hyper.get({
     uri: new URI([domain || req.params.domain, 'sys', 'action', 'siteinfo'])

sys/action.js

@@ -292,13 +292,31 @@ class ActionService {
                 body: {
                     action: 'query',
                     meta: 'siteinfo|filerepoinfo',
-                    siprop: 'general|namespaces|namespacealiases|specialpagealiases',
+                    siprop: 'general' +
+                        '|namespaces' +
+                        '|namespacealiases' +
+                        '|specialpagealiases' +
+                        '|languagevariants',
                     format: 'json'
                 }
             }, {}, (apiReq, res) => {
                 if (!res || !res.body || !res.body.query || !res.body.query.general) {
                     throw new Error(`SiteInfo is unavailable for ${rp.domain}`);
                 }
+                // Transform from original response format to
+                // {
+                //   'lang' => ['variant1', 'variant2' ... ]
+                // }
+                // for ease of use.
+                const origVariants = res.body.query.languagevariants;
+                const variants = {};
+                if (origVariants) {
+                    Object.keys(origVariants).forEach((lang) => {
+                        variants[lang] = Object.keys(origVariants[lang])
+                        // Filter out non-specific variants like `en`, `zh` etc.
+                        .filter(variant => /-/.test(variant));
+                    });
+                }
                 return {
                     status: 200,
                     body: {
@@ -311,6 +329,7 @@ class ActionService {
                         namespaces: res.body.query.namespaces,
                         namespacealiases: res.body.query.namespacealiases,
                         specialpagealiases: res.body.query.specialpagealiases,
+                        languagevariants: variants,
                         sharedRepoRootURI: findSharedRepoDomain(res),
                         baseUri: this._getBaseUri(req)
                     }

sys/parsoid.js

@@ -713,7 +713,7 @@ class ParsoidService {
             parsoidTo = 'pagebundle';
         }
         let parsoidFrom = from;
-        if (from === 'html' && req.body.original && req.body.original['data-parsoid']) {
+        if (from === 'html' && req.body.original) {
             parsoidFrom = 'pagebundle';
         }
         const parsoidExtras = [];
@@ -735,6 +735,7 @@ class ParsoidService {
             headers: {
                 'content-type': 'application/json',
                 'user-agent': req['user-agent'],
+                'content-language': req.headers['content-language']
             },
             body: req.body
         };
@@ -761,7 +762,10 @@ class ParsoidService {
         return (hyper, req) => {
             const rp = req.params;
             if ((!req.body && req.body !== '')
-                    || (!req.body[from] && req.body[from] !== '')) {
+                    // The html/to/html endpoint is a bit different so the `html`
+                    // might not be provided.
+                    || (!(from === 'html' && to === 'html')
+                        && !req.body[from] && req.body[from] !== '')) {
                 throw new HTTPError({
                     status: 400,
                     body: {

v1/content.yaml

@@ -127,6 +127,7 @@ paths:
     get:
       x-route-filters:
         - path: lib/ensure_content_type.js
+        - path: ./lib/language_variants_filter.js
       tags:
         - Page content
       summary: Get latest HTML for a title.
@@ -399,6 +400,7 @@ paths:
     get:
       x-route-filters:
         - path: lib/ensure_content_type.js
+        - path: ./lib/language_variants_filter.js
       tags:
         - Page content
       summary: Get HTML for a specific title/revision & optionally timeuuid.