Skip to content

Commit

Permalink
Merge 1ff0085 into d08ad0e
Browse files Browse the repository at this point in the history
  • Loading branch information
hknustwmf committed Sep 6, 2019
2 parents d08ad0e + 1ff0085 commit 24de582
Show file tree
Hide file tree
Showing 7 changed files with 144 additions and 9 deletions.
4 changes: 3 additions & 1 deletion config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,10 @@ services:
# XXX Check the base RESTBase URI
baseUriTemplate: "{{'http://{domain}:7231/{domain}/v1'}}"
parsoid:
# XXX Check Parsoid URL!
# XXX Check Parsoid/JS URL!
host: http://localhost:8142
# XXX Check Parsoid/PHP URL!
host_php: http://localhost:8142
table:
backend: sqlite
dbname: db.sqlite3
Expand Down
1 change: 1 addition & 0 deletions config.frontend.test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ default_project: &default_project
options: &default_options
parsoid:
host: https://parsoid-beta.wmflabs.org
host_php: https://parsoid-beta.wmflabs.org
grace_ttl: 1000000
action:
apiUriTemplate: "{{'https://{domain}/w/api.php'}}"
Expand Down
1 change: 1 addition & 0 deletions config.fullstack.test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ default_project: &default_project
options: &default_options
parsoid:
host: https://parsoid-beta.wmflabs.org
host_php: https://parsoid-beta.wmflabs.org
grace_ttl: 1000000
action:
apiUriTemplate: "{{'https://{domain}/w/api.php'}}"
Expand Down
15 changes: 13 additions & 2 deletions projects/sys/default.wmf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,13 @@ paths:
- path: sys/page_save.js
/parsoid:
x-modules:
- path: sys/parsoid.js
options:
- path: sys/parsoid_proxy.js
/parsoidjs:
x-modules:
- path: sys/parsoid.js
options: &parsoidopts
parsoidHost: '{{options.parsoid.host}}'
bucketName: 'parsoid'
response_cache_control: '{{options.purged_cache_control}}'
grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
# A list of pages that we don't currently want to re-render on
Expand Down Expand Up @@ -116,6 +120,13 @@ paths:
- '/Commons:Undeletion_requests\//'
- '/Commons:WikiProject_Aviation\/recent_uploads\//'
- '/^(?:User|Benutzer):/'
/parsoidphp:
x-modules:
- path: sys/parsoid.js
options:
<<: *parsoidopts
parsoidHost: '{{options.parsoid.host_php}}'
bucketName: 'parsoidphp'
/events:
x-modules:
- path: sys/events.js
Expand Down
13 changes: 7 additions & 6 deletions sys/parsoid.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function extractTidMeta(html) {
* in `if-unmodified-since` header of the request
* @param {Object} req the request
* @param {Object} res the response
* @return {boolean} true if content has beed modified
* @return {boolean} true if content has been modified
*/
function isModifiedSince(req, res) {
try {
Expand Down Expand Up @@ -125,10 +125,11 @@ class ParsoidService {
_initOpts(opts = {}) {
this.options = opts;
this.parsoidHost = opts.parsoidHost;
this.bucketName = opts.bucketName || 'parsoid';
this.options.stash_ratelimit = opts.stash_ratelimit || 5;
this.options.grace_ttl = opts.grace_ttl || 86400;
this._blacklist = compileReRenderBlacklist(opts.rerenderBlacklist);
if (!opts.parsoidHost) {
if (!this.parsoidHost) {
throw new Error('Parsoid module: the option parsoidHost must be provided!');
}
}
Expand Down Expand Up @@ -170,7 +171,7 @@ class ParsoidService {
*/
getLatestBucketURI(domain, title) {
return new URI([
domain, 'sys', 'key_value', 'parsoid', title
domain, 'sys', 'key_value', this.bucketName, title
]);
}

Expand All @@ -186,7 +187,7 @@ class ParsoidService {
*/
getStashBucketURI(domain, title, revision, tid) {
return new URI([
domain, 'sys', 'key_value', 'parsoid-stash', `${title}:${revision}:${tid}`
domain, 'sys', 'key_value', `${this.bucketName}-stash`, `${title}:${revision}:${tid}`
]);
}

Expand Down Expand Up @@ -799,7 +800,7 @@ module.exports = (options) => {
// Dynamic resource dependencies, specific to implementation
resources: [
{
uri: '/{domain}/sys/key_value/parsoid',
uri: `/{domain}/sys/key_value/${ps.bucketName}`,
headers: {
'content-type': 'application/json'
},
Expand All @@ -808,7 +809,7 @@ module.exports = (options) => {
}
},
{
uri: '/{domain}/sys/key_value/parsoid-stash',
uri: `/{domain}/sys/key_value/${ps.bucketName}-stash`,
headers: {
'content-type': 'application/json'
},
Expand Down
48 changes: 48 additions & 0 deletions sys/parsoid_proxy.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
'use strict';

/*
* Simple proxy to route requests to the client-requested
* Parsoid variant (JS or PHP) during the transition
* period. Parsoid/JS is being phased out and replaced by
* Parsoid/PHP.
*/

const HyperSwitch = require('hyperswitch');
const URI = HyperSwitch.URI;
const mwUtil = require('../lib/mwUtil');
const VARIANT_HDR_NAME = 'x-parsoid-variant';

module.exports = () => {
return {
spec: {
paths: {
'/{+path}': {
all: {
operationId: 'proxy_parsoid_variant'
}
}
}
},
operations: {
proxy_parsoid_variant: (hyper, req) => {
const rootReqHeaders = hyper._rootReq.headers || {};
rootReqHeaders[VARIANT_HDR_NAME] = rootReqHeaders[VARIANT_HDR_NAME] || 'JS';
const isPhpVariant = /PHP/i.test(rootReqHeaders[VARIANT_HDR_NAME]);

return hyper.request({
method: req.method,
uri: new URI(req.uri.toString().replace('/parsoid/',
isPhpVariant ? '/parsoidphp/' : '/parsoidjs/')),
headers: req.headers,
body: req.body,
query: req.query
})
.tap((res) => {
res.headers = res.headers || {};
res.headers[VARIANT_HDR_NAME] = rootReqHeaders[VARIANT_HDR_NAME];
mwUtil.addVaryHeader(res, VARIANT_HDR_NAME);
});
}
}
};
};
71 changes: 71 additions & 0 deletions test/features/parsoid/php_variant.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
'use strict';

const assert = require('../../utils/assert.js');
const Server = require('../../utils/server.js');
const preq = require('preq');

const testPage = {
title: 'User:Pchelolo%2fRestbase_Test',
revision: '275854',
wikitext: '<div id=bar>Selser test'
// html is fetched dynamically
};

describe('php parsoid variant', function() {
this.timeout(20000);
let contentTypes;
const server = new Server();
before(() => {
return server.start()
.then(() => {
contentTypes = server.config.conf.test.content_types;
return preq.get({
uri: `${server.config.bucketURL('en.wikipedia.beta.wmflabs.org')}/html/${testPage.title}/${testPage.revision}`
});
})
.then((res) => {
testPage.html = res.body;
});
});
after(() => server.stop());

it('wt2html default variant', () => {
return preq.post({
uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/transform/wikitext/to/html/${testPage.title}`,
body: {
wikitext: '== Heading =='
}
})
.then((res) => {
assert.deepEqual(res.status, 200);
assert.contentType(res, contentTypes.html);
assert.deepEqual(res.headers['x-parsoid-variant'], 'JS');
const pattern = /<h2.*>Heading<\/h2>/;
if (!pattern.test(res.body)) {
throw new Error(`Expected pattern in response: ${pattern}\nSaw: ${res.body}`);
}
});
});

it('wt2html php variant', () => {
return preq.post({
uri: `${server.config.baseURL('en.wikipedia.beta.wmflabs.org')}/transform/wikitext/to/html/${testPage.title}`,
body: {
wikitext: '== Heading =='
},
headers: {
'x-parsoid-variant': 'PHP'
}
})
.then((res) => {
assert.deepEqual(res.status, 200);
assert.contentType(res, contentTypes.html);
assert.deepEqual(res.headers['x-parsoid-variant'], 'PHP');
const pattern = /<h2.*>Heading<\/h2>/;
if (!pattern.test(res.body)) {
throw new Error(`Expected pattern in response: ${pattern}\nSaw: ${res.body}`);
}
});
});
});

0 comments on commit 24de582

Please sign in to comment.