From 597932be72082eb00999eb71565c2ac43f724205 Mon Sep 17 00:00:00 2001 From: Arlo Breault Date: Mon, 6 Aug 2018 18:59:23 -0400 Subject: [PATCH] Prepare for MCR Send rvslots so that the mwapi stops warning about using the legacy structure. Also, removes the use of rvcontentformat that was added in e08a13b since it's been deprecated. See https://lists.wikimedia.org/pipermail/mediawiki-api-announce/2018-August/000140.html Bug: T201115 Change-Id: Ic076c4deac896ffced5ba32a7cf3e9c6208d097e --- lib/config/MWParserEnvironment.js | 17 +++-- lib/mw/ApiRequest.js | 21 ++---- lib/utils/Util.js | 8 ++ tests/mocha/api.js | 10 +++ tests/mockAPI.js | 120 +++++++++++++++++++++++------- 5 files changed, 129 insertions(+), 47 deletions(-) diff --git a/lib/config/MWParserEnvironment.js b/lib/config/MWParserEnvironment.js index 9366f8cd9..50c16a197 100644 --- a/lib/config/MWParserEnvironment.js +++ b/lib/config/MWParserEnvironment.js @@ -404,9 +404,13 @@ MWParserEnvironment.prototype.cacheReusableExpansions = function(obj) { * sha1: * size: // in bytes * comment: - * contentmodel: - * contentformat: - * "* ": // actual source text --> copied to this.page.src + * slots: { + * main: { + * contentmodel: + * contentformat: + * "* ": // actual source text --> copied to this.page.src + * } + * } * } * } * ``` @@ -428,14 +432,15 @@ MWParserEnvironment.prototype.setPageSrcInfo = function(srcOrMetadata) { m.title = metadata.title; var r = m.revision; if (!r) { r = m.revision = {}; } + var content = Util.getStar(metadata.revision); if (metadata.revision) { r.revid = metadata.revision.revid; r.parentid = metadata.revision.parentid; r.timestamp = metadata.revision.timestamp; r.sha1 = metadata.revision.sha1; r.size = metadata.revision.size; - r.contentmodel = metadata.revision.contentmodel; - r.contentformat = metadata.revision.contentformat; + r.contentmodel = content && content.contentmodel; + r.contentformat = content && content.contentformat; } // Update other page properties @@ -444,7 +449,7 @@ MWParserEnvironment.prototype.setPageSrcInfo = function(srcOrMetadata) { this.page.latest = metadata.latest; this.page.pagelanguage = metadata.pagelanguage; this.page.pagelanguagedir = metadata.pagelanguagedir; - this.page.src = (metadata.revision && metadata.revision['*']) || ''; + this.page.src = (content && content['*']) || ''; this.page.setVariant(null); }; diff --git a/lib/mw/ApiRequest.js b/lib/mw/ApiRequest.js index 7ffb880db..4bec3a2e0 100644 --- a/lib/mw/ApiRequest.js +++ b/lib/mw/ApiRequest.js @@ -11,6 +11,7 @@ var url = require('url'); var util = require('util'); var Promise = require('../utils/promise.js'); +var Util = require('../utils/Util.js').Util; function setupConnectionTimeout(env, protocol) { @@ -57,9 +58,6 @@ function setupConnectionTimeout(env, protocol) { var latestSerial = 0; -// all revision properties which parsoid is interested in. -var PARSOID_RVPROP = ('content|ids|timestamp|size|sha1|contentmodel'); - var logAPIWarnings = function(req, data) { if (req.env.conf.parsoid.logMwApiWarnings && data && data.hasOwnProperty('warnings')) { @@ -478,15 +476,11 @@ function TemplateRequest(env, title, oldid, opts) { action: 'query', prop: 'info|revisions', rawcontinue: 1, - rvprop: PARSOID_RVPROP, + // all revision properties which parsoid is interested in. + rvprop: 'content|ids|timestamp|size|sha1|contentmodel', + rvslots: 'main', }; - // Allow request to give a specific contentformat - // (otherwise wikitext is the default) - if (opts.contentformat) { - apiargs.rvcontentformat = opts.contentformat; - } - if (oldid) { this.oldid = oldid; apiargs.revids = oldid; @@ -533,7 +527,7 @@ TemplateRequest.prototype._handleJSON = function(error, data) { return; } - var metadata; + var metadata, content; if (!data.query.pages) { if (data.query.interwiki) { // Essentially redirect, but don't actually redirect. @@ -572,7 +566,8 @@ TemplateRequest.prototype._handleJSON = function(error, data) { pagelanguage: page.pagelanguage, pagelanguagedir: page.pagelanguagedir, }; - if (metadata.revision.texthidden || !metadata.revision.hasOwnProperty("*")) { + content = Util.getStar(metadata.revision); + if (metadata.revision.texthidden || !content || !content.hasOwnProperty('*')) { error = new DoesNotExistError("Source is hidden for " + this.title); } return true; @@ -590,7 +585,7 @@ TemplateRequest.prototype._handleJSON = function(error, data) { // Add the source to the cache // (both original title as well as possible redirected title) - this.env.pageCache[this.queueKey] = this.env.pageCache[this.title] = metadata.revision['*']; + this.env.pageCache[this.queueKey] = this.env.pageCache[this.title] = content['*']; this._processListeners(null, metadata); }; diff --git a/lib/utils/Util.js b/lib/utils/Util.js index 0f2d2724b..f1fda92f9 100644 --- a/lib/utils/Util.js +++ b/lib/utils/Util.js @@ -1653,6 +1653,14 @@ Util.validateMediaParam = function(num) { return num > 0; }; +// Extract content in a backwards compatible way +Util.getStar = function(revision) { + var content = revision; + if (revision && revision.slots) { + content = revision.slots.main; + } + return content; +}; if (typeof module === "object") { module.exports.Util = Util; diff --git a/tests/mocha/api.js b/tests/mocha/api.js index e89abbab8..130cc4170 100644 --- a/tests/mocha/api.js +++ b/tests/mocha/api.js @@ -567,6 +567,16 @@ describe('Parsoid API', function() { .end(done); }); + it('should get from a title and revision (html, pre-mcr)', function(done) { + request(api) + .get(mockDomain + '/v3/page/html/Old_Response/999') + .expect(validHtmlResponse(function(doc) { + // SECTION -> P + doc.body.firstChild.firstChild.textContent.should.equal('MediaWiki was successfully installed.'); + })) + .end(done); + }); + it('should get from a title and revision (html, json content)', function(done) { request(api) .get(mockDomain + '/v3/page/html/JSON_Page/101') diff --git a/tests/mockAPI.js b/tests/mockAPI.js index 173e49e0f..c66857d44 100644 --- a/tests/mockAPI.js +++ b/tests/mockAPI.js @@ -88,9 +88,35 @@ var mainPage = { { revid: 1, parentid: 0, + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': 'MediaWiki has been successfully installed.\n\nConsult the [//meta.wikimedia.org/wiki/Help:Contents User\'s Guide] for information on using the wiki software.\n\n== Getting started ==\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Manual:Configuration_settings Configuration settings list]\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Manual:FAQ MediaWiki FAQ]\n* [https://lists.wikimedia.org/mailman/listinfo/mediawiki-announce MediaWiki release mailing list]\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Localisation#Translation_resources Localise MediaWiki for your language]', + }, + }, + }, + ], + }, + }, + }, +}; + +// Old response structure, pre-mcr +var oldResponse = { + query: { + pages: { + '999': { + pageid: 999, + ns: 0, + title: 'Old Response', + revisions: [ + { + revid: 999, + parentid: 0, contentmodel: 'wikitext', contentformat: 'text/x-wiki', - '*': 'MediaWiki has been successfully installed.\n\nConsult the [//meta.wikimedia.org/wiki/Help:Contents User\'s Guide] for information on using the wiki software.\n\n== Getting started ==\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Manual:Configuration_settings Configuration settings list]\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Manual:FAQ MediaWiki FAQ]\n* [https://lists.wikimedia.org/mailman/listinfo/mediawiki-announce MediaWiki release mailing list]\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Localisation#Translation_resources Localise MediaWiki for your language]', + '*': 'MediaWiki was successfully installed.\n\nConsult the [//meta.wikimedia.org/wiki/Help:Contents User\'s Guide] for information on using the wiki software.\n\n== Getting started ==\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Manual:Configuration_settings Configuration settings list]\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Manual:FAQ MediaWiki FAQ]\n* [https://lists.wikimedia.org/mailman/listinfo/mediawiki-announce MediaWiki release mailing list]\n* [//www.mediawiki.org/wiki/Special:MyLanguage/Localisation#Translation_resources Localise MediaWiki for your language]', }, ], }, @@ -109,9 +135,13 @@ var junkPage = { { revid: 2, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': '2. This is just some junk. See the comment above.', + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': '2. This is just some junk. See the comment above.', + }, + }, }, ], }, @@ -130,9 +160,13 @@ var largePage = { { revid: 3, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': 'a'.repeat(parsoidOptions.limits.wt2html.maxWikitextSize + 1), + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': 'a'.repeat(parsoidOptions.limits.wt2html.maxWikitextSize + 1), + }, + }, }, ], }, @@ -151,9 +185,13 @@ var reusePage = { { revid: 100, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': '{{colours of the rainbow}}', + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': '{{colours of the rainbow}}', + }, + }, }, ], }, @@ -172,9 +210,13 @@ var jsonPage = { { revid: 101, parentid: 0, - contentmodel: 'json', - contentformat: 'text/json', - '*': '[1]', + slots: { + main: { + contentmodel: 'json', + contentformat: 'text/json', + '*': '[1]', + }, + }, }, ], }, @@ -193,9 +235,13 @@ var lintPage = { { revid: 102, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': '{|\nhi\n|ho\n|}', + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': '{|\nhi\n|ho\n|}', + }, + }, }, ], }, @@ -214,9 +260,13 @@ var redlinksPage = { { revid: 103, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': '[[Special:Version]] [[Doesnotexist]] [[Redirected]]', + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': '[[Special:Version]] [[Doesnotexist]] [[Redirected]]', + }, + }, }, ], }, @@ -235,9 +285,13 @@ var variantPage = { { revid: 104, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': 'абвг abcd', + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': 'абвг abcd', + }, + }, }, ], pagelanguage: 'sr', @@ -258,9 +312,13 @@ var noVariantPage = { { revid: 105, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': 'абвг abcd\n__NOCONTENTCONVERT__', + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': 'абвг abcd\n__NOCONTENTCONVERT__', + }, + }, }, ], pagelanguage: 'sr', @@ -281,9 +339,13 @@ var revisionPage = { { revid: 63, parentid: 0, - contentmodel: 'wikitext', - contentformat: 'text/x-wiki', - '*': '{{REVISIONID}}', + slots: { + main: { + contentmodel: 'wikitext', + contentformat: 'text/x-wiki', + '*': '{{REVISIONID}}', + }, + }, }, ], }, @@ -547,6 +609,8 @@ var availableActions = { return cb(null , variantPage); } else if (body.revids === '105' || body.titles === 'No_Variant_Page') { return cb(null , noVariantPage); + } else if (body.revids === '999' || body.titles === 'Old_Response') { + return cb(null , oldResponse); } else { return cb(null, { query: { pages: { '-1': {