Merge "Read default wiki list from a cached sitematrix"

api/localsettings.js.example

@@ -3,27 +3,26 @@
  *
  * Copy this file to localsettings.js and edit that file to fit your needs.
  *
- * Also see the file ParserService.js for more information.
+ * Also see the file server.js for more information.
  */
 
 exports.setup = function( parsoidConfig ) {
-	// The URL of your MediaWiki API endpoint
-	//
-	// We pre-define wikipedias as 'enwiki', 'dewiki' etc. Similarly for
-	// other projects: 'enwiktionary', 'enwikiquote', 'enwikibooks',
-	// 'enwikivoyage' etc.
-	//
-	// Optionally, you can also pass in a proxy specific to this prefix
-	// (overrides defaultAPIProxyURI), or null to disable proxying for
-	// this end point.
+	// The URL of your MediaWiki API endpoint. Optionally, you can also pass
+	// in a proxy specific to this prefix (overrides defaultAPIProxyURI), or
+	// null to disable proxying for this end point.
 	parsoidConfig.setInterwiki( 'localhost', 'http://localhost/w/api.php' );
 
+	// We pre-define wikipedias as 'enwiki', 'dewiki' etc. Similarly
+	// for other projects: 'enwiktionary', 'enwikiquote', 'enwikibooks',
+	// 'enwikivoyage' etc. (default true)
+	//parsoidConfig.loadWMF = false;
+
 	// A default proxy to connect to the API endpoints. Default: undefined
 	// (no proxying). Overridden by per-wiki proxy config in setInterwiki.
-	// parsoidConfig.defaultAPIProxyURI = 'http://proxy.example.org:8080';
+	//parsoidConfig.defaultAPIProxyURI = 'http://proxy.example.org:8080';
 
 	// Enable debug mode (prints extra debugging messages)
-	// parsoidConfig.debug = true;
+	//parsoidConfig.debug = true;
 
 	// Use the PHP preprocessor to expand templates via the MW API (default true)
 	//parsoidConfig.usePHPPreProcessor = false;

api/server.js

@@ -5,12 +5,13 @@
  *
  * Local configuration:
  *
- * To configure locally, add localsettings.js to this directory and export a setup function.
+ * To configure locally, add localsettings.js to this directory and export a
+ * setup function.
  *
  * example:
- *	exports.setup = function( config, env ) {
- *		env.setInterwiki( 'localhost', 'http://localhost/wiki' );
- *	};
+ *   exports.setup = function( parsoidConfig ) {
+ *     parsoidConfig.setInterwiki( 'localhost', 'http://localhost/wiki' );
+ *   };
  *
  * Alternatively, specify a --config file explicitly. See --help for other
  * options.

lib/fetch-sitematrix.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+/**
+ * Simple script to update sitematrix.json
+ */
+
+"use strict";
+require("./core-upgrade.js");
+
+var fs = require("fs"),
+	writeFile = Promise.promisify(fs.writeFile, false, fs),
+	request = Promise.promisify(require('request'), true),
+	downloadUrl = "https://en.wikipedia.org/w/api.php?action=sitematrix&format=json",
+	filename = "sitematrix.json";
+
+request({
+	url: downloadUrl,
+	json: true
+}).spread(function(res, body) {
+	if ( res.statusCode !== 200 ) {
+		throw "Error fetching sitematrix! Returned " + res.statusCode;
+	}
+	return writeFile(filename, JSON.stringify(body));
+}).then(function() {
+	console.log("Success!");
+}).done();

lib/mediawiki.ApiRequest.js

@@ -170,7 +170,7 @@ ApiRequest.prototype._requestCB = function (error, response, body) {
 			this.env.log("warning", "non-200 response:", response.statusCode, body);
 		}
 		error = new Error( this.reqType + ' failure for '
-					+ JSON.stringify(this.queueKey.substr(0, 20)) + ': ' + response.statusCode );
+					+ JSON.stringify(this.queueKey.substr(0, 80)) + ': ' + response.statusCode );
 		this._handleBody( error, '{}' );
 	}
 

lib/mediawiki.ParsoidConfig.js

@@ -7,79 +7,9 @@
 require('./core-upgrade.js');
 var url = require('url'),
 	Cite = require('./ext.Cite.js').Cite,
-	Util = require('./mediawiki.Util.js').Util;
+	Util = require('./mediawiki.Util.js').Util,
+	sitematrix = require('./sitematrix.json').sitematrix;
 
-var languages = "en|de|fr|nl|it|pl|es|ru|ja|pt|zh|sv|vi|uk|ca|no|fi|cs|hu|ko|fa|id|tr|ro|ar|sk|eo|da|sr|lt|ms|eu|he|sl|bg|kk|vo|war|hr|hi|et|az|gl|simple|nn|la|th|el|new|roa-rup|oc|sh|ka|mk|tl|ht|pms|te|ta|be-x-old|ceb|br|be|lv|sq|jv|mg|cy|lb|mr|is|bs|yo|an|hy|fy|bpy|lmo|pnb|ml|sw|bn|io|af|gu|zh-yue|ne|nds|ku|ast|ur|scn|su|qu|diq|ba|tt|my|ga|cv|ia|nap|bat-smg|map-bms|wa|kn|als|am|bug|tg|gd|zh-min-nan|yi|vec|hif|sco|roa-tara|os|arz|nah|uz|sah|mn|sa|mzn|pam|hsb|mi|li|ky|si|co|gan|glk|ckb|bo|fo|bar|bcl|ilo|mrj|fiu-vro|nds-nl|tk|vls|se|gv|ps|rue|dv|nrm|pag|koi|pa|rm|km|kv|udm|csb|mhr|fur|mt|wuu|lij|ug|lad|pi|zea|sc|bh|zh-classical|nov|ksh|or|ang|kw|so|nv|xmf|stq|hak|ay|frp|frr|ext|szl|pcd|ie|gag|haw|xal|ln|rw|pdc|pfl|krc|crh|eml|ace|gn|to|ce|kl|arc|myv|dsb|vep|pap|bjn|as|tpi|lbe|wo|mdf|jbo|kab|av|sn|cbk-zam|ty|srn|kbd|lo|ab|lez|mwl|ltg|ig|na|kg|tet|za|kaa|nso|zu|rmy|cu|tn|chr|got|sm|bi|mo|bm|iu|chy|ik|pih|ss|sd|pnt|cdo|ee|ha|ti|bxr|om|ks|ts|ki|ve|sg|rn|dz|cr|lg|ak|tum|fj|st|tw|ch|ny|ff|xh|ng|ii|cho|mh|aa|kj|ho|mus|kr|hz|tyv|min|mai";
-
-var interwikiMap = new Map();
-var reverseIWMap = new Map();
-
-function insertInMaps( prefix, domain, path, protocol ) {
-	interwikiMap.set( prefix, (protocol || 'http://') + domain + path );
-	reverseIWMap.set( domain, prefix );
-}
-
-languages.split('|').forEach(function(lang) {
-	[ 'wikipedia', 'wikivoyage', 'wikibooks', 'wikisource', 'wikinews',
-	  'wikiquote', 'wikiversity', 'wiktionary'
-	].forEach(function(suffix) {
-		insertInMaps(
-			lang.replace(/-/g, '_') + suffix.replace('pedia', ''),
-			lang + '.' + suffix + '.org',
-			'/w/api.php'
-		 );
-	});
-});
-
-// Add mediawiki.org, commons, old wikisource and localhost too
-insertInMaps( 'mediawikiwiki', 'www.mediawiki.org', '/w/api.php' );
-insertInMaps( 'commonswiki', 'commons.wikimedia.org', '/w/api.php' );
-insertInMaps( 'sourceswiki', 'wikisource.org', '/w/api.php' );
-insertInMaps( 'localhost', 'localhost', '/wiki/api.php' );
-
-// Build the interwiki regexp
-var it = interwikiMap.keys(),
-	key = it.next(),
-	interwikiRegexp = key.value;
-
-while ( !key.done ) {
-	interwikiRegexp += "|" + key.value;
-	key = it.next();
-}
-
-// Subclass a Map to avoid overwriting defaults
-// Maybe just let that happen ... it only seems to occur
-// from localSettings
-function DefaultMap(defaultMap) {
-	Map.call(this);
-	this.defaultMap = defaultMap;
-}
-Object.setPrototypeOf(DefaultMap, Map);
-DefaultMap.prototype = Object.create(Map.prototype, {
-	constructor: { value: DefaultMap },
-	get: {
-		writeable: false,
-		value: function(key) {
-			return this.has(key)
-				? Map.prototype.get.call(this, key)
-				: this.defaultMap.get(key);
-		}
-	},
-	keys: {
-		writeable: false,
-		value: function() {
-			// This return an array because the es6-shim doesn't
-			// expose Iterators.
-			var keys = Array.from(Map.prototype.keys.call(this));
-			this.defaultMap.forEach(function(val, key) {
-				if ( keys.indexOf(key) < 0 ) {
-					keys.push(key);
-				}
-			});
-			return keys;
-		}
-	}
-});
 
 /**
  * @class
@@ -94,15 +24,10 @@ DefaultMap.prototype = Object.create(Map.prototype, {
  * @param {Object} options Any options we want to set over the defaults. Will not overwrite things set by the localSettings.setup function. See the class properties for more information.
  */
 function ParsoidConfig( localSettings, options ) {
-	// 'enwiki' etc for per-wiki proxy. Used by setInterwiki.
+	this.interwikiMap = new Map();
+	this.reverseIWMap = new Map();
 	this.apiProxyURIs = new Map();
-
-	// The default api proxy, overridden by apiProxyURIs entries
-	this.defaultAPIProxyURI = undefined;
-
-	this.interwikiMap = new DefaultMap(interwikiMap);
-	this.reverseIWMap = new DefaultMap(reverseIWMap);
-	this.interwikiRegexp = interwikiRegexp;
+	this.interwikiRegexp = "";
 
 	if ( localSettings && localSettings.setup ) {
 		localSettings.setup( this );
@@ -114,6 +39,10 @@ function ParsoidConfig( localSettings, options ) {
 		Object.assign( this, options );
 	}
 
+	if ( this.loadWMF ) {
+		this.initInterwikiMap();
+	}
+
 	// SSS FIXME: Hardcoded right now, but need a generic registration mechanism
 	// for native handlers
 	this.nativeExtensions = {
@@ -135,15 +64,18 @@ function ParsoidConfig( localSettings, options ) {
  * null to explicitly disable API request proxying for this wiki. Will fall
  * back to ParsoidConfig.defaultAPIProxyURI if undefined (default value).
  */
-ParsoidConfig.prototype.setInterwiki = function ( prefix, apiURI, apiProxyURI ) {
+ParsoidConfig.prototype.setInterwiki = function( prefix, apiURI, apiProxyURI ) {
 	this.interwikiMap.set( prefix, apiURI );
 	this.reverseIWMap.set( url.parse( apiURI ).host, prefix );
 
 	if ( apiProxyURI !== undefined ) {
 		this.apiProxyURIs.set(prefix, apiProxyURI);
+	} else {
+		this.apiProxyURIs.delete(prefix);
 	}
-	if ( this.interwikiRegexp.match( '\\|' + prefix + '\\|' ) === null ) {
-		this.interwikiRegexp += '|' + prefix;
+
+	if ( this.interwikiRegexp.match( '(^|\\|)' + prefix + '(\\||$)' ) === null ) {
+		this.interwikiRegexp += (this.interwikiRegexp ? '|' : '') + prefix;
 	}
 };
 
@@ -154,10 +86,14 @@ ParsoidConfig.prototype.setInterwiki = function ( prefix, apiURI, apiProxyURI )
  *
  * @param {string} prefix
  */
-ParsoidConfig.prototype.removeInterwiki = function ( prefix ) {
+ParsoidConfig.prototype.removeInterwiki = function( prefix ) {
+	if ( !this.interwikiMap.has(prefix) ) {
+		return;
+	}
 	var u = url.parse( this.interwikiMap.get(prefix) );
-	this.reverseIWMap.delete( u.host, null );
-	this.interwikiMap.delete( prefix, null );
+	this.reverseIWMap.delete(u.host);
+	this.interwikiMap.delete(prefix);
+	this.apiProxyURIs.delete(prefix);
 	this.interwikiRegexp = this.interwikiRegexp.replace(
 		new RegExp( '(^|\\|)' + prefix + '(\\||$)' ), function() {
 			return arguments[0] === ("|" + prefix + "|") ? "|" : '';
@@ -276,8 +212,41 @@ ParsoidConfig.prototype.tracerBackend = null;
  */
 ParsoidConfig.prototype.strictSSL = true;
 
-/* Expose the list of languages for use in localsettings. */
-ParsoidConfig.prototype.languages = languages;
+/**
+ * The default api proxy, overridden by apiProxyURIs entries.
+ */
+ParsoidConfig.prototype.defaultAPIProxyURI = undefined;
+
+/**
+ * Load WMF sites in the interwikiMap from the cached sitematrix.json
+ */
+ParsoidConfig.prototype.loadWMF = true;
+
+/**
+ * Initialize the interwikiMap and friends.
+ */
+ParsoidConfig.prototype.initInterwikiMap = function() {
+	var insertInMaps = function( proxy, site ) {
+		// Avoid overwriting those already set in localsettings setup.
+		if ( !this.interwikiMap.has( site.dbname ) ) {
+			this.setInterwiki( site.dbname, site.url + "/w/api.php", proxy );
+		}
+	};
+
+	// See MWParserEnvironment.prototype.getAPIProxyURI for these values.
+	var normal = insertInMaps.bind(this, undefined);
+	var special = insertInMaps.bind(this, null);
+
+	Object.keys( sitematrix ).forEach(function( key ) {
+		var val = sitematrix[key];
+		if ( !Number.isNaN( Number(key) ) ) {
+			val.site.forEach(normal);
+		} else if ( key === "specials" ) {
+			val.forEach(special);
+		}
+	});
+};
+
 
 if (typeof module === "object") {
 	module.exports.ParsoidConfig = ParsoidConfig;

tests/parserTests.js

@@ -1648,9 +1648,10 @@ ParserTests.prototype.main = function ( options, popts ) {
 
 	options.expandExtensions = true;
 
-	var i, key,
-		parsoidConfig = new ParsoidConfig( null, options ),
-		iwmap = parsoidConfig.interwikiMap.keys();
+	var parsoidConfig = new ParsoidConfig( null, options );
+	parsoidConfig.interwikiMap.forEach(function( val, key ) {
+		parsoidConfig.interwikiMap.set(key, mockAPIServerURL);
+	});
 
 	// Set tracing and debugging before the env. object is
 	// constructed since tracing backends are registered there.
@@ -1659,11 +1660,6 @@ ParserTests.prototype.main = function ( options, popts ) {
 	Util.setDebuggingFlags(parsoidConfig, options);
 	Util.setTemplatingAndProcessingFlags( parsoidConfig, options );
 
-	for ( i = 0; i < iwmap.length; i++ ) {
-		key = iwmap[i];
-		parsoidConfig.interwikiMap.set(key, mockAPIServerURL);
-	}
-
 	// Create a new parser environment
 	MWParserEnvironment.getParserEnv( parsoidConfig, null, { prefix: 'enwiki' }, function( err, env ) {
 		// For posterity: err will never be non-null here, because we expect the WikiConfig