Skip to content

Commit

Permalink
Add subpages and wikilinks sources to Massviews
Browse files Browse the repository at this point in the history 
Rescue a bunch of i18n messages that otherwise be lost with this commit

Bug: T144251, T144238
  • Loading branch information
@MusikAnimal
MusikAnimal committed Sep 19, 2016
1 parent 6f019dc commit b95f99c
Show file tree
Hide file tree
Showing 61 changed files with 1,317 additions and 526 deletions.
10 changes: 9 additions & 1 deletion javascripts/massviews/config.js
View file
Expand Up @@ -37,10 +37,18 @@ const config = {
placeholder: 'https://en.wikipedia.org/wiki/Category:Folk_musicians_from_New_York',
type: 'text'
},
wikilinks: {
placeholder: 'https://en.wikipedia.org/wiki/Book:New_York_City',
type: 'text'
},
pagepile: {
placeholder: '12345',
type: 'number'
},
subpages: {
placeholder: 'https://en.wikipedia.org/wiki/User:Example',
type: 'text'
},
transclusions: {
placeholder: 'https://en.wikipedia.org/wiki/Template:Infobox_Olympic_games',
type: 'text'
Expand All @@ -59,7 +67,7 @@ const config = {
validParams: {
direction: ['-1', '1'],
sort: ['title', 'views', 'original'],
source: ['pagepile', 'category', 'transclusions', 'quarry'],
source: ['pagepile', 'category', 'subpages', 'transclusions', 'quarry'],
view: ['list', 'chart'],
subjectpage: ['0', '1']
}
Expand Down
233 changes: 222 additions & 11 deletions javascripts/massviews/massviews.js
View file
Expand Up @@ -652,10 +652,7 @@ class MassViews extends mix(Pv).with(ChartHelpers, ListHelpers) {
});
}

processCategory(cb) {
const [project, category] = this.getWikiPageFromURL($(this.config.sourceInput).val());
if (!this.validateProject(project)) return;

processCategory(project, category, cb) {
if (!category) {
return this.setState('initial', () => {
this.writeMessage($.i18n('invalid-category-url'));
Expand All @@ -672,11 +669,6 @@ class MassViews extends mix(Pv).with(ChartHelpers, ListHelpers) {
titles: decodeURIComponent(category)
};

if ($('.category-subject-toggle--input').is(':checked')) {
requestData.meta = 'siteinfo';
requestData.siprop = 'namespaces';
}

const promise = $.ajax({
url: `https://${project}/w/api.php`,
jsonp: 'callback',
Expand Down Expand Up @@ -704,7 +696,9 @@ class MassViews extends mix(Pv).with(ChartHelpers, ListHelpers) {
}

const size = data.query.pages[queryKey].categoryinfo.size,
namespaces = data.query.namespaces;
// siteInfo is only populated if they've opted to see subject pages instead of talk pages
// Otherwise namespaces are not needed by this.mapCategoryPageNames
namespaces = this.siteInfo ? this.siteInfo.namespaces : undefined;
let pages = data.query.categorymembers;

if (!pages.length) {
Expand Down Expand Up @@ -751,6 +745,114 @@ class MassViews extends mix(Pv).with(ChartHelpers, ListHelpers) {
});
}

processSubpages(project, targetPage, cb) {
if (!targetPage) {
return this.setState('initial', () => {
this.writeMessage($.i18n('invalid-page-url'));
});
}

this.sourceProject = project; // for caching purposes

// determine what namespace the targetPage is in
const descoredTargetPage = targetPage.descore();
let namespace = 0, queryTargetPage;
for (const ns in this.siteInfo.namespaces) {
if (ns === '0') continue; // skip mainspace

const nsName = this.siteInfo.namespaces[ns]['*'] + ':';
if (descoredTargetPage.startsWith(nsName)) {
namespace = this.siteInfo.namespaces[ns].id;
queryTargetPage = targetPage.substring(nsName.length);
}
}

// get namespace number of corresponding talk or subject page
const inverseNamespace = namespace % 2 === 0 ? namespace + 1 : namespace - 1;

let promises = [];

[namespace, inverseNamespace].forEach(apnamespace => {
promises.push(
$.ajax({
url: `https://${project}/w/api.php`,
jsonp: 'callback',
dataType: 'jsonp',
data: {
action: 'query',
format: 'json',
list: 'allpages',
aplimit: 500,
apnamespace,
apprefix: decodeURIComponent(queryTargetPage) + '/'
}
})
);
});

const pageLink = this.getPageLink(decodeURIComponent(targetPage), project);

$.when(...promises).done((data, data2) => {
// show errors, if any
const errors = [data, data2].filter(resp => !!resp[0].error);
if (errors.length) {
errors.forEach(error => {
this.setState('initial', () => {
this.writeMessage(
`${$.i18n('api-error', 'Allpages API')}: ${error[0].error.info.escape()}`
);
});
});
return false;
}

let pages = data[0].query.allpages.concat(data2[0].query.allpages);
const size = pages.length;

if (size === 0) {
return this.setState('initial', () => {
this.writeMessage($.i18n('api-error-no-data'));
});
}

if (size > this.config.pageLimit) {
this.writeMessage(
$.i18n('massviews-oversized-set', pageLink, this.formatNumber(size), this.config.pageLimit)
);

pages = pages.slice(0, this.config.pageLimit);
}

/**
* XXX: throttling
* At this point we know we have data to process,
* so set the throttle flag to disallow additional requests for the next 90 seconds
*/
if (size > 10) this.setThrottle();

const pageNames = pages.map(page => page.title).concat([targetPage]);

this.getPageViewsData(project, pageNames).done(pageViewsData => {
$('.output-title').html(pageLink);
$('.output-params').html($(this.config.dateRangeSelector).val());
this.buildMotherDataset(targetPage, pageLink, pageViewsData);

cb();
});
}).fail(data => {
this.setState('initial');

/** structured error comes back as a string, otherwise we don't know what happened */
if (data && typeof data.error === 'string') {
this.writeMessage(
$.i18n('api-error', pageLink + ': ' + data.error)
);
} else {
this.writeMessage($.i18n('api-error-unknown', pageLink));
}
});
}

processTemplate(cb) {
const [project, template] = this.getWikiPageFromURL($(this.config.sourceInput).val());
if (!this.validateProject(project)) return;
Expand Down Expand Up @@ -836,6 +938,91 @@ class MassViews extends mix(Pv).with(ChartHelpers, ListHelpers) {
});
}

processWikiPage(cb) {
const [project, page] = this.getWikiPageFromURL($(this.config.sourceInput).val());
if (!this.validateProject(project)) return;

if (!page) {
return this.setState('initial', () => {
this.writeMessage($.i18n('invalid-page-url'));
});
}

const promise = $.ajax({
url: `https://${project}/w/api.php`,
jsonp: 'callback',
dataType: 'jsonp',
data: {
action: 'query',
format: 'json',
pllimit: 500,
titles: decodeURIComponent(page),
prop: 'links'
}
});
const pageLink = this.getPageLink(decodeURIComponent(page), project);
this.sourceProject = project; // for caching purposes

promise.done(data => {
if (data.error) {
return this.setState('initial', () => {
this.writeMessage(
`${$.i18n('api-error', 'Links API')}: ${data.error.info.escape()}`
);
});
}

const queryKey = Object.keys(data.query.pages)[0];

if (queryKey === '-1' || !data.query.pages[queryKey].links) {
return this.setState('initial', () => {
this.writeMessage($.i18n('api-error-no-data'));
});
}

const pages = data.query.pages[queryKey].links.map(page => page.title);

if (!pages.length) {
return this.setState('initial', () => {
this.writeMessage($.i18n('massviews-empty-set', pageLink));
});
}

// in this case we are limited by the API to 500 pages, not this.config.pageLimit
if (data.continue) {
this.writeMessage(
$.i18n('massviews-oversized-set-unknown', pageLink, 500)
);
}

/**
* XXX: throttling
* At this point we know we have data to process,
* so set the throttle flag to disallow additional requests for the next 90 seconds
*/
if (pages.length > 10) this.setThrottle();

this.getPageViewsData(project, pages).done(pageViewsData => {
$('.output-title').html(pageLink);
$('.output-params').html($(this.config.dateRangeSelector).val());
this.buildMotherDataset(page, pageLink, pageViewsData);

cb();
});
}).fail(data => {
this.setState('initial');

/** structured error comes back as a string, otherwise we don't know what happened */
if (data && typeof data.error === 'string') {
this.writeMessage(
$.i18n('api-error', pageLink + ': ' + data.error)
);
} else {
this.writeMessage($.i18n('api-error-unknown', pageLink));
}
});
}

processQuarry(cb) {
const project = $('.quarry-project').val(),
id = $(this.config.sourceInput).val();
Expand Down Expand Up @@ -946,12 +1133,36 @@ class MassViews extends mix(Pv).with(ChartHelpers, ListHelpers) {
this.renderData();
};

let project, target;

switch ($('#source_button').data('value')) {
case 'pagepile':
this.processPagePile(cb);
break;
case 'category':
this.processCategory(cb);
// parse input before calling processCategory, so we can query for siteinfo if needed
[project, target] = this.getWikiPageFromURL($(this.config.sourceInput).val());
if (!this.validateProject(project)) return;

// fetch siteinfo to get namespaces if they've opted to use subject page instead of talk
if ($('.category-subject-toggle--input').is(':checked')) {
this.getSiteInfo(project).then(() => {
this.processCategory(project, target, cb);
});
} else {
this.processCategory(project, target, cb);
}
break;
case 'subpages':
// parse input before calling processSubpages so we can query for siteinfo
[project, target] = this.getWikiPageFromURL($(this.config.sourceInput).val());
if (!this.validateProject(project)) return;

// fetch namespaces first
this.getSiteInfo(project).then(() => this.processSubpages(project, target, cb));
break;
case 'wikilinks':
this.processWikiPage(cb);
break;
case 'transclusions':
this.processTemplate(cb);
Expand Down
41 changes: 41 additions & 0 deletions javascripts/shared/pv.js
View file
Expand Up @@ -598,6 +598,47 @@ class Pv extends PvConfig {
}
}

/**
* Get general information about a project, such as namespaces, title of the main page, etc.
* Data returned by the api is also stored in this.siteInfo
* @param {String} project - project such as en.wikipedia (with or without .org)
* @returns {Deferred} promise resolving with siteinfo
*/
getSiteInfo(project) {
project = project.replace(/\.org$/, '');
const dfd = $.Deferred(),
cacheKey = `pageviews-siteinfo-${project}`;

// use cached site info if present
if (simpleStorage.hasKey(cacheKey)) {
this.siteInfo = simpleStorage.get(cacheKey);
dfd.resolve(this.siteInfo);
} else {
// otherwise fetch siteinfo and store in cache
$.ajax({
url: `https://${project}.org/w/api.php`,
data: {
action: 'query',
meta: 'siteinfo',
siprop: 'general|namespaces',
format: 'json'
},
dataType: 'jsonp'
}).done(data => {
this.siteInfo = data.query;

// cache for one week (TTL is in milliseconds)
simpleStorage.set(cacheKey, this.siteInfo, {TTL: 1000 * 60 * 60 * 24 * 7});

dfd.resolve(this.siteInfo);
}).fail(data => {
dfd.reject(data);
});
}

return dfd;
}

/**
* Get user agent, if supported
* @returns {string} user-agent
Expand Down

0 comments on commit b95f99c

Please sign in to comment.