Subtitle Downloading

package.json

@@ -96,7 +96,7 @@
     "semver": "^5.6.0",
     "service-mobileapp-node": "git+https://github.com/wikimedia/mediawiki-services-mobileapps.git#d244439",
     "service-runner": "^2.7.2",
-    "sharp": "^0.25.2",
+    "sharp": "^0.25.3",
     "swig-templates": "^2.0.2",
     "tslint": "^5.20.1",
     "typescript": "^3.8.3",

src/S3.ts

@@ -28,7 +28,6 @@ class S3 {
                 return true;
             }
         } catch (err) {
-            logger.log(err);
             throw new Error(`Unable to connect to S3, either S3 login credentials are wrong or bucket cannot be found
                             Bucket used: ${this.bucketName}
                             End point used: ${s3UrlBase.href}

src/util/misc.ts

@@ -311,7 +311,23 @@ export function deDup<T>(_arr: T[], getter: (o: T) => any) {
   });
 }
 
-export function getRelativeFilePath(parentArticleId: string, fileBase: string, resourceNamespace: 'I' | 'A' | 'M') {
+let lastCalled = 0;
+export function throttle(fn: (...args: any[]) => any, wait: number) {
+
+  return function (...args: any[]) {
+    const canCall = (Date.now() - lastCalled) >= wait;
+    if (canCall) {
+      fn(...args);
+      lastCalled = Date.now();
+    }
+  };
+}
+
+export const keepAlive = throttle(function keepAlive() {
+  logger.log(`Heartbeat - OK`);
+}, 1000 * 60 * 9);
+
+export function getRelativeFilePath(parentArticleId: string, fileBase: string, resourceNamespace: 'I' | 'A' | 'M'| '-') {
   const slashesInUrl = parentArticleId.split('/').length - 1;
   const upStr = '../'.repeat(slashesInUrl + 1);
   const newUrl = `${upStr}${resourceNamespace}/` + fileBase;

src/util/saveArticles.ts

@@ -1,22 +1,23 @@
 import logger from '../Logger';
 import Downloader from '../Downloader';
 import MediaWiki from '../MediaWiki';
-import {ZimArticle, ZimCreator} from '@openzim/libzim';
+import { ZimArticle, ZimCreator } from '@openzim/libzim';
 import htmlMinifier from 'html-minifier';
 import * as urlParser from 'url';
+import * as QueryStringParser from 'querystring';
 
 import DU from '../DOMUtils';
 import * as domino from 'domino';
-import {Dump} from '../Dump';
-import {mapLimit} from 'promiso';
-import {contains, genCanonicalLink, genHeaderCSSLink, genHeaderScript, getFullUrl, getMediaBase, jsPath} from '.';
-import {config} from '../config';
-import {footerTemplate, htmlTemplateCode} from '../Templates';
-import {articleDetailXId, filesToDownloadXPath, filesToRetryXPath} from '../stores';
-import {getRelativeFilePath, getSizeFromUrl, encodeArticleIdForZimHtmlUrl, interpolateTranslationString} from './misc';
-import {RedisKvs} from './RedisKvs';
-import {rewriteUrl} from './rewriteUrls';
-import {CONCURRENCY_LIMIT} from './const';
+import { Dump } from '../Dump';
+import { mapLimit } from 'promiso';
+import { contains, genCanonicalLink, genHeaderCSSLink, genHeaderScript, getFullUrl, getMediaBase, jsPath } from '.';
+import { config } from '../config';
+import { footerTemplate, htmlTemplateCode } from '../Templates';
+import { articleDetailXId, filesToDownloadXPath, filesToRetryXPath } from '../stores';
+import { getRelativeFilePath, getSizeFromUrl, encodeArticleIdForZimHtmlUrl, interpolateTranslationString } from './misc';
+import { RedisKvs } from './RedisKvs';
+import { rewriteUrl } from './rewriteUrls';
+import { CONCURRENCY_LIMIT } from './const';
 
 const genericJsModules = config.output.mw.js;
 const genericCssModules = config.output.mw.css;
@@ -111,6 +112,7 @@ async function downloadBulk(listOfArguments: any[], downloader: Downloader): Pro
                 resp.namespace = arg.val.namespace;
                 resp.mult = arg.val.mult;
                 resp.width = arg.val.width;
+
                 return downloader.downloadContent(arg.val.url).then((r) => {
                     resp.result = r;
                     return resp;
@@ -148,7 +150,7 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
                             continue;
                         }
 
-                        const { articleDoc: _articleDoc, mediaDependencies } = await processArticleHtml(articleHtml, downloader, mw, dump, articleId);
+                        const { articleDoc: _articleDoc, mediaDependencies, subtitles } = await processArticleHtml(articleHtml, downloader, mw, dump, articleId);
                         let articleDoc = _articleDoc;
 
                         if (dump.customProcessor?.shouldKeepArticle) {
@@ -172,6 +174,10 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
                             }
                         }
 
+                        for (const subtitle of subtitles) {
+                            await filesToDownloadXPath.set(subtitle.path, { url: subtitle.url, namespace: '-' });
+                        }
+
                         const _moduleDependencies = await getModuleDependencies(nonPaginatedArticleId, mw, downloader);
 
                         for (const dep of _moduleDependencies.jsDependenciesList) {
@@ -293,9 +299,11 @@ async function getModuleDependencies(articleId: string, mw: MediaWiki, downloade
 
 async function processArticleHtml(html: string, downloader: Downloader, mw: MediaWiki, dump: Dump, articleId: string) {
     let mediaDependencies: Array<{ url: string, path: string }> = [];
+    let subtitles: Array<{ url: string, path: string }> = [];
     let doc = domino.createDocument(html);
     const tmRet = await treatMedias(doc, mw, dump, articleId);
     doc = tmRet.doc;
+
     mediaDependencies = mediaDependencies.concat(
         tmRet.mediaDependencies
             .filter((a) => a)
@@ -305,6 +313,15 @@ async function processArticleHtml(html: string, downloader: Downloader, mw: Medi
             }),
     );
 
+    subtitles = subtitles.concat(
+        tmRet.subtitles
+            .filter((a) => a)
+            .map((url) => {
+                const { title, lang } = QueryStringParser.parse(url) as { title: string, lang: string };
+                const path = `${title}-${lang}.vtt`;
+                return { url, path };
+            }),
+    );
     const ruRet = await rewriteUrls(doc, articleId, downloader, mw, dump);
     doc = ruRet.doc;
     mediaDependencies = mediaDependencies.concat(
@@ -319,6 +336,7 @@ async function processArticleHtml(html: string, downloader: Downloader, mw: Medi
     return {
         articleDoc: doc,
         mediaDependencies,
+        subtitles
     };
 }
 
@@ -336,10 +354,11 @@ function widthXHeightSorter(a: DominoElement, b: DominoElement) {
     return aVal > bVal ? 1 : -1;
 }
 
-async function treatVideo(mw: MediaWiki, dump: Dump, srcCache: KVS<boolean>, articleId: string, videoEl: DominoElement): Promise<{ mediaDependencies: string[] }> {
+export async function treatVideo(mw: MediaWiki, dump: Dump, srcCache: KVS<boolean>, articleId: string, videoEl: DominoElement): Promise<{ mediaDependencies: string[], subtitles: string[] }> {
     // This function handles audio tags as well as video tags
     const webUrlHost = urlParser.parse(mw.webUrl).host;
     const mediaDependencies: string[] = [];
+    const subtitles: string[] = [];
     // Worth noting:
     // Video tags are used for audio files too (as opposed to the audio tag)
     // When it's only audio, there will be a single OGG file
@@ -358,7 +377,7 @@ async function treatVideo(mw: MediaWiki, dump: Dump, srcCache: KVS<boolean>, art
 
     if (dump.nopic || dump.novid || dump.nodet) {
         DU.deleteNode(videoEl);
-        return { mediaDependencies };
+        return { mediaDependencies, subtitles };
     }
 
     videoSources = videoSources.sort(widthXHeightSorter);
@@ -402,7 +421,23 @@ async function treatVideo(mw: MediaWiki, dump: Dump, srcCache: KVS<boolean>, art
     }
 
     sourceEl.setAttribute('src', newUrl);
-    return { mediaDependencies };
+
+    /* Scrape subtitle */
+    for (const track of Array.from(videoEl.querySelectorAll('track'))) {
+        subtitles.push(await treatSubtitle(track, webUrlHost, mw, articleId));
+    }
+
+    return { mediaDependencies, subtitles };
+}
+
+export async function treatSubtitle(trackEle: DominoElement, webUrlHost: string, mw: MediaWiki, articleId: string): Promise<string> {
+    const subtitleSourceUrl = getFullUrl(webUrlHost, trackEle.getAttribute('src'), mw.base);
+    const { title, lang } = QueryStringParser.parse(subtitleSourceUrl) as { title: string, lang: string };
+    // The source URL we get from Mediawiki article is in srt format, so we replace it to vtt which is standard subtitle trackformat for <track> src attribute.
+    const vttFormatUrl =  new URL(subtitleSourceUrl);
+    vttFormatUrl.searchParams.set('trackformat', 'vtt');
+    trackEle.setAttribute('src', `${getRelativeFilePath(articleId, title, '-')}-${lang}.vtt`);
+    return vttFormatUrl.href;
 }
 
 function shouldKeepImage(dump: Dump, img: DominoElement) {
@@ -538,6 +573,7 @@ function treatImageFrames(mw: MediaWiki, dump: Dump, parsoidDoc: DominoElement,
 
 export async function treatMedias(parsoidDoc: DominoElement, mw: MediaWiki, dump: Dump, articleId: string) {
     let mediaDependencies: string[] = [];
+    let subtitles: string[] = [];
     /* Clean/rewrite image tags */
     const imgs = Array.from(parsoidDoc.getElementsByTagName('img'));
     const videos: DominoElement = Array.from(parsoidDoc.querySelectorAll('video, audio'));
@@ -546,6 +582,7 @@ export async function treatMedias(parsoidDoc: DominoElement, mw: MediaWiki, dump
     for (const videoEl of videos) { // <video /> and <audio />
         const ret = await treatVideo(mw, dump, srcCache, articleId, videoEl);
         mediaDependencies = mediaDependencies.concat(ret.mediaDependencies);
+        subtitles = subtitles.concat(ret.subtitles);
     }
 
     for (const imgEl of imgs) {
@@ -561,7 +598,7 @@ export async function treatMedias(parsoidDoc: DominoElement, mw: MediaWiki, dump
         treatImageFrames(mw, dump, parsoidDoc, imageNode);
     }
 
-    return { doc: parsoidDoc, mediaDependencies };
+    return { doc: parsoidDoc, mediaDependencies, subtitles };
 }
 
 async function rewriteUrls(parsoidDoc: DominoElement, articleId: string, downloader: Downloader, mw: MediaWiki, dump: Dump) {

test/unit/saveArticles.test.ts

@@ -2,9 +2,9 @@ import './bootstrap.test';
 import test from 'blue-tape';
 import domino from 'domino';
 
-import { setupScrapeClasses } from 'test/util';
+import { setupScrapeClasses, convertWikicodeToHtml, testHtmlRewritingE2e } from 'test/util';
 import { articleDetailXId } from 'src/stores';
-import { saveArticles, treatMedias, applyOtherTreatments } from '../../src/util/saveArticles';
+import { saveArticles, treatMedias, applyOtherTreatments, treatSubtitle, treatVideo } from 'src/util/saveArticles';
 import { ZimArticle } from '@openzim/libzim';
 import { Dump } from 'src/Dump';
 import { mwRetToArticleDetail } from 'src/util';
@@ -238,3 +238,36 @@ test('--customFlavour', async (t) => {
     t.ok(ParisDocument.querySelector('#PRE_PROCESSOR'), `Paris was correctly pre-processed`);
     t.ok(PragueDocument.querySelector('#POST_PROCESSOR'), `Prague was correctly post-processed`);
 });
+
+test('treat one subtitle', async(t) => {
+    const { downloader, mw, dump } = await setupScrapeClasses({ format: '' });
+
+    // Wikicode is taken from article "Mechanical energy" which has a video with subtitle
+    const wikicode = `[[File:Physicsworks.ogv|thumb|200px|alt="Lecture demonstrating conservation of mechanical energy"|MIT professor [[Walter Lewin]] demonstrating conservation of mechanical energy]]`;
+    const htmlStr = await convertWikicodeToHtml(wikicode, dump.mwMetaData.base);
+
+    const htmlDoc = domino.createDocument(htmlStr.data);
+    const contentRes = await treatSubtitle(htmlDoc.querySelector('track'), 'en.wikipedia.org', mw, 'Mechanical energy');
+    testHtmlRewritingE2e(t, wikicode, htmlStr.data, 'Converted wikicode to HTML for one subtitle');
+    t.equals(contentRes, 'https://commons.wikimedia.org/w/api.php?action=timedtext&title=File%3APhysicsworks.ogv&lang=en&trackformat=vtt&origin=*', 'Video subtitle rewriting matches');
+});
+
+test('treat multiple subtitles in one video', async(t) => {
+    const { downloader, mw, dump } = await setupScrapeClasses({ format: '' });
+
+    // Wikicode is taken from article "User:Charliechlorine/sandbox" which has multiple(4) subtitles in this video
+    const wikicode = `[[File:Videoonwikipedia.ogv|thumb|thumbtime=0:58|left|320px|Video about kola nuts ]]`;
+    const htmlStr = await convertWikicodeToHtml(wikicode, dump.mwMetaData.base);
+
+    const htmlDoc = domino.createDocument(htmlStr.data);
+    const contentRes = await treatVideo(mw, dump, {}, 'User:Charliechlorine/sandbox', htmlDoc.querySelector('video'));
+    testHtmlRewritingE2e(t, wikicode, htmlStr.data, 'Converted wikicode to HTML for multiple subtitle');
+    t.deepEqual(
+        contentRes.subtitles,
+        [ 'https://commons.wikimedia.org/w/api.php?action=timedtext&title=File%3AVideoonwikipedia.ogv&lang=en&trackformat=vtt&origin=*',
+        'https://commons.wikimedia.org/w/api.php?action=timedtext&title=File%3AVideoonwikipedia.ogv&lang=eu&trackformat=vtt&origin=*',
+        'https://commons.wikimedia.org/w/api.php?action=timedtext&title=File%3AVideoonwikipedia.ogv&lang=fr&trackformat=vtt&origin=*',
+        'https://commons.wikimedia.org/w/api.php?action=timedtext&title=File%3AVideoonwikipedia.ogv&lang=sv&trackformat=vtt&origin=*' ],
+        'Video multiple subtitles rewriting matches');
+    t.equals(contentRes.subtitles.length, 4, 'All subtitles are found for this video');
+});

test/unit/util.test.ts

@@ -1,6 +1,7 @@
 import './bootstrap.test';
 import test from 'blue-tape';
 import { encodeArticleIdForZimHtmlUrl, interpolateTranslationString } from 'src/util';
+import { testHtmlRewritingE2e } from 'test/util';
 
 
 test('util -> interpolateTranslationString', async (t) => {
@@ -33,3 +34,11 @@ test('Encoding ArticleId for Zim HTML Url', async(t) => {
         t.equal(encoded, encodeArticleIdForZimHtmlUrl(unencoded), `encodeArticleIdForZimHtmlUrl() encoding`);
     }
 });
+
+test('wikitext comparison', async(t) => {
+    testHtmlRewritingE2e(
+        t,
+        `An [[isolated system]] remains the system is free.`,
+        `<p id="mwAQ">An <a rel="mw:WikiLink" href="./Isolated_system" title="Isolated system" id="mwAg">isolated system</a> remains the system is free.</p>`,
+        'HTML and Wikitext match')
+})

test/util.ts

@@ -1,7 +1,9 @@
 import MediaWiki from '../src/MediaWiki';
 import Downloader from '../src/Downloader';
 import { Dump } from '../src/Dump';
+import axios from 'axios';
 import execa = require('execa');
+import logger from '../src/Logger';
 
 export function leftPad(_num: number, length: number) {
     const num = `${_num}`;
@@ -64,3 +66,20 @@ export async function zimcheckAvailable() {
 export async function zimcheck(filePath: string) {
     return execa.command(`${zimcheckPath} ${filePath}`);
 }
+
+export async function convertWikicodeToHtml(wikicode: string, baseUrl: string): Promise<any> {
+    try {
+        return await axios.post(`${baseUrl}api/rest_v1/transform/wikitext/to/html`,  {
+            wikitext: wikicode,
+            body_only: true,
+        })
+    } catch (err){
+        logger.log(`Got error during conversion of wikicode to HTML due to ${err}`);
+        return err;
+    }
+}
+
+export async function testHtmlRewritingE2e(t: any, wikicode: string, html: string, comment: string) {
+    const resultHtml = await convertWikicodeToHtml(wikicode, 'https://en.wikipedia.org/');
+    t.equal(html, resultHtml.data, comment);
+}