[repo] Add bulk importer

Author: andrewnicols

package.json

@@ -84,6 +84,7 @@
     "lint-staged": "^12.3.7",
     "markdownlint-cli2": "^0.4.0",
     "markdownlint-rule-helpers": "^0.16.0",
+    "node-fetch": "^3.2.4",
     "nodemw": "^0.17.0",
     "stylelint": "^14.6.1",
     "stylelint-config-standard": "^25.0.0",

scripts/migration/phases/05-wikilinks/rule.js

@@ -23,6 +23,17 @@ const {
     getLineMetadata,
 } = require('markdownlint-rule-helpers');
 
+const path = require('path');
+
+const getFileName = (matches) => {
+    const [link] = matches.groups.link.split('|');
+    if (link.startsWith('File:')) {
+        return link.replace(/^File:/, '');
+    }
+
+    return null;
+};
+
 const getWikilinkLink = (matches) => {
     const [link] = matches.groups.link.split('|');
 
@@ -52,18 +63,30 @@ const getWikilinkDescription = (matches) => {
         .replaceAll('_', ' ');
 };
 
-const replaceWikiLinks = (line, lineNumber, onError) => {
+const getFixInfo = (matches, filename) => {
+    const imageFilename = getFileName(matches);
+    if (imageFilename) {
+        const description = matches.groups.link.split('|').pop().replace('File:', '');
+        const imageDirName = `./_${path.basename(filename.replace(/\.md.?/, '').replaceAll(' ', '_'))}`;
+        return `![${description}](${imageDirName}/${imageFilename.replaceAll(' ', '_')})`;
+    }
+
+    const description = getWikilinkDescription(matches);
+    const link = getWikilinkLink(matches);
+    return `[${description}](${link})`;
+};
+
+const replaceWikiLinks = (line, lineNumber, onError, filename) => {
     const linkSearch = /(?<wikilink>\[\[(?<link>[^\]]*)\]\])/g;
     let matches = linkSearch.exec(line);
     if (!matches) {
         return;
     }
     do {
         const foundWikilink = matches.groups.wikilink;
-        const link = getWikilinkLink(matches);
-        const description = getWikilinkDescription(matches);
+        const replacement = getFixInfo(matches, filename);
         const column = matches.index + 1;
-        const replacement = `[${description}](${link})`;
+
         const fixInfo = {
             editColumn: column,
             deleteCount: matches.groups.link.length + 4,
@@ -96,7 +119,7 @@ module.exports = {
             }
 
             const lineNumber = lineIndex + 1;
-            replaceWikiLinks(line, lineNumber, onError);
+            replaceWikiLinks(line, lineNumber, onError, params.name);
         });
     },
 };

scripts/utils.js

@@ -15,6 +15,7 @@
  * along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+/* eslint-disable import/no-extraneous-dependencies */
 const Bot = require('nodemw'); // cspell:ignore nodemw
 const path = require('path');
 const winston = require('winston');
@@ -43,6 +44,17 @@ const getFetchDoc = (logger) => (client) => (pageTitle) => new Promise((resolve,
     });
 });
 
+const getGetPagesByPrefix = (logger) => (client) => (prefix) => new Promise((resolve, reject) => {
+    logger.debug(`Fetching all articles with prefix ${prefix}`);
+    client.getPagesByPrefix(prefix, (err, data) => {
+        if (err) {
+            reject(err);
+        }
+
+        resolve(data);
+    });
+});
+
 const getGetMigratedPageIds = (logger) => (client) => async (pageTitle) => {
     const articleData = await getFetchDoc(logger)(client)(pageTitle);
 
@@ -124,6 +136,26 @@ const getGetPagesTranscluding = (logger) => (client) => (templateName) => new Pr
     });
 });
 
+const getGetImageInfo = (logger) => (client) => (filename) => new Promise((resolve, reject) => {
+    logger.info(`Fetching image info for ${filename}`);
+    client.getImageInfo(filename, (err, data) => {
+        if (err) {
+            reject(new Error(err));
+        }
+        resolve(data);
+    });
+});
+
+const getGetImagesFromArticle = (logger) => (client) => (title) => new Promise((resolve, reject) => {
+    logger.info(`Fetching images forpage ${title}`);
+    client.getImagesFromArticle(title, (err, data) => {
+        if (err) {
+            reject(new Error(err));
+        }
+        resolve(data);
+    });
+});
+
 const getLogIn = (logger) => (client) => async () => new Promise((resolve, reject) => {
     if (!process.env.WIKIMEDIA_SECRET) {
         reject(new Error('No credentials to log in with'));
@@ -350,12 +382,15 @@ module.exports = {
     getClient,
     getFetchDoc,
     getGetMigratedPageIds,
+    getGetImagesFromArticle,
+    getGetImageInfo,
     getGetPagesTranscluding,
     getLogIn,
     getLogger,
     getMigrationPagePath,
     getObsoletePagePath,
     getNormalizedPath,
+    getGetPagesByPrefix,
     getUpdateMigratedPages,
     getUpdateMigratedPagesProtection,
     guessSlug,