feat: scrape archwings and companions from wikia (#506)

build/scraper.mjs

@@ -5,6 +5,8 @@ import { Generator as RelicGenerator } from '@wfcd/relics';
 import patchlogs from 'warframe-patchlogs';
 
 import Progress from './progress.mjs';
+import ArchwingScraper from './wikia/scrapers/ArchwingScraper.mjs';
+import CompanionScraper from './wikia/scrapers/CompanionScraper.mjs';
 import ModScraper from './wikia/scrapers/ModScraper.mjs';
 import WeaponScraper from './wikia/scrapers/WeaponScraper.mjs';
 import WarframeScraper from './wikia/scrapers/WarframeScraper.mjs';
@@ -195,7 +197,7 @@ class Scraper {
    * @returns {WikiaData}
    */
   async fetchWikiaData() {
-    const bar = new Progress('Fetching Wikia Data', 5);
+    const bar = new Progress('Fetching Wikia Data', 7);
     const ducats = [];
     const ducatsWikia = await get('http://warframe.wikia.com/wiki/Ducats/Prices/All', true);
     const $ = load(ducatsWikia);
@@ -215,13 +217,19 @@ class Scraper {
     bar.tick();
     const versions = await new VersionScraper().scrape();
     bar.tick();
+    const archwings = await new ArchwingScraper().scrape();
+    bar.tick();
+    const companions = await new CompanionScraper().scrape();
+    bar.tick();
 
     return {
       weapons,
       warframes,
       mods,
       versions,
       ducats,
+      archwings,
+      companions,
     };
   }
 

build/wikia/scrapers/ArchwingScraper.mjs

@@ -0,0 +1,8 @@
+import WikiaDataScraper from '../WikiaDataScraper.mjs';
+import transformWarframe from '../transformers/transformWarframe.mjs';
+
+export default class ArchwingScraper extends WikiaDataScraper {
+  constructor() {
+    super('https://warframe.fandom.com/wiki/Module:Warframes/data?action=edit', 'Archwing', transformWarframe);
+  }
+}

build/wikia/scrapers/CompanionScraper.mjs

@@ -0,0 +1,8 @@
+import WikiaDataScraper from '../WikiaDataScraper.mjs';
+import transformCompanion from '../transformers/transformCompanion.mjs';
+
+export default class CompanionScraper extends WikiaDataScraper {
+  constructor() {
+    super('https://warframe.fandom.com/wiki/Module:Companions/data?action=edit', 'Companion', transformCompanion);
+  }
+}

build/wikia/transformers/transformCompanion.mjs

@@ -0,0 +1,36 @@
+/**
+ * Transform wikia lua companions into usable standardized json
+ * @param {Object} oldCompanion old companion in lua format
+ * @param {Record<string, unknown>} imageUrls name-url pairs
+ * @returns {Promise<WikiaCompanion>}
+ */
+export default async (oldCompanion, imageUrls) => {
+  let newCompanion;
+  if (!oldCompanion || !oldCompanion.Name) {
+    return undefined;
+  }
+
+  try {
+    const { Image, Mastery, Polarities, Stamina, Introduced, Vaulted, VaultDate, EstimatedVaultDate } = oldCompanion;
+    const { Name } = oldCompanion;
+
+    newCompanion = {
+      name: Name,
+      url: `https://warframe.fandom.com/wiki/${encodeURIComponent(
+        Name.replace(/\s/g, '_').replace('_Prime', '/Prime')
+      )}`,
+      mr: Mastery || 0,
+      polarities: Polarities,
+      stamina: Stamina,
+      introduced: Introduced,
+      vaulted: Vaulted || undefined,
+      vaultDate: VaultDate,
+      estimatedVaultDate: EstimatedVaultDate,
+      thumbnail: imageUrls?.[Image],
+    };
+  } catch (error) {
+    console.error(`Error parsing ${oldCompanion.Name}`);
+    console.error(error);
+  }
+  return newCompanion;
+};

build/wikia/transformers/transformerArchwing.mjs

@@ -0,0 +1,37 @@
+import transformPolarity from './transformPolarity.mjs';
+
+/**
+ * Transform wikia lua archwings into usable standardized json
+ * @param {Object} oldWings old archwing in lua format
+ * @param {Record<string, unknown>} imageUrls name-url pairs
+ * @returns {Promise<WikiaArchwing>}
+ */
+export default async (oldWings, imageUrls) => {
+  let newWings;
+  if (!oldWings || !oldWings.Name) {
+    return undefined;
+  }
+
+  try {
+    const { Image, Mastery, Polarities, Sprint, Introduced, Vaulted } = oldWings;
+    const { Name } = oldWings;
+
+    newWings = {
+      name: Name,
+      url: `https://warframe.fandom.com/wiki/${encodeURIComponent(
+        Name.replace(/\s/g, '_').replace('_Prime', '/Prime')
+      )}`,
+      mr: Mastery || 0,
+      polarities: Polarities,
+      sprint: Sprint,
+      introduced: Introduced,
+      vaulted: Vaulted || undefined,
+      thumbnail: imageUrls?.[Image],
+    };
+    newWings = transformPolarity(oldWings, newWings);
+  } catch (error) {
+    console.error(`Error parsing ${oldWings.Name}`);
+    console.error(error);
+  }
+  return newWings;
+};