Speed up wiki scraping (#42)

* Scrape 20 pages at a time

This significantly speeds up wiki scraping

* Wipe output folder by default

Is there a reason not to do this?

Author: robotboy655

package.json

@@ -20,7 +20,7 @@
   "scripts": {
     "clear-output": "rm -rf ./output/",
     "wiki-check-changed": "tsx ./src/cli-change-checker.ts",
-    "scrape-wiki": "tsx ./src/cli-scraper.ts --output ./output/ --customOverrides ./custom/",
+    "scrape-wiki": "tsx ./src/cli-scraper.ts --output ./output/ --customOverrides ./custom/ --wipe",
     "pack-release": "tsx ./src/cli-release-packer.ts --input ./output/ --output ./dist/release/",
     "publish-library": "tsx ./src/cli-library-publisher.ts --input ./output/ --output ./dist/libraries/garrysmod",
     "stylua-custom": "npx --yes @johnnymorganz/stylua-bin@0.17.1 ./custom",

src/cli-scraper.ts

@@ -82,6 +82,7 @@ async function startScrape() {
 
   const pageIndexes = await scrapeAndCollect(pageListScraper);
 
+  let queue: Promise<any>[] = [];
   for (const pageIndex of pageIndexes) {
     const pageMarkupScraper = new WikiPageMarkupScraper(`${baseUrl}/${pageIndex.address}?format=text`);
 
@@ -119,7 +120,13 @@ async function startScrape() {
       fs.writeFileSync(path.join(baseDirectory, moduleName, `${fileName}.json`), json);
     });
 
-    await pageMarkupScraper.scrape();
+    queue.push(pageMarkupScraper.scrape());
+
+    if (queue.length > 20)
+    {
+      await Promise.allSettled(queue);
+      queue = [];
+    }
   }
 
   console.log(`Done with scraping! You can find the output in ${baseDirectory}`);