diff --git a/package.json b/package.json index 8d2038b..383d27d 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "scripts": { "clear-output": "rm -rf ./output/", "wiki-check-changed": "tsx ./src/cli-change-checker.ts", - "scrape-wiki": "tsx ./src/cli-scraper.ts --output ./output/ --customOverrides ./custom/", + "scrape-wiki": "tsx ./src/cli-scraper.ts --output ./output/ --customOverrides ./custom/ --wipe", "pack-release": "tsx ./src/cli-release-packer.ts --input ./output/ --output ./dist/release/", "publish-library": "tsx ./src/cli-library-publisher.ts --input ./output/ --output ./dist/libraries/garrysmod", "stylua-custom": "npx --yes @johnnymorganz/stylua-bin@0.17.1 ./custom", diff --git a/src/cli-scraper.ts b/src/cli-scraper.ts index f1243f9..f3eec80 100644 --- a/src/cli-scraper.ts +++ b/src/cli-scraper.ts @@ -82,6 +82,7 @@ async function startScrape() { const pageIndexes = await scrapeAndCollect(pageListScraper); + let queue: Promise[] = []; for (const pageIndex of pageIndexes) { const pageMarkupScraper = new WikiPageMarkupScraper(`${baseUrl}/${pageIndex.address}?format=text`); @@ -119,7 +120,13 @@ async function startScrape() { fs.writeFileSync(path.join(baseDirectory, moduleName, `${fileName}.json`), json); }); - await pageMarkupScraper.scrape(); + queue.push(pageMarkupScraper.scrape()); + + if (queue.length > 20) + { + await Promise.allSettled(queue); + queue = []; + } } console.log(`Done with scraping! You can find the output in ${baseDirectory}`);