/
index.js
101 lines (83 loc) · 2.49 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
import fs from 'fs';
import { Pool } from 'multiprocessing';
const WORKER_COUNT = 64;
const WORKERS = new Pool(WORKER_COUNT);
const DIR_NAME = 'metadata';
const DIR_FILE_EXT = '';
const FILE_NAME = 'metadata_summary.json';
const BASE_URI = "https://divineanarchy.mypinata.cloud/ipfs/Qmaq9VcnrGQiBmGww1aWd6jLYbsgnczJQZdavyEku75LAE";
const START_ID = 1;
const END_ID = 5050;
async function main() {
console.time();
if(!BASE_URI) {
console.log('Please pass an base uri when running this script after execution');
process.exit(1);
}
let urls = [];
for(let i = START_ID; i <= END_ID; i++) {
urls.push(`${BASE_URI}/${i}`);
}
// Create DIR to hold all metadata files
if(fs.existsSync(DIR_NAME)) {
fs.rmdirSync(DIR_NAME, { recursive: true });
}
fs.mkdirSync(DIR_NAME);
let worker_promises = [];
let worker_results = [];
const chunk_amount = Math.ceil(urls.length / WORKER_COUNT);
let chunk_count = 0;
for(let thread = 1; thread <= WORKER_COUNT; thread++) {
const chunk_urls = urls.slice(chunk_count, chunk_count + chunk_amount);
chunk_count += chunk_amount;
const promise = WORKERS.apply([chunk_urls, DIR_NAME, DIR_FILE_EXT], getMetadata, {
onResult: result => {
if(result instanceof Array) {
worker_results = worker_results.concat(result);
}
}
});
worker_promises.push(promise);
}
await Promise.all(worker_promises);
if(fs.existsSync(FILE_NAME)) {
fs.rmSync(FILE_NAME);
}
console.log('Writing to file...');
fs.writeFileSync(FILE_NAME, JSON.stringify(worker_results, null, 2));
console.timeEnd();
}
async function getMetadata(args) {
const axios = require('axios');
const child_fs = require('fs');
const [ chunk_urls, dir, ext ] = args;
console.log(`${process.pid} - Starting thread for: ${chunk_urls.length} requests`);
let metadata = [];
for(const index in chunk_urls) {
const loadMetadata = async () => {
const url = chunk_urls[index];
const id = url.split('/').pop();
const { data } = await axios.get(url, { timeout: 15000 });
child_fs.writeFileSync(`${dir}/${id}${ext}`, JSON.stringify(data, null, 2));
metadata.push(data);
}
try {
await loadMetadata();
} catch(err) {
console.log('Shit broke, but we will try again just incase');
try {
await loadMetadata();
} catch(err) {
console.log(err.message);
}
}
}
console.log(`Finished with pid: ${process.pid}`);
return metadata;
}
main().then(() => {
process.exit();
}).catch(err => {
console.log(err)
process.exit();
});