-
Notifications
You must be signed in to change notification settings - Fork 0
/
pinata.js
126 lines (112 loc) · 4.08 KB
/
pinata.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
const pinataSDK = require("@pinata/sdk");
const pinata = pinataSDK("xx", "xx");
const IPFSGatewayTools = require("@pinata/ipfs-gateway-tools/dist/node");
const gatewayTools = new IPFSGatewayTools();
const Downloader = require("nodejs-file-downloader");
const Queue = require("async-await-queue");
const hashArray = [""];
const run = async () => {
await downloadLinks();
// for await (let hashToPin of hashArray) {
// await pinata.pinByHash(hashToPin);
// await delay(200);
// console.log(hashToPin);
// }
};
const saveFileBackupOneAtATime = async () => {
const sourceUrl = "ipfs://" + hashToPin;
const desiredGatewayPrefix = "https://xxxxx.mypinata.cloud";
const convertedGatewayUrl = gatewayTools.convertToDesiredGateway(
sourceUrl,
desiredGatewayPrefix
);
console.log(convertedGatewayUrl);
const downloader = new Downloader({
url: convertedGatewayUrl, //If the file name already exists, a new file with the name 200MB1.zip is created.
directory: "./downloads", //This folder will be created, if it doesn't exist.
maxAttempts: 3, //Default is 1.
onProgress: function (percentage, chunk, remainingSize) {
//Gets called with each chunk.
console.log("% ", percentage);
console.log("Current chunk of data: ", chunk);
console.log("Remaining bytes: ", remainingSize);
},
});
try {
await downloader.download();
console.log("All done");
} catch (error) {
//IMPORTANT: Handle a possible error. An error is thrown in case of network errors, or status codes of 400 and above.
//Note that if the maxAttempts is set to higher than 1, the error is thrown only if all attempts fail.
console.log("Download failed", error);
}
};
/** Below is a faster download mechanism */
const saveFilesInParallel = async () => {
//split into chuncks
const myq = new Queue(10, 200);
const myPriority = -1;
//Note check below and modify accordingly to get the array that u want
//let difference = hashArray.filter(x => !hashAlreadyDownloaded.includes(x));
const q = [];
for (let hashToPin of hashArray) {
const sourceUrl = "ipfs://" + hashToPin;
const desiredGatewayPrefix = "https://xxxxxx.mypinata.cloud";
const convertedGatewayUrl = gatewayTools.convertToDesiredGateway(
sourceUrl,
desiredGatewayPrefix
);
console.log(convertedGatewayUrl);
q.push(
myq.run(() =>
downloadTheKittens(convertedGatewayUrl).catch((e) => console.error(e))
)
);
}
return Promise.all(q);
};
async function downloadTheKittens(convertedGatewayUrl) {
const downloader = new Downloader({
url: convertedGatewayUrl, //If the file name already exists, a new file with the name 200MB1.zip is created.
directory: "./downloads", //This folder will be created, if it doesn't exist.
maxAttempts: 3, //Default is 1.
onProgress: function (percentage, chunk, remainingSize) {
//Gets called with each chunk.
console.log("% ", percentage);
console.log("Current chunk of data: ", chunk);
console.log("Remaining bytes: ", remainingSize);
},
});
try {
await downloader.download(); //Downloader.download() returns a promise.
console.log("All done");
} catch (error) {
//IMPORTANT: Handle a possible error. An error is thrown in case of network errors, or status codes of 400 and above.
//Note that if the maxAttempts is set to higher than 1, the error is thrown only if all attempts fail.
console.log("Download failed", error);
}
}
/**
* Note that in this function one has to save it into a file and process to get the data right
*/
const downloadLinks = async () => {
const ipfs = await IPFS.create(
"https://XXXX:YYYYY@ipfs.infura.io:5001/api/v0"
);
for (let i = 0; i < 10000; i++) {
const results = await ipfs.get("QmaASDASDSADASDASDASDASDSA/" + i, {
compress: false,
});
delay(1000);
for await (let file of results) {
const data = Buffer.from(file).toString();
if (data.indexOf("image")) {
console.log(data);
}
}
}
};
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
run();