it works again

config.default.yaml

@@ -10,3 +10,4 @@ globalBlacklistNoticeShown: false
 analytics: "PROMPT"
 analyticsId: null
 blacklistedTags: []
+threads: 3

package.json

@@ -135,7 +135,7 @@
 		"@typescript-eslint/eslint-plugin": "^2.34.0",
 		"@typescript-eslint/parser": "^2.34.0",
 		"dotenv": "^8.2.0",
-		"electron": "10.1.2",
+		"electron": "^12.0.0-beta.6",
 		"eslint": "^7.9.0",
 		"eslint-plugin-import": "^2.22.0",
 		"typescript": "^4.2.0-dev.20201109"
@@ -145,6 +145,7 @@
 		"arg": "^5.0.0",
 		"commander": "^6.2.0",
 		"deasync": "^0.1.21",
+		"e621downloader.js": "^1.0.7",
 		"electron-squirrel-startup": "^1.0.0",
 		"electron-window-state": "^5.0.3",
 		"express": "^4.17.1",

src/Analytics.ts

@@ -27,7 +27,7 @@ export default class Analytics {
 			const req = https
 				.request({
 					method: "POST",
-					hostname: "e621downloader.furrybot.co",
+					hostname: "e621.download",
 					path: "/track",
 					port: 443,
 					protocol: "https:",

src/ConfigManager.ts

@@ -24,6 +24,7 @@ export interface ConfigProperties {
 	blacklistedTags: string[];
 	analyticsId: null | string;
 	analytics: boolean;
+	threads: 1 | 2 | 3;
 }
 
 export default class ConfigManager {

src/Downloader.ts

@@ -0,0 +1,41 @@
+import Electron from "electron";
+import E621Downloader from "e621downloader.js";
+import ConfigManager from "./ConfigManager";
+
+export default class Downloader {
+	static CURRENT: {
+		instance: E621Downloader;
+		ipc: Electron.IpcMainEvent;
+	} | null = null;
+	static ACTIVE = false;
+	static async start(tags: string[], folder: string, ipc: Electron.IpcMainEvent) {
+		this.ACTIVE = true;
+		const cnf = ConfigManager.get();
+		const { instance: c } = this.CURRENT = {
+			instance: new E621Downloader({
+				cacheFile: `${cnf.saveDirectory}/cache.json`,
+				auth: cnf.username && cnf.key ? {
+					username: cnf.username,
+					apiKey: cnf.key
+				} : null,
+				...cnf
+			}),
+			ipc
+		};
+
+		c
+			.on("error", ipc.reply.bind(ipc, "message", "error"))
+			.on("ready", ipc.reply.bind(ipc, "message", "ready"))
+			.on("start-recieved", ipc.reply.bind(ipc, "message", "start-recieved"))
+			.on("thread-done", ipc.reply.bind(ipc, "message", "thread-done"))
+			.on("post-finish", ipc.reply.bind(ipc, "message", "post-finish"))
+			.on("skip", ipc.reply.bind(ipc, "message", "skip"))
+			.on("download-done", ipc.reply.bind(ipc, "message", "download-done"))
+			.on("fetch-page", ipc.reply.bind(ipc, "message", "fetch-page"))
+			.on("fetch-finish", ipc.reply.bind(ipc, "message", "fetch-finish"))
+			.on("download-start", ipc.reply.bind(ipc, "message", "download-start"))
+			.on("thread-spawn", ipc.reply.bind(ipc, "message", "thread-spawn"))
+			.on("download-done", () => (this.CURRENT = null, this.ACTIVE = false)) // a kind of reset
+			.startDownload(tags, folder, cnf.threads);
+	}
+}

src/Utility.ts

@@ -9,6 +9,7 @@ import { performance } from "perf_hooks";
 import { BrowserWindow, dialog, ipcMain, shell } from "electron";
 import Logger from "./Logger";
 import Analytics from "./Analytics";
+import Downloader from "./Downloader";
 
 interface MsResponse {
 	ms: number;
@@ -272,11 +273,12 @@ export default class Utility {
 		const c = ConfigManager.get();
 		const dir = path.resolve(`${c.saveDirectory}/${folder.trim()}`);
 		if (!fs.existsSync(dir)) fs.mkdirpSync(dir);
-		let auth: string | null = null;
-		if (c.username && c.key) auth = Buffer.from(`${c.username}:${c.key}`).toString("base64");
-		const posts = await Utility.getPosts(tags, auth, 1, null, ev);
-		let i = 0;
 		ev.reply("debug", "dir", dir);
+
+		Downloader.start(tags, folder, ev);
+
+		/* const posts = await Utility.getPosts(tags, auth, 1, null, ev);
+		let i = 0;
 		const cache: {
 			[k: string]: number[];
 		} = c.useCache ? fs.existsSync(`${c.saveDirectory}/cache.json`) ? JSON.parse(fs.readFileSync(`${c.saveDirectory}/cache.json`).toString()) : {} : {};
@@ -307,7 +309,7 @@ export default class Utility {
 		if (c.useCache) fs.writeFileSync(`${c.saveDirectory}/cache.json`, JSON.stringify(cache, null, "\t"));
 		if (fs.readdirSync(dir).length === 0) fs.unlinkSync(dir);
 		window.setProgressBar(-1);
-		ev.reply("debug", "end", tags, posts.length, parseFloat((end - start).toFixed(3)), this.ms(parseFloat((end - start).toFixed(3)), true, true));
+		ev.reply("debug", "end", tags, posts.length, parseFloat((end - start).toFixed(3)), this.ms(parseFloat((end - start).toFixed(3)), true, true)); */
 	}
 
 	static mergeObjects<A = object, B = object>(a: A, b: B) {
@@ -346,7 +348,7 @@ export default class Utility {
 		}>((a, b) => {
 			https.request({
 				method: "GET",
-				hostname: "e621downloader.furrybot.co",
+				hostname: "e621.download",
 				path: `/updates?version=v${pkg.version}`,
 				port: 443,
 				protocol: "https:",
@@ -469,18 +471,4 @@ export default class Utility {
 		if (fs.existsSync(ConfigManager.DEFAULT_FILE)) fs.unlinkSync(ConfigManager.DEFAULT_FILE);
 		return ConfigManager.getDefaults();
 	}
-
-	static cliMode(window: BrowserWindow, opts: {
-		[k: string]: any;
-	}, debug: boolean) {
-		Analytics.track("cli-mode");
-		const t = opts.tags?.split(" ");
-		if (!t) throw new TypeError("You must provide at least one tag via the \"--tags \"<tags\">\" option.");
-		const f = (opts.folderName || t[0]).replace(/[^a-z0-9_\-]/gi, "_").replace(/_{2,}/g, "_").toLowerCase();
-		window.webContents.send("cli-start", t, f, debug);
-		ipcMain.on("cli-end", (ev) => {
-			Logger.debug("CLIMode", "Finished.");
-			process.exit(0);
-		});
-	}
 }

src/assets/js/main.js

@@ -123,7 +123,64 @@ async function start(tags, folder) {
 	ipcRenderer.send("start", tags, folder);
 	window.active = true;
 	const l = (ev, type, ...args) => {
-		if (window.debugLog) console.debug(type, ...args);
+		/* if (window.debugLog)  */console.debug(type, ...args);
+		switch (type) {
+			case "download-start": {
+				const [tags, folder, dir, threads, usingAuth] = args;
+				createLogEntry(`Using ${threads} threads for downloads.`, "info");
+				createLogEntry(`Set download directory to "${dir}"`, "info");
+				return createLogEntry(`Starting a search with the tags "${tags.join(" ")}" (using auth: ${usingAuth ? "YES" : "NO"})`, "info");
+				break;
+			}
+
+			case "thread-spawn": {
+				const [id, workerId] = args;
+				return createLogEntry(`Spawned thread #${id + 1}`, "info");
+				break;
+			}
+
+			case "ready": {
+				const [id, workerId] = args;
+				return createLogEntry(`Thread #${id + 1} is ready.`, "info");
+			}
+
+			case "fetch-page": {
+				const [page, count, time] = args;
+				return createLogEntry(`Got ${count} posts on page #${page} (${ms(time)})`, "info");
+			}
+
+			case "fetch-finish": {
+				const [total, time] = args;
+				return createLogEntry(`Finished fetching ${total} posts in ${ms(time)}`, "info");
+			}
+
+			case "skip": {
+				const [thread, id, reason, current, total] = args;
+				const t =
+					reason === "cache" ? "Post is cached." :
+						reason === "fileExists" ? "File already exists." :
+							reason === "video" ? "Post is a video." :
+								reason === "flash" ? "Post is flash." :
+									reason;
+				return createLogEntry(`[${current}/${total}] Skipped post #${id}, reason: ${t}`, "info");
+			}
+
+			case "post-finish": {
+				const [thread, id, time, current, total] = args;
+				return createLogEntry(`[Thread #${thread}][${current}/${total}] Finished downloading post #${id} in ${ms(time)}`, "info");
+			}
+
+			case "thread-done": {
+				const [thread, amount, time] = args;
+				return createLogEntry(`[Thread #${thread}] Finished downloading ${amount} posts in ${ms(time)}`, "info");
+			}
+
+			case "download-done": {
+				const [total, time] = args;
+				return createLogEntry(`Finished downloading ${total} posts in ${ms(time)}`, "info");
+			}
+		}
+		return;
 		switch (type) {
 			case "fetch-begin": {
 				const [tags, usingAuth] = args;
@@ -179,7 +236,7 @@ async function start(tags, folder) {
 			case "end": {
 				const [tags, amount, timeMS, time] = args;
 				console.debug("end");
-				ipcRenderer.removeListener("debug", l);
+				ipcRenderer.removeListener("message", l);
 				window.active = false;
 				createLogEntry(`Finished downloading ${amount} posts in ${time}`, "info");
 				if (window.cliMode) {
@@ -200,7 +257,7 @@ async function start(tags, folder) {
 			}
 		}
 	};
-	ipcRenderer.on("debug", l);
+	ipcRenderer.on("message", l);
 }
 
 ipcRenderer.on("progress", (ev, current, total) => {

src/assets/js/ms.js

@@ -0,0 +1,73 @@
+// https://github.com/FurryBotCo/FurryBot/blob/master/src/util/Functions/Time.ts#L31
+
+/**
+ * Convert milliseconds into readable time.
+ *
+ * @static
+ * @param {number} time - The time to convert.
+ * @returns {(Promise<string>)}
+ * @memberof Time
+ * @example Time.ms(120000);
+ */
+window.ms = function ms(time = 0) {
+	const seconds = true, words = true;
+	if (time < 0) throw new TypeError("Negative time provided.");
+	if (time === 0) return words ? "0 seconds" : "0s";
+	const r = {
+		s: 0,
+		m: 0,
+		h: 0,
+		d: 0,
+		w: 0,
+		mn: 0,
+		y: 0
+	};
+
+	while (time >= 1e3) {
+		r.s++; time -= 1e3;
+	}
+	while (r.s >= 60) {
+		r.m++; r.s -= 60;
+	}
+	while (r.m >= 60) {
+		r.h++; r.m -= 60;
+	}
+	while (r.h >= 24) {
+		r.d++; r.h -= 24;
+	}
+	// while (r.d >= 7) { r.w++; r.d -= 7; }
+	// while (r.w >= 4 && r.d >= 2) { r.mn++; r.w -= 4; r.d -= 2; }
+	while (r.d >= 30) {
+		r.mn++; r.d -= 30;
+	}
+	while (r.mn >= 12) {
+		r.y++; r.mn -= 12;
+	}
+	if (time > 0) r.s += time / 1000;
+
+	Object.keys(r).map(k => r[k] = Math.floor(r[k]));
+
+	const str = [];
+	if (time > 0) str.push(`${time.toFixed(0)} millisecond${time === 1 ? "" : "s"}`);
+	if (r.s > 0) str.push(`${r.s} second${r.s === 1 ? "" : "s"}`);
+	if (r.m > 0) str.push(`${r.m} minute${r.m === 1 ? "" : "s"}`);
+	if (r.h > 0) str.push(`${r.h} hour${r.h === 1 ? "" : "s"}`);
+	if (r.d > 0) str.push(`${r.d} day${r.d === 1 ? "" : "s"}`);
+	// if (r.w > 0) str.push(`${r.w} week${r.w === 1 ? "" : "s"}`);
+	if (r.mn > 0) str.push(`${r.mn} month${r.mn === 1 ? "" : "s"}`);
+	if (r.y > 0) str.push(`${r.y} year${r.y === 1 ? "" : "s"}`);
+
+	if (words && str.length > 1) str[0] = `and ${str[0]}`;
+
+	if (!seconds) {
+		if (words) {
+			const e = str.find(v => v.indexOf("second") !== -1);
+			if (e) {
+				str.splice(str.indexOf(e), 1);
+				if (str.length < 1) str.push("less than 1 minute");
+			}
+		} else delete r.s;
+	}
+
+	return words ? str.reverse().join(", ") : Object.keys(r).filter(k => r[k] > 0).map(k => `${Math.floor(r[k])}${k}`).reverse().reduce((a, b) => a + b, "");
+};

src/index.ts

@@ -17,14 +17,7 @@ import os from "os";
 program
 	// .option("-h, --help", "Show this list.")
 	.version(pkg.version, "-v, --version", "Output the current application version.")
-	.option("--run, --cli", "Run application in cli mode.")
 	.option("--dev", "Run in development mode.")
-	.option("-c, --config <file>", "Set the path to the config file.")
-	.option("--default-config <file>", "Set the path to the default config file.")
-	.option("-d, --dir <directory>", "Set the path to the save directory.")
-	.option("--tags <tags>", "Set the tags to use in cli mode.")
-	.option("--folder-name <folder>", "Set the folder name to use in cli mode. Defaults to first tag.")
-	.option("--debug-cli", "Keep debug output on when in cli mode (a lot of spammy logging).")
 	.parse([process.argv0, process.argv0, ...args]);
 const o = program.opts();
 // because of the description override
@@ -56,7 +49,6 @@ ipcMain
 	.on("setup", async (ev) => {
 		const v = await Utility.getRelease();
 		ev.reply("setup", ConfigManager.get(), ConfigManager.get(true), v);
-		if (o.cli) Utility.cliMode(window, o, !!o.debugCli);
 	})
 	.on("open-dev-tools", (ev) => window.webContents.openDevTools());