New: Thumbnail generation speed has been improved

Author: ollm

CHANGELOG.md

@@ -11,7 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Improve chapter detection in tracking [`ad05a23`](https://github.com/ollm/OpenComic/commit/ad05a2364882227688cccfa2fdf58a21f8e9615c)
 - Support for `JP2` image format [`e11046b`](https://github.com/ollm/OpenComic/commit/e11046b4eca758f8b15c1caeb831a235a5f3d62c)
 - Support for `JXR` image format [`0e24922`](https://github.com/ollm/OpenComic/commit/0e2492264e1dbc94126c7d79e3e3c50fc426846c)
-- Support for `JXL` image format
+- Support for `JXL` image format [`bd6fb0a`](https://github.com/ollm/OpenComic/commit/bd6fb0a8c8be3685b48c88fa21bbe2d3c384a491)
+- Thumbnail generation speed has been improved
+
 
 ## [v1.4.1](https://github.com/ollm/OpenComic/releases/tag/v1.4.1) (08-02-2025)
 

scripts/cache.js

@@ -16,8 +16,6 @@ catch (error)
 	zstd = zstdEncoder = zstdDecoder = false;
 }
 
-var queuedImages = [], processingTheImageQueue = false;
-
 var cacheFolder = p.join(electronRemote.app.getPath('cache'), 'opencomic');
 
 if(process.platform == 'win32' || process.platform == 'win64')
@@ -27,13 +25,8 @@ if(!fs.existsSync(cacheFolder)) fs.mkdirSync(cacheFolder);
 cacheFolder = p.join(cacheFolder, 'cache');
 if(!fs.existsSync(cacheFolder)) fs.mkdirSync(cacheFolder);
 
-var imagesWithoutSaving = 0;
-
-async function processTheImageQueue()
+async function processTheImageQueue(img = false)
 {
-	let img = queuedImages.shift();
-	if(!img) return;
-
 	let sha = img.sha;
 
 	let realPath = fileManager.realPath(img.file);
@@ -51,65 +44,31 @@ async function processTheImageQueue()
 	if(inArray(fileExtension(realPath), imageExtensions.blob)) // Convert unsupported images to blob
 		blob = await workers.convertImageToBlob(realPath);
 
-	image.resize(blob || realPath, toImage, {blob: blob ? true : false, width: img.size, height: Math.round(img.size * (img.type ? ratios[img.type][forceSize] : 1.5)), quality: 95, fit: img.type ? fit[img.type] : 'inside'}).then(function(){
+	return new Promise(function(resolve, reject) {
 
-		//if(blob) fileManager.revokeObjectURL(path);
+		image.resize(blob || realPath, toImage, {blob: blob ? true : false, width: img.size, height: Math.round(img.size * (img.type ? ratios[img.type][forceSize] : 1.5)), quality: 95, fit: img.type ? fit[img.type] : 'inside'}).then(function(){
 
-		if(typeof data[sha] == 'undefined') data[sha] = {lastAccess: app.time()};
+			//if(blob) fileManager.revokeObjectURL(path);
 
-		data[sha].size = img.size;
+			if(typeof data[sha] == 'undefined') data[sha] = {lastAccess: app.time()};
 
-		img.callback({cache: true, path: escapeBackSlash(addCacheVars(toImage, img.size, img.sha)), sha: sha}, img.vars);
+			data[sha].size = img.size;
 
-		if(queuedImages.length > 0)
-		{
-			imagesWithoutSaving++;
+			img.callback({cache: true, path: escapeBackSlash(addCacheVars(toImage, img.size, img.sha)), sha: sha}, img.vars);
 
-			process.nextTick(function() {
-				processTheImageQueue();
-			});
+			checkImagesWithoutSaving();
+			resolve();
 
-			if(imagesWithoutSaving > 50)
-			{
-				imagesWithoutSaving = 0;
-				storage.setThrottle('cache', data);
-			}
-		}
-		else
-		{
-			imagesWithoutSaving = 0;
-			processingTheImageQueue = false;
+		}).catch(function(){
 
-			storage.setThrottle('cache', data);
-		}
+			//if(blob) fileManager.revokeObjectURL(path);
 
-	}).catch(function(){
+			img.callback({cache: true, path: escapeBackSlash(realPath), sha: sha}, img.vars);
 
-		//if(blob) fileManager.revokeObjectURL(path);
-
-		img.callback({cache: true, path: escapeBackSlash(realPath), sha: sha}, img.vars);
-
-		if(queuedImages.length > 0)
-		{
-			imagesWithoutSaving++;
-
-			process.nextTick(function() {
-				processTheImageQueue();
-			});
-
-			if(imagesWithoutSaving > 50)
-			{
-				imagesWithoutSaving = 0;
-				storage.setThrottle('cache', data);
-			}
-		}
-		else
-		{
-			imagesWithoutSaving = 0;
-			processingTheImageQueue = false;
+			checkImagesWithoutSaving();
+			resolve();
 
-			storage.setThrottle('cache', data);
-		}
+		});
 
 	});
 }
@@ -121,44 +80,42 @@ async function addImageToQueue(file, size, sha, callback, vars, type, forceSize)
 	if(inArray(extention, imageExtensions.convert)) // Convert unsupported images
 		await workers.convertImage(file);
 
-	queuedImages.push({file: file, size: size, sha: sha, callback: callback, vars: vars, type: type, forceSize: forceSize});
-
-	if(!processingTheImageQueue && !stopTheImageQueue)
-	{
-		processingTheImageQueue = true;
+	threads.job('cache', {useThreads: 0.49}, processTheImageQueue, {file: file, size: size, sha: sha, callback: callback, vars: vars, type: type, forceSize: forceSize});
 
-		setTimeout(function(){
+	threads.end('cache', function(){
 
-			process.nextTick(function() {
-				processTheImageQueue();
-			});
+		imagesWithoutSaving = 0;
+		storage.setThrottle('cache', data);
 
-		}, 0);
-	}
+	});
 }
 
-var stopTheImageQueue = false;
-
 function stopQueue()
 {
-	stopTheImageQueue = true;
+	threads.stop('cache');
 }
 
 function resumeQueue()
 {
-	stopTheImageQueue = false;
+	threads.resume('cache');
+}
 
-	if(!processingTheImageQueue && queuedImages.length > 0)
-	{
-		processingTheImageQueue = true;
+function cleanQueue()
+{
+	threads.clean('cache');
+	if(data !== false) storage.setThrottle('cache', data);
+}
 
-		setTimeout(function(){
+var imagesWithoutSaving = 0;
 
-			process.nextTick(function() {
-				processTheImageQueue();
-			});
+function checkImagesWithoutSaving()
+{
+	imagesWithoutSaving++;
 
-		}, 0);
+	if(imagesWithoutSaving > 50)
+	{
+		imagesWithoutSaving = 0;
+		storage.setThrottle('cache', data);
 	}
 }
 
@@ -235,12 +192,6 @@ function addImageVars(image)
 	return image.path+(vars.length ? '?'+vars.join('&') : '');
 }
 
-function cleanQueue()
-{
-	queuedImages.splice(1, queuedImages.length - 1);
-	if(data !== false) storage.setThrottle('cache', data);
-}
-
 var data = false;
 
 function returnThumbnailsImages(images, callback, file = false)
@@ -890,8 +841,6 @@ module.exports = {
 	imageSizeSha: imageSizeSha,
 	addImageVars: addImageVars,
 	jsonMemory: function(){return jsonMemory},
-	queuedImages: function(){return queuedImages},
-	processingTheImageQueue: function(){return processingTheImageQueue},
 	stopQueue: stopQueue,
 	resumeQueue: resumeQueue,
 	purge: purge,

scripts/opencomic.js

@@ -424,6 +424,7 @@ const app = require(p.join(appDir, 'scripts/app.js')),
 	events = require(p.join(appDir, 'scripts/events.js')),
 	ebook = require(p.join(appDir, 'scripts/ebook.js')),
 	workers = require(p.join(appDir, 'scripts/workers.js')),
+	threads = require(p.join(appDir, 'scripts/threads.js')),
 	fileManager = require(p.join(appDir, 'scripts/file-manager.js')),
 	serverClient = require(p.join(appDir, 'scripts/server-client.js')),
 	reading = require(p.join(appDir, 'scripts/reading.js')),

scripts/threads.js

@@ -0,0 +1,178 @@
+var threads = false;
+var threadsList = {};
+var queue = {default: []};
+var queueIsStop = {default: false};
+var onEnd = {};
+
+function job(key = 'default', options = {}, callback = false)
+{
+	const _arguments = [];
+
+	for(let i = 3, len = arguments.length; i < len; i++)
+	{
+		_arguments.push(arguments[i]);
+	}
+
+	let resolve = false;
+	let reject = false;
+
+	const promise = new Promise(function(_resolve, _reject) {
+
+		resolve = _resolve;
+		reject = _reject;
+
+	});
+
+	const _promise = {
+		promise: promise,
+		resolve: resolve,
+		reject: reject,
+	};
+
+	addToQueue(key, options, callback, _arguments, _promise);
+
+	return promise;
+}
+
+function addToQueue(key, options, callback, _arguments, promise)
+{
+	if(!queue[key]) queue[key] = [];
+
+	if(options.priorize)
+	{
+		queue[key].unshift({
+			options: options,
+			callback: callback,
+			arguments: _arguments,
+			promise: promise,
+		});
+	}
+	else
+	{
+		queue[key].push({
+			options: options,
+			callback: callback,
+			arguments: _arguments,
+			promise: promise,
+		});
+	}
+
+	processQueue(key);
+}
+
+function processQueue(key = 'default')
+{
+	if(!threads)
+		threads = os.cpus().length || 1;
+
+	if(queueIsStop[key])
+		return;
+
+	for(let thread = 0; thread < threads; thread++)
+	{
+		const status = processJob(key, thread);
+		if(status === null) break;
+	}
+}
+
+function processJob(key = 'default', _thread = 0)
+{
+	if(queueIsStop[key])
+		return;
+
+	const thread = getThread(key, _thread);
+
+	if(thread.busy)
+		return false;
+
+	const _queue = queue[key];
+
+	if(!_queue.length)
+	{
+		checkEnd(key);
+		return null;
+	}
+
+	const job = _queue.shift();
+
+	if(job.options.useThreads < 1 && _thread !== 0)
+	{
+		const useThreads = _thread / threads;
+
+		if(useThreads > job.options.useThreads)
+		{
+			_queue.unshift(job);
+			return null;
+		}
+	}
+
+	thread.busy = true;
+	thread.currentJob = job;
+	startJob(thread);
+}
+
+async function startJob(thread)
+{
+	const result = await thread.currentJob.callback(...thread.currentJob.arguments);
+
+	thread.currentJob.promise.resolve(result);
+	thread.currentJob = false;
+	thread.busy = false;
+
+	processJob(thread.key, thread.thread);
+}
+
+function getThread(key = 'default', thread = 0)
+{
+	if(!threadsList[key]) threadsList[key] = {};
+
+	if(threadsList[key][thread])
+		return threadsList[key][thread];
+
+	threadsList[key][thread] = {
+		key: key,
+		thread: thread,
+		busy: false,
+		currentJob: false,
+	};
+
+	return threadsList[key][thread];
+}
+
+function stop(key = 'default')
+{
+	queueIsStop[key] = true;
+}
+
+function resume(key = 'default')
+{
+	queueIsStop[key] = false;
+	processQueue(key);
+}
+
+function clean(key = 'default')
+{
+	queue[key] = [];
+}
+
+function checkEnd(key = 'default')
+{
+	if(onEnd[key])
+	{
+		onEnd[key]();
+		onEnd[key] = false;
+	}
+}
+
+function end(key, callback)
+{
+	onEnd[key] = callback;
+}
+
+module.exports = {
+	job: job,
+	stop: stop,
+	resume: resume,
+	clean: clean,
+	end: end,
+}