New: Delete downloaded compressed files for thumbnail generation if t…

…hey exceed 50% of the maximum tmp size

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Option to open file location of current reading from the file menu [`c9215dc`](https://github.com/ollm/OpenComic/commit/c9215dc5cb29a3b5a759d80d21f7ff734053f23c)
 - Setting to enable/disable go next/previous chapter with mouse scroll (Vertical reading) [`37612bf`](https://github.com/ollm/OpenComic/commit/37612bfdce13ce73348bda997bf3aeb32b8915af)
 - About this file dialog [`38f72f3`](https://github.com/ollm/OpenComic/commit/38f72f3c573aaa2a1923f3e6704261a9b600b3ab)
+- Delete downloaded compressed files for thumbnail generation if they exceed 50% of the maximum tmp size
 
 ##### 🐛 Bug Fixes
 

scripts/app.js

@@ -202,6 +202,39 @@ function validateUrl(value, protocols = 'https?|ftp')
 	return new RegExp('^(?:(?:(?:'+protocols+'):)?\\/\\\/)(?:\\S+(?::\\S*)?@)?(?:(?!(?:10|127)(?:\\.\\d{1,3}){3})(?!(?:169\\.254|192\\.168)(?:\\.\\d{1,3}){2})(?!172\\.(?:1[6-9]|2\\d|3[0-1])(?:\\.\\d{1,3}){2})(?:[1-9]\\d?|1\\d\\d|2[01]\\d|22[0-3])(?:\\.(?:1?\\d{1,2}|2[0-4]\\d|25[0-5])){2}(?:\\.(?:[1-9]\\d?|1\\d\\d|2[0-4]\\d|25[0-4]))|(?:(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)(?:\.(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)*(?:\\.(?:[a-z\\u00a1-\\uffff]{2,})))(?::\\d{2,5})?(?:[/?#]\\S*)?$', 'i').test(value);
 }
 
+function copy(toCopy)
+{
+	if(typeof toCopy !== 'object' || toCopy === null)
+	{
+		return toCopy;
+	}
+	else if(Array.isArray(toCopy))
+	{
+		let len = toCopy.length;
+		let result = new Array(len);
+
+		for(let i = 0; i < len; i++)
+		{
+			let _toCopy = toCopy[i];
+			result[i] = typeof _toCopy !== 'object' || _toCopy === null ? _toCopy : copy(_toCopy);
+		}
+
+		return result;
+	}
+	else
+	{
+		let result = {};
+
+		for(let i in toCopy)
+		{
+			let _toCopy = toCopy[i];
+			result[i] = typeof _toCopy !== 'object' || _toCopy === null ? _toCopy : copy(_toCopy);
+		}
+
+		return result;
+	}
+}
+
 function time()
 {
 	return Math.floor(Date.now() / 1000);
@@ -242,6 +275,7 @@ module.exports = {
 	capitalize: capitalize,
 	stripTagsWithDOM: stripTagsWithDOM,
 	validateUrl: validateUrl,
+	copy: copy,
 	time: time,
 	sleep: sleep,
 	setImmediate: setImmediate,

scripts/cache.js

@@ -21,7 +21,15 @@ function processTheImageQueue()
 	let realPath = fileManager.realPath(img.file);
 	let toImage = p.join(cacheFolder, sha+'.jpg');
 
-	image.resize(realPath, toImage, {width: img.size, height: Math.round(img.size * (img.poster ? 1.51369 : 1.5)), quality: 95, fit: img.poster ? 'cover' : 'inside'}).then(function(){
+	let fit = {
+		poster: 'cover',
+	};
+
+	let ratios = {
+		poster: 1.51369,
+	};
+
+	image.resize(realPath, toImage, {width: img.size, height: Math.round(img.size * (img.type ? ratios[img.type] : 1.5)), quality: 95, fit: img.type ? fit[img.type] : 'inside'}).then(function(){
 
 		if(typeof data[sha] == 'undefined') data[sha] = {lastAccess: app.time()};
 
@@ -66,9 +74,9 @@ function processTheImageQueue()
 	});
 }
 
-function addImageToQueue(file, size, sha, callback, vars, poster)
+function addImageToQueue(file, size, sha, callback, vars, type)
 {
-	queuedImages.push({file: file, size: size, sha: sha, callback: callback, vars: vars, poster: poster});
+	queuedImages.push({file: file, size: size, sha: sha, callback: callback, vars: vars, type: type});
 
 	if(!processingTheImageQueue && !stopTheImageQueue)
 	{
@@ -125,7 +133,10 @@ function returnThumbnailsImages(images, callback, file = false)
 	images = single ? [images] : images;
 
 	let size = Math.round(window.devicePixelRatio * 150);
-	let sizePoster = Math.round(window.devicePixelRatio * 146);
+
+	sizes = {
+		poster: Math.round(window.devicePixelRatio * 146),
+	};
 
 	let thumbnail = {};
 	let thumbnails = {};
@@ -138,17 +149,17 @@ function returnThumbnailsImages(images, callback, file = false)
 	{
 		let image = images[i];
 
-		let sha = (image.poster) ? sha1(image.path+'?type=poster') : (image.sha || sha1(image.path));
+		let sha = (image.type) ? sha1(image.path+'?type='+image.type) : (image.sha || sha1(image.path));
 		let imgCache = data[sha];
 
-		let _size = image.poster ? sizePoster : size;
+		let _size = image.type ? sizes[image.type] : size;
 
 		let path = addCacheVars(p.join(cacheFolder, sha+'.jpg'), _size, sha);
 
 		if(typeof imgCache == 'undefined' || !fs.existsSync(p.join(cacheFolder, sha+'.jpg')))
 		{
 			toGenerateThumbnails.push(image);
-			toGenerateThumbnailsData[image.path] = {sha: sha, vars: image.vars, poster: image.poster || false};
+			toGenerateThumbnailsData[image.path] = {sha: sha, vars: image.vars, type: image.type || false};
 
 			thumbnails[sha] = thumbnail = {cache: false, path: '', sha: sha};
 		}
@@ -159,7 +170,7 @@ function returnThumbnailsImages(images, callback, file = false)
 			if(imgCache.size != _size)
 			{
 				toGenerateThumbnails.push(image);
-				toGenerateThumbnailsData[image.path] = {sha: sha, vars: image.vars, poster: image.poster || false};
+				toGenerateThumbnailsData[image.path] = {sha: sha, vars: image.vars, type: image.type || false};
 
 				thumbnails[sha] = thumbnail = {cache: true, path: escapeBackSlash(path), sha: sha};
 			}
@@ -176,10 +187,10 @@ function returnThumbnailsImages(images, callback, file = false)
 		file.makeAvailable(toGenerateThumbnails, function(image) {
 
 			let data = toGenerateThumbnailsData[image.path];
-			let _size = data.poster ? sizePoster : size;
-			addImageToQueue(image.path, _size, data.sha, callback, data.vars || false, data.poster);
+			let _size = data.type ? sizes[data.type] : size;
+			addImageToQueue(image.path, _size, data.sha, callback, data.vars || false, data.type);
 
-		});
+		}, false, true);
 	}
 
 	return single ? thumbnail : thumbnails;
@@ -258,7 +269,7 @@ function readJsonInMemory(name)
 
 		jsonMemory[name].lastUsage = app.time();
 
-		return jsonMemory[name].json;
+		return app.copy(jsonMemory[name].json);
 	}
 
 	return false;

scripts/dom.js

@@ -1135,7 +1135,7 @@ async function _getFolderThumbnails(file, images, _images, path, folderSha, isAs
 	{
 		if(isAsync) dom.queryAll('.sha-'+folderSha+' .folder-images').remove();
 
-		poster = cache.returnThumbnailsImages({path: _images.path, sha: _images.sha, poster: true}, function(data){
+		poster = cache.returnThumbnailsImages({path: _images.path, sha: _images.sha, type: 'poster'}, function(data){
 
 			addImageToDom(data.sha, data.path);
 			addImageToDom(folderSha+'-0', data.path);
@@ -1171,7 +1171,7 @@ async function getFolderThumbnails(path)
 
 	try
 	{
-		let file = fileManager.file(path);
+		let file = fileManager.file(path, {fromThumbnailsGeneration: true});
 		file.updateConfig({cacheOnly: true});
 		let _images = await file.images(4, false, true);
 
@@ -1188,7 +1188,9 @@ async function getFolderThumbnails(path)
 		{
 			queue.add('folderThumbnails', async function(path, folderSha) {
 
-				let file = fileManager.file(path);
+				console.log(path);
+
+				let file = fileManager.file(path, {fromThumbnailsGeneration: true});
 				let _images = await file.images(4, false, true);
 
 				await _getFolderThumbnails(file, images, _images, path, folderSha, true);

scripts/file-manager.js

@@ -190,6 +190,9 @@ var file = function(path, _config = false) {
 		if(this.config.progress)
 			this.compressedOpened[path].compressed.progress = this.config.progress;
 
+		if(this.config.fromThumbnailsGeneration)
+			this.compressedOpened[path].compressed.config.fromThumbnailsGeneration = true;
+
 		return this.compressedOpened[path].compressed;
 
 	}
@@ -302,6 +305,9 @@ var file = function(path, _config = false) {
 			// Download file to tmp
 			let file = await serverClient.download(path, {only: [firstCompressed]});
 
+			if(this.config.fromThumbnailsGeneration)
+				downloadedCompressedFile(firstCompressed);
+
 			return this.readCompressed(path);
 		}
 		else
@@ -324,6 +330,9 @@ var file = function(path, _config = false) {
 		{
 			// Download file to tmp
 			let file = await serverClient.download(path, {only: [firstCompressed]});
+
+			if(this.config.fromThumbnailsGeneration)
+				downloadedCompressedFile(firstCompressed);
 		}
 
 		return this.readInsideCompressed(path, _realPath);
@@ -651,7 +660,7 @@ var file = function(path, _config = false) {
 	}
 
 	// Makes the files available, extracting them from the respective compressed files if necessary
-	this.makeAvailable = async function(files, callbackWhenFileAvailable = false, forceCheck = false) {
+	this.makeAvailable = async function(files, callbackWhenFileAvailable = false, forceCheck = false, fromThumbnailsGeneration = false) {
 
 		let filesToDecompress = false, filesToDecompressNum = 0;
 
@@ -716,6 +725,9 @@ var file = function(path, _config = false) {
 							if(callbackWhenFileAvailable) callbackWhenFileAvailable(file);
 
 						});
+
+						if(fromThumbnailsGeneration || _this.config.fromThumbnailsGeneration)
+							downloadedCompressedFile(firstCompressed);
 					}
 					else
 					{
@@ -2576,6 +2588,73 @@ async function removeTmpVector()
 	}
 }
 
+var downloadedCompressedFiles = {
+	list: [],
+	sizes: {},
+};
+
+function downloadedCompressedFile(path)
+{
+	let realPath = fileManager.realPath(path, -1);
+
+	let totalSize = 0;
+	let list = [];
+
+	for(let i = 0, len = downloadedCompressedFiles.list.length; i < len; i++)
+	{
+		let _realPath = downloadedCompressedFiles.list[i];
+
+		if(fs.existsSync(_realPath) && _realPath !== realPath)
+		{
+			let size = downloadedCompressedFiles.sizes[_realPath] || fs.statSync(_realPath).size;
+
+			downloadedCompressedFiles.sizes[_realPath] = size;
+			list.push(_realPath);
+
+			totalSize += size;
+		}
+	}
+
+	downloadedCompressedFiles.list = list;
+	downloadedCompressedFiles.list.push(realPath);
+
+	let size = downloadedCompressedFiles.sizes[realPath] || fs.statSync(realPath).size;
+
+	downloadedCompressedFiles.sizes[realPath] = size;
+	totalSize += size;
+
+	let maxSize = ((config.tmpMaxSize || 2) / 2) * 1000 * 1000 * 1000; // 50% of tmpMaxSize
+
+	if(totalSize > maxSize)
+	{
+		let list = [];
+		let sizes = {};
+
+		for(let i = 0, len = downloadedCompressedFiles.list.length; i < len; i++)
+		{
+			let _realPath = downloadedCompressedFiles.list[i];
+			let size = downloadedCompressedFiles.sizes[_realPath] || fs.statSync(_realPath).size;
+
+			if(totalSize > maxSize)
+			{
+				if(fs.existsSync(_realPath))
+				{
+					totalSize -= size;
+					fs.unlinkSync(_realPath);
+				}
+			}
+			else
+			{
+				list.push(_realPath);
+				sizes[_realPath] = size;
+			}
+		}
+
+		downloadedCompressedFiles.list = list;
+		downloadedCompressedFiles.sizes = sizes;
+	}
+}
+
 function realPath(path, index = 0, prefixes = false)
 {
 	let segments = path.split(p.sep);

scripts/server-client.js

@@ -952,8 +952,6 @@ var client = function(path) {
 
 		let files = [];
 
-		console.time('downloadSsh');
-
 		let _this = this;
 		let _only = this.config._only; 
 

scripts/settings.js

@@ -123,11 +123,13 @@ function removeTemporaryPath(path)
 	return size;
 }
 
-function purgeTemporaryFiles()
+function purgeTemporaryFiles(tmpMaxSize = false)
 {
+	tmpMaxSize = tmpMaxSize || config.tmpMaxSize;
+
 	try
 	{
-		if(config.tmpMaxSize == 0)
+		if(tmpMaxSize == 0)
 		{
 			settings.removeTemporaryFiles(true);
 		}
@@ -136,7 +138,7 @@ function purgeTemporaryFiles()
 			let time = app.time();
 			let tmpUsage = storage.get('tmpUsage') || {};
 
-			let tmpMaxSize = config.tmpMaxSize * 1000 * 1000 * 1000;
+			tmpMaxSize = tmpMaxSize * 1000 * 1000 * 1000;
 			let tmpMaxOld = config.tmpMaxOld * 60 * 60 * 24;
 
 			let dataArray = [];