diff --git a/.gitignore b/.gitignore index 69d180f3b..7db72b2ca 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ www-ghdeploy /.vs/ /.vscode/ scripts/github_token +.prettierrc diff --git a/i18n/en.jsonp.js b/i18n/en.jsonp.js index 2f4b691c6..b9b313ff3 100644 --- a/i18n/en.jsonp.js +++ b/i18n/en.jsonp.js @@ -19,8 +19,14 @@ document.localeJson = { "configure-title": "Configuration", "configure-about-usage-link": "About (Usage)", "configure-btn-library": "Browse ZIM Library", + "configure-btn-folderselect": "Select Folder", + "configure-btn-rescan": "Rescan", + "configure-about-rescan-tip": "Rescans your SD Cards and internal memory", + "configure-select-file-numbers": "archive(s) found in selected location. ", "configure-download-instructions": "This application needs a ZIM archive to work.
For full instructions, please see the section", "configure-select-instructions": "Please select or drag and drop a .zim file (or all the .zimaa, .zimab etc in case of a split ZIM file):", + "configure-select-file-instructions": "Please select the archive you want to use:", + "configure-select-file-first-option": "Select an archive...", "configure-selectordisplay": "Drag and drop a new ZIM file, or", "configure-selectordisplay-link": "display file selectors", "configure-static-content": "Only ZIMs with static content (e.g. Wiki-style) are supported in JQuery mode.
", diff --git a/i18n/es.jsonp.js b/i18n/es.jsonp.js index ba3a173bf..72809957a 100644 --- a/i18n/es.jsonp.js +++ b/i18n/es.jsonp.js @@ -19,8 +19,14 @@ document.localeJson = { "configure-title": "Configuración", "configure-about-usage-link": "Información (Uso)", "configure-btn-library": "Biblioteca ZIM", + "configure-btn-folderselect": "Seleccione carpeta", + "configure-btn-rescan": "Volver a escanear", + "configure-about-rescan-tip": "Vuelve a escanear las tarjetas SD y la memoria interna", + "configure-select-file-numbers": "archivo(s) encontrado(s) en la ubicación seleccionada. ", "configure-download-instructions": "Esta aplicación necesita un archivo ZIM para funcionar.
Para instrucciones completas, vea la sección", "configure-select-instructions": "Seleccione o arrastre y suelte un archivo .zim (o todos los .zimaa, .zimab etc en caso de un archivo dividido):", + "configure-select-file-instructions": "Seleccione el archivo que desea utilizar:", + "configure-select-file-first-option": "Seleccione un archivo...", "configure-selectordisplay": "Arrastre y suelte un nuevo archivo ZIM, o", "configure-selectordisplay-link": "mostrar selector de archivos", "configure-static-content": "Sólo funcionan bien los ZIM con contenido estático (tipo Wiki) en el modo JQuery.
", diff --git a/i18n/fr.jsonp.js b/i18n/fr.jsonp.js index abec3ef36..7da891ce2 100644 --- a/i18n/fr.jsonp.js +++ b/i18n/fr.jsonp.js @@ -19,8 +19,14 @@ document.localeJson = { "configure-title": "Configuration", "configure-about-usage-link": "Informations (Utilisation)", "configure-btn-library": "Bibliothèque ZIM", + "configure-btn-folderselect": "Sélectionner un dossier", + "configure-btn-rescan": "Réanalyser", + "configure-about-rescan-tip": "Réanalyser la carte SD et la mérmoire interne", + "configure-select-file-numbers": "archive(s) trouvée(s) dans le lieu sélectionné. ", "configure-download-instructions": "Cette application a besoin d'un fichier ZIM pour fonctionner.
Pour des instructions complètes, veuillez consulter la section", "configure-select-instructions": "Veuillez sélectionner ou glisser-déposer un fichier .zim (ou tous les .zimaa, .zimab etc. dans le cas d'un fichier ZIM découpé) :", + "configure-select-file-instructions": "Veuillez sélectionner l'archive que vous souhaitez utiliser :", + "configure-select-file-first-option": "Sélectionner une archive...", "configure-selectordisplay": "Glisser-déposer un nouveau fichier ZIM, ou", "configure-selectordisplay-link": "afficher le sélecteur de fichiers", "configure-static-content": "Seuls les ZIM avec un contenu statique (de type Wiki) sont pris en charge en mode JQuery.
", diff --git a/service-worker.js b/service-worker.js index 7c7fa5de5..416a7c6e2 100644 --- a/service-worker.js +++ b/service-worker.js @@ -125,6 +125,7 @@ const precacheFiles = [ 'www/js/lib/abstractFilesystemAccess.js', 'www/js/lib/arrayFromPolyfill.js', 'www/js/lib/filecache.js', + 'www/js/lib/cache.js', 'www/js/lib/promisePolyfill.js', 'www/js/lib/settingsStore.js', 'www/js/lib/translateUI.js', diff --git a/tests/e2e/spec/gutenberg_ro.e2e.spec.js b/tests/e2e/spec/gutenberg_ro.e2e.spec.js index 4232085d1..08e854f13 100644 --- a/tests/e2e/spec/gutenberg_ro.e2e.spec.js +++ b/tests/e2e/spec/gutenberg_ro.e2e.spec.js @@ -1,7 +1,7 @@ /** * legacy-ray_charles.e2e.spec.js : End-to-end tests implemented with Selenium WebDriver and Mocha * - * Copyright 2023 Jaifroid, RG7279805 and contributors + * Copyright 2023 Jaifroid, Rishabhg71 and contributors * Licence GPL v3: * * This file is part of Kiwix. @@ -192,6 +192,9 @@ function runTests (driver, modes) { const archiveFiles = await driver.findElement(By.id('archiveFiles')); if (!isFileLoaded) await archiveFiles.sendKeys(gutenbergRoBaseFile); filesLength = await driver.executeScript('return document.getElementById("archiveFiles").files.length'); + // In new browsers Files are loaded using the FileSystem API, so we have to set the local archives using JavaScript + // which were selected using the file input + await driver.executeScript('window.setLocalArchiveFromFileSelect();'); // Check that we loaded 1 file assert.equal(1, filesLength); } else { diff --git a/tests/e2e/spec/legacy-ray_charles.e2e.spec.js b/tests/e2e/spec/legacy-ray_charles.e2e.spec.js index 162497289..7691ddcc3 100644 --- a/tests/e2e/spec/legacy-ray_charles.e2e.spec.js +++ b/tests/e2e/spec/legacy-ray_charles.e2e.spec.js @@ -207,6 +207,9 @@ function runTests (driver, modes) { filesLength = await driver.executeScript('return document.getElementById("archiveFiles").files.length'); return filesLength === 15; }, 5000); + // In new browsers Files are loaded using the FileSystem API, so we have to set the local archives using JavaScript + // which were selected using the file input + await driver.executeScript('window.setLocalArchiveFromFileSelect();'); // Check that we loaded 15 files assert.equal(15, filesLength); } else { diff --git a/www/css/app.css b/www/css/app.css index 620093e7e..fbb0d0a3c 100644 --- a/www/css/app.css +++ b/www/css/app.css @@ -98,6 +98,21 @@ margin: 0 1rem; } +#filesSelectionInfoAndCount { + padding-top: 0.5rem; +} +#filesSelectionInfoAndCount p{ + display: inline; +} + +#archiveList { + width: 60%; + min-width: auto; +} + +#rescanButtonAndText button { + margin-top: 0.5rem; +} /* Custom file input */ input[type="file"] { @@ -330,4 +345,10 @@ button { padding-bottom: 1px !important; font-size: 16px !important; } + + #archiveList { + width: 100%; + min-width: auto; + } + } diff --git a/www/index.html b/www/index.html index 0944c1328..c80df97c2 100644 --- a/www/index.html +++ b/www/index.html @@ -477,13 +477,17 @@

Configuration

display file selectors.

-

Display settings

diff --git a/www/js/app.js b/www/js/app.js index 2b6e5242d..9ed4aee50 100644 --- a/www/js/app.js +++ b/www/js/app.js @@ -188,6 +188,7 @@ function resizeIFrame () { document.addEventListener('DOMContentLoaded', function () { getDefaultLanguageAndTranslateApp(); resizeIFrame(); + abstractFilesystemAccess.loadPreviousZimFile(); }); window.addEventListener('resize', resizeIFrame); @@ -1189,6 +1190,7 @@ window.onpopstate = function (event) { function populateDropDownListOfArchives (archiveDirectories) { document.getElementById('scanningForArchives').style.display = 'none'; document.getElementById('chooseArchiveFromLocalStorage').style.display = ''; + document.getElementById('rescanButtonAndText').style.display = ''; var comboArchiveList = document.getElementById('archiveList'); comboArchiveList.options.length = 0; for (var i = 0; i < archiveDirectories.length; i++) { @@ -1281,11 +1283,26 @@ function resetCssCache () { } } +let webKitFileList = null /** * Displays the zone to select files from the archive */ function displayFileSelect () { + const isFireFoxOsNativeFileApiAvailable = typeof navigator.getDeviceStorages === 'function'; + let isPlatformMobilePhone = false; + if (/Android/i.test(navigator.userAgent)) isPlatformMobilePhone = true; + if (/iphone|ipad|ipod/i.test(navigator.userAgent) || navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1) isPlatformMobilePhone = true; + + console.debug(`File system api is ${params.isFileSystemApiSupported ? '' : 'not '}supported`); + console.debug(`Webkit directory api ${params.isWebkitDirApiSupported ? '' : 'not '}supported`); + console.debug(`Firefox os native file ${isFireFoxOsNativeFileApiAvailable ? '' : 'not '}support api`) + document.getElementById('openLocalFiles').style.display = 'block'; + if ((params.isFileSystemApiSupported || params.isWebkitDirApiSupported) && !isPlatformMobilePhone) { + document.getElementById('chooseArchiveFromLocalStorage').style.display = ''; + document.getElementById('folderSelect').style.display = ''; + } + // Set the main drop zone if (!params.disableDragAndDrop) { configDropZone.addEventListener('dragover', handleGlobalDragover); @@ -1300,8 +1317,89 @@ function displayFileSelect () { }); globalDropZone.addEventListener('drop', handleFileDrop); } - // This handles use of the file picker - document.getElementById('archiveFiles').addEventListener('change', setLocalArchiveFromFileSelect); + + if (isFireFoxOsNativeFileApiAvailable) { + useLegacyFilePicker(); + return; + } + + document.getElementById('archiveList').addEventListener('change', async function (e) { + // handle zim selection from dropdown if multiple files are loaded via webkitdirectory or filesystem api + localStorage.setItem('previousZimFileName', e.target.value); + if (params.isFileSystemApiSupported) { + const files = await abstractFilesystemAccess.getSelectedZimFromCache(e.target.value) + setLocalArchiveFromFileList(files); + } else { + if (webKitFileList === null) { + const element = localStorage.getItem('zimFilenames').split('|').length === 1 ? 'archiveFiles' : 'archiveFolders'; + if ('showPicker' in HTMLInputElement.prototype) { + document.getElementById(element).showPicker(); + return; + } + document.getElementById(element).click() + return; + } + const files = abstractFilesystemAccess.getSelectedZimFromWebkitList(webKitFileList, e.target.value) + setLocalArchiveFromFileList(files); + } + }); + + if (params.isFileSystemApiSupported) { + // Handles Folder selection when showDirectoryPicker is supported + document.getElementById('folderSelect').addEventListener('click', async function (e) { + e.preventDefault(); + const previousZimFiles = await abstractFilesystemAccess.selectDirectoryFromPickerViaFileSystemApi() + if (previousZimFiles.length !== 0) setLocalArchiveFromFileList(previousZimFiles); + }) + } + if (params.isWebkitDirApiSupported) { + // Handles Folder selection when webkitdirectory is supported but showDirectoryPicker is not + document.getElementById('folderSelect').addEventListener('change', async function (e) { + e.preventDefault(); + const filenames = []; + + const previousZimFile = [] + const lastFilename = localStorage.getItem('previousZimFileName') ?? ''; + const filenameWithoutExtension = lastFilename.replace(/\.zim\w\w$/i, ''); + const regex = new RegExp(`\\${filenameWithoutExtension}.zim\\w\\w$`, 'i'); + + for (const file of e.target.files) { + filenames.push(file.name); + if (regex.test(file.name) || file.name === lastFilename) previousZimFile.push(file); + } + webKitFileList = e.target.files; + localStorage.setItem('zimFilenames', filenames.join('|')); + // will load the old file if the selected folder contains the same file + if (previousZimFile.length !== 0) setLocalArchiveFromFileList(previousZimFile); + await abstractFilesystemAccess.updateZimDropdownOptions(filenames, previousZimFile.length !== 0 ? lastFilename : ''); + }) + } + if (params.isFileSystemApiSupported) { + // Handles File selection when showOpenFilePicker is supported and uses the filesystem api + document.getElementById('archiveFiles').addEventListener('click', async function (e) { + e.preventDefault(); + const files = await abstractFilesystemAccess.selectFileFromPickerViaFileSystemApi(e); + setLocalArchiveFromFileList(files); + }); + } else { + // Fallbacks to simple file input with multi file selection + useLegacyFilePicker(); + } +} + +/** + * Adds a event listener to the file input to handle file selection (if no other file picker is supported) + */ +function useLegacyFilePicker () { + // Fallbacks to simple file input with multi file selection + document.getElementById('archiveFiles').addEventListener('change', async function (e) { + if (params.isWebkitDirApiSupported || params.isFileSystemApiSupported) { + const activeFilename = e.target.files[0].name; + localStorage.setItem('zimFilenames', [activeFilename].join('|')); + await abstractFilesystemAccess.updateZimDropdownOptions([activeFilename], activeFilename); + } + setLocalArchiveFromFileSelect(); + }); } function handleGlobalDragover (e) { @@ -1321,17 +1419,29 @@ function handleIframeDrop (e) { e.preventDefault(); } -function handleFileDrop (packet) { +async function handleFileDrop (packet) { packet.stopPropagation(); packet.preventDefault(); configDropZone.style.border = ''; var files = packet.dataTransfer.files; - document.getElementById('openLocalFiles').style.display = 'none'; + document.getElementById('selectInstructions').style.display = 'none'; + document.getElementById('fileSelectionButtonContainer').style.display = 'none'; document.getElementById('downloadInstruction').style.display = 'none'; document.getElementById('selectorsDisplay').style.display = 'inline'; - setLocalArchiveFromFileList(files); - // This clears the display of any previously picked archive in the file selector document.getElementById('archiveFiles').value = null; + + // value will be set to true if a folder is dropped then there will be no need to + // call the `setLocalArchiveFromFileList` + let loadZim = true; + + // no previous file will be loaded in case of FileSystemApi + if (params.isFileSystemApiSupported) loadZim = await abstractFilesystemAccess.handleFolderOrFileDropViaFileSystemAPI(packet); + else if (params.isWebkitDirApiSupported) { + const ret = await abstractFilesystemAccess.handleFolderOrFileDropViaWebkit(packet); + loadZim = ret.loadZim; + webKitFileList = ret.files; + } + if (loadZim) setLocalArchiveFromFileList(files); } document.getElementById('libraryBtn').addEventListener('click', function (e) { @@ -1353,7 +1463,9 @@ document.getElementById('libraryBtn').addEventListener('click', function (e) { // Add event listener to link which allows user to show file selectors document.getElementById('selectorsDisplayLink').addEventListener('click', function (e) { e.preventDefault(); - document.getElementById('openLocalFiles').style.display = 'block'; + document.getElementById('selectInstructions').style.display = 'block'; + document.getElementById('downloadInstruction').style.display = 'block'; + document.getElementById('fileSelectionButtonContainer').style.display = 'block'; document.getElementById('selectorsDisplay').style.display = 'none'; }); @@ -1392,6 +1504,7 @@ function archiveReadyCallback (archive) { function setLocalArchiveFromFileSelect () { setLocalArchiveFromFileList(document.getElementById('archiveFiles').files); } +window.setLocalArchiveFromFileSelect = setLocalArchiveFromFileSelect; /** * Reads a remote archive with given URL, and returns the response in a Promise. diff --git a/www/js/init.js b/www/js/init.js index 199999a13..d670f89c7 100644 --- a/www/js/init.js +++ b/www/js/init.js @@ -51,6 +51,10 @@ * @property {boolean} useCanvasElementsForWebpTranscoding - A parameter to circumvent anti-fingerprinting technology in browsers that do not support WebP natively by substituting images directly with the canvas elements produced by the WebP polyfill. * @property {string} libraryUrl - The URL of the Kiwix library. * @property {string} altLibraryUrl - The alternative URL of the Kiwix library in non-supported browsers. + * @property {string} cacheAPI - Name of the prefix used to identify the cache in Cache API + * @property {string} cacheIDB - Name of the Indexed DB database + * @property {boolean} isFileSystemApiSupported - A boolean indicating whether the FileSystem API is supported. + * @property {boolean} isWebkitDirApiSupported - A boolean indicating whether the Webkit Directory API is supported. * @property {DecompressorAPI} decompressorAPI /** @@ -115,6 +119,10 @@ params['contentInjectionMode'] = getSetting('contentInjectionMode') || params['useCanvasElementsForWebpTranscoding'] = null; // Value is determined in uiUtil.determineCanvasElementsWorkaround(), called when setting the content injection mode params['libraryUrl'] = 'https://library.kiwix.org/'; // Url for iframe that will be loaded to download new zim files params['altLibraryUrl'] = 'https://download.kiwix.org/zim/'; // Alternative Url for iframe (for use with unsupported browsers) that will be loaded to download new zim files +params['cacheAPI'] = 'kiwix-js'; // Sets name of the prefix used to identify the cache in Cache API +params['cacheIDB'] = 'kiwix-zim'; // Sets name of the Indexed DB database +params['isFileSystemApiSupported'] = typeof window.showOpenFilePicker === 'function'; // Sets a boolean indicating whether the FileSystem API is supported +params['isWebkitDirApiSupported'] = 'webkitdirectory' in document.createElement('input'); // Sets a boolean indicating whether the Webkit Directory API is supported /** * Apply any override parameters that might be in the querystring. diff --git a/www/js/lib/abstractFilesystemAccess.js b/www/js/lib/abstractFilesystemAccess.js index 03b58ddaf..63d35752a 100644 --- a/www/js/lib/abstractFilesystemAccess.js +++ b/www/js/lib/abstractFilesystemAccess.js @@ -1,36 +1,38 @@ /** * abstractFilesystemAccess.js: Abstraction layer for file access. - * This is currently only implemented for FirefoxOS, but could be extended to + * This is currently only implemented for FirefoxOS and Standard browser (using File System Access API), but could be extended to * Cordova, Electron or other ways to directly browse and read files from the * filesystem. - * It is unfortunately not possible to do that inside a standard browser - * (even inside an extension). - * - * Copyright 2014 Kiwix developers - * Licence GPL v3: - * - * This file is part of Kiwix. - * - * Kiwix is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public Licence as published by - * the Free Software Foundation, either version 3 of the Licence, or - * (at your option) any later version. - * - * Kiwix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* +* Copyright 2014-2023 Kiwix developers and Rishabhg71 +* Licence GPL v3: +* +* This file is part of Kiwix. +* +* Kiwix is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public Licence as published by +* the Free Software Foundation, either version 3 of the Licence, or +* (at your option) any later version. +* +* Kiwix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public Licence for more details. - * - * You should have received a copy of the GNU General Public Licence - * along with Kiwix (file LICENSE-GPLv3.txt). If not, see - */ +* +* You should have received a copy of the GNU General Public Licence +* along with Kiwix (file LICENSE-GPLv3.txt). If not, see +*/ 'use strict'; -function StorageFirefoxOS(storage) { +import cache from './cache.js'; +import translateUI from './translateUI.js'; + +function StorageFirefoxOS (storage) { this._storage = storage; this.storageName = storage.storageName; } + /** * Access the given file. * @param {String} path absolute path to the file @@ -88,6 +90,247 @@ StorageFirefoxOS.prototype.enumerate = function (path) { return this._storage.enumerate(); }; +// refer to this article for easy explanation of File System API https://developer.chrome.com/articles/file-system-access/ + +/** + * @param {Array} files All the File names to be shown in the dropdown + * @param {string} selectedFile The name of the file to be selected in the dropdown + * @returns {Promise>} Array of unique filenames (if a split zim is considered a single file) + */ +async function updateZimDropdownOptions (files, selectedFile) { + const isFireFoxOsNativeFileApiAvailable = typeof navigator.getDeviceStorages === 'function'; + // This will make sure that there is no race around condition when platform is firefox os + // as other function will handle the dropdown UI updates + if (isFireFoxOsNativeFileApiAvailable) return // do nothing let other function handle it + + const select = document.getElementById('archiveList'); + const options = []; + let count = 0; + select.innerHTML = ''; + if (files.length !== 0) { + const placeholderOption = new Option(translateUI.t('configure-select-file-first-option'), ''); + placeholderOption.disabled = true; + select.appendChild(placeholderOption); + }; + + files.forEach((fileName) => { + if (fileName.endsWith('.zim') || fileName.endsWith('.zimaa')) { + options.push(new Option(fileName, fileName)); + select.appendChild(new Option(fileName, fileName)); + count++; + } + }); + document.getElementById('archiveList').value = selectedFile; + document.getElementById('numberOfFilesCount').style.display = ''; + document.getElementById('fileCountDisplay').style.display = ''; + + document.getElementById('numberOfFilesCount').innerText = count.toString(); + document.getElementById('fileCountDisplay').innerText = translateUI.t('configure-select-file-numbers'); +} + +/** + * Opens the File System API to select a directory + * @returns {Promise>} Previously selected file if available in selected folder + */ +async function selectDirectoryFromPickerViaFileSystemApi () { + const handle = await window.showDirectoryPicker(); + const fileNames = []; + const previousZimFile = [] + + const lastZimNameWithoutExtension = (localStorage.getItem('previousZimFileName') ?? '').replace(/\.zim\w\w$/i, ''); + const regex = new RegExp(`\\${lastZimNameWithoutExtension}.zim\\w\\w$`, 'i'); + + for await (const entry of handle.values()) { + fileNames.push(entry.name); + if (regex.test(entry.name) || entry.name === (localStorage.getItem('previousZimFileName') ?? '')) previousZimFile.push(await entry.getFile()); + } + + localStorage.setItem('zimFilenames', fileNames.join('|')); + updateZimDropdownOptions(fileNames, previousZimFile.length !== 0 ? localStorage.getItem('previousZimFileName') : ''); + cache.idxDB('zimFiles', handle, function () { + // save file in DB + }); + return previousZimFile; +} + +/** + * Opens the File System API to select a file + * @returns {Promise>} The selected file from picker + */ +async function selectFileFromPickerViaFileSystemApi () { + const fileHandles = await window.showOpenFilePicker({ multiple: false }); + const [selectedFile] = fileHandles; + const file = await selectedFile.getFile(); + const filenameList = [selectedFile.name]; + localStorage.setItem('zimFilenames', filenameList.join('|')); + cache.idxDB('zimFiles', selectedFile, function () { + // file saved in DB + updateZimDropdownOptions(filenameList, selectedFile.name); + }); + return [file]; +} + +/** + * Gets the selected zim file from the IndexedDB + * @param {string} selectedFilename The name of the file to get back from DB + * @returns {Promise>} The selected File Object from cache + */ +function getSelectedZimFromCache (selectedFilename) { + return new Promise((resolve, _reject) => { + cache.idxDB('zimFiles', async function (fileOrDirHandle) { + // Left it here for debugging purposes as its sometimes asking for permission even when its granted + console.debug('FileHandle and Permission', fileOrDirHandle, await fileOrDirHandle.queryPermission()) + if ((await fileOrDirHandle.queryPermission()) !== 'granted') await fileOrDirHandle.requestPermission(); + + if (fileOrDirHandle.kind === 'directory') { + const files = []; + for await (const entry of fileOrDirHandle.values()) { + const filenameWithoutExtension = selectedFilename.replace(/\.zim\w\w$/i, ''); + const regex = new RegExp(`\\${filenameWithoutExtension}.zim\\w\\w$`, 'i'); + if (regex.test(entry.name) || entry.name === selectedFilename) { + files.push(await entry.getFile()); + } + } + resolve(files); + } else { + const file = await fileOrDirHandle.getFile(); + resolve([file]); + } + }); + }); +} + +/** + * @typedef {Object.} WebkitFileList + */ + +/** + * Gets the selected zim file from the WebkitFileList + * @param {WebkitFileList} webKitFileList The WebkitFileList to get the selected file from + * @param {string} filename The name of the file to get back from webkitFileList + * @returns {Array} The selected Files Object from webkitFileList + */ +function getSelectedZimFromWebkitList (webKitFileList, filename) { + const filenameWithoutExtension = filename.replace(/\.zim\w\w$/i, ''); + + const regex = new RegExp(`\\${filenameWithoutExtension}.zim\\w\\w$`, 'i'); + const files = []; + for (const file of webKitFileList) { + if (regex.test(file.name) || file.name === filename) { + files.push(file); + } + } + return files; +} + +/** + * Loads the Previously loaded zim filename(s) via local storage + */ +function loadPreviousZimFile () { + // If we call `updateZimDropdownOptions` without any delay it will run before the internationalization is initialized + // It's a bit hacky but it works and I am not sure if there is any other way ATM + setTimeout(() => { + if (window.params.isFileSystemApiSupported || window.params.isWebkitDirApiSupported) { + const filenames = localStorage.getItem('zimFilenames'); + if (filenames) updateZimDropdownOptions(filenames.split('|'), ''); + } + }, 200); +} + +/** + * Handles the folder drop event via File System API + * @param {DragEvent} packet The DragEvent packet + * @returns {Promise} Whether the dropped item is a file or directory + */ +async function handleFolderOrFileDropViaFileSystemAPI (packet) { + if (!window.params.isFileSystemApiSupported) return true; + + // Only runs when browser support File System API + const fileInfo = packet.dataTransfer.items[0]; + const fileOrDirHandle = await fileInfo.getAsFileSystemHandle(); + if (fileOrDirHandle.kind === 'file') { + localStorage.setItem([fileOrDirHandle.name], [fileOrDirHandle.name].join('|')); + cache.idxDB('zimFiles', fileOrDirHandle, function () { + // save file in DB + updateZimDropdownOptions([fileOrDirHandle.name], fileOrDirHandle.name); + }); + localStorage.setItem('zimFilenames', [fileOrDirHandle.name].join('|')); + return true; + } + if (fileOrDirHandle.kind === 'directory') { + const fileNames = []; + for await (const entry of fileOrDirHandle.values()) { + fileNames.push(entry.name); + } + localStorage.setItem('zimFilenames', fileNames.join('|')); + cache.idxDB('zimFiles', fileOrDirHandle, function () { + updateZimDropdownOptions(fileNames, ''); + // save file in DB + }); + return false; + } +} + +/** + * Handles the folder drop event via WebkitGetAsEntry + * @param {DragEvent} event The DragEvent packet + * @returns {Promise<{loadZim: boolean, files: Array} | void>} Whether the dropped item is a file or directory and FileList + */ +async function handleFolderOrFileDropViaWebkit (event) { + var dt = event.dataTransfer; + + var entry = dt.items[0].webkitGetAsEntry(); + if (entry.isFile) { + localStorage.setItem('zimFilenames', [entry.name].join('|')); + await updateZimDropdownOptions([entry.name], entry.name); + return { loadZim: true, files: [entry.file] }; + } else if (entry.isDirectory) { + var reader = entry.createReader(); + const files = await getFilesFromReader(reader); + const fileNames = []; + files.forEach((file) => fileNames.push(file.name)); + localStorage.setItem('zimFilenames', fileNames.join('|')); + await updateZimDropdownOptions(fileNames, ''); + return { loadZim: false, files: files }; + } +} + +/** + * Gets the files from the FileSystemReader + * @param {FileSystemDirectoryReader} reader The FileSystemReader to get files from + * @returns {Promise>} The files from the reader + */ +async function getFilesFromReader (reader) { + const files = []; + const promise = new Promise(function (resolve, _reject) { + reader.readEntries(function (entries) { + resolve(entries); + }); + }); + const entries = await promise; + + for (let index = 0; index < entries.length; index++) { + const fileOrDir = entries[index]; + if (fileOrDir.isFile) { + const filePromise = await new Promise(function (resolve, _reject) { + fileOrDir.file(function (file) { + resolve(file); + }); + }); + files.push(filePromise); + } + } + return files; +} + export default { - StorageFirefoxOS: StorageFirefoxOS -}; \ No newline at end of file + StorageFirefoxOS: StorageFirefoxOS, + updateZimDropdownOptions: updateZimDropdownOptions, + selectDirectoryFromPickerViaFileSystemApi: selectDirectoryFromPickerViaFileSystemApi, + selectFileFromPickerViaFileSystemApi: selectFileFromPickerViaFileSystemApi, + getSelectedZimFromCache: getSelectedZimFromCache, + loadPreviousZimFile: loadPreviousZimFile, + handleFolderOrFileDropViaWebkit: handleFolderOrFileDropViaWebkit, + handleFolderOrFileDropViaFileSystemAPI: handleFolderOrFileDropViaFileSystemAPI, + getSelectedZimFromWebkitList: getSelectedZimFromWebkitList +}; diff --git a/www/js/lib/cache.js b/www/js/lib/cache.js new file mode 100644 index 000000000..1335c3bab --- /dev/null +++ b/www/js/lib/cache.js @@ -0,0 +1,977 @@ +/** + * cache.js : Provide a cache for assets from the ZIM archive using indexedDB, localStorage or memory cache + * + * Copyright 2018 Mossroy, Jaifroid and contributors + * License GPL v3: + * + * This file is part of Kiwix. + * + * Kiwix is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Kiwix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Kiwix (file LICENSE-GPLv3.txt). If not, see + */ + +/* globals params, appstate, caches, assetsCache */ + +'use strict'; +import settingsStore from './settingsStore.js'; +import uiUtil from './uiUtil.js'; + +const CACHEAPI = params.cacheAPI; // Set the database or cache name here, and synchronize with Service Worker +const CACHEIDB = params.cacheIDB; // Slightly different name to disambiguate +var objStore = 'kiwix-assets'; // Name of the object store +const APPCACHE = 'kiwix-appCache-' + params.appVersion; // Ensure this is the same as in Service Worker + +// DEV: Regex below defines the permitted MIME types for the cache; add further types as needed +var regexpMimeTypes = /\b(?:javascript|css|ico|html)\b/; + +/** + * Tests the enviornment's caching capabilities and sets assetsCache.capability to the supported level + * + * @param {Function} callback Function to indicate that the capability level has been set + */ +function test (callback) { + // Test for indexedDB capability + if (typeof assetsCache.capability !== 'undefined') { + callback(true); + return; + } + // Set baseline capability + assetsCache.capability = 'memory'; + idxDB('count', function (result) { + if (result !== false) { + assetsCache.capability = 'indexedDB|' + assetsCache.capability; + } else { + console.log('inexedDB is not supported'); + } + // Test for Cache API + if ('caches' in window && /https?:/i.test(window.location.protocol)) { + assetsCache.capability = 'cacheAPI|' + assetsCache.capability; + } else { + console.log('CacheAPI is not supported' + (/https?:/i.test(window.location.protocol) ? '' + : ' with the ' + window.location.protocol + ' protocol')); + } + // Test for localCache capability (this is a fallback, indexedDB is preferred because it permits more storage) + if (typeof Storage !== 'undefined') { + try { + // If localStorage is really supported, this won't produce an error + // eslint-disable-next-line no-unused-vars + var item = window.localStorage.length; + assetsCache.capability = assetsCache.capability + '|localStorage'; + } catch (err) { + console.log('localStorage is not supported'); + } + } + console.log('Setting storage type to ' + assetsCache.capability.match(/^[^|]+/)[0]); + if (/localStorage/.test(assetsCache.capability)) { + console.debug("DEV: 'UnknownError' may be produced as part of localStorage capability detection"); + } + callback(result); + }); +} + +/** + * Counts the numnber of cached assets + * + * @param {Function} callback which will receive an array containing [cacheType, cacheCount] + */ +function count (callback) { + test(function (result) { + var type = null; + var description = null; + var cacheCount = null; + + switch (assetsCache.capability.match(/^[^|]+/)[0]) { + case 'memory': + type = 'memory'; + description = 'Memory'; + cacheCount = assetsCache.size; + break; + case 'localStorage': + type = 'localStorage'; + description = 'LocalStorage'; + cacheCount = localStorage.length; + break; + case 'indexedDB': + type = 'indexedDB'; + description = 'IndexedDB'; + // Sometimes we already have the count as a result of test, so no need to look again + if (typeof result !== 'boolean' && (result === 0 || result > 0)) { + cacheCount = result; + } else { + idxDB('count', function (cacheCount) { + callback({ type: type, description: description, count: cacheCount }); + }); + } + break; + case 'cacheAPI': + type = 'cacheAPI'; + description = 'CacheAPI'; + caches.open(CACHEAPI).then(function (cache) { + cache.keys().then(function (keys) { + callback({ type: type, description: description, count: keys.length }); + }); + }); + break; + default: + // User has turned off caching + type = 'none'; + description = 'None'; + cacheCount = 'null'; + } + + if (cacheCount || cacheCount === 0) { + callback({ type: type, description: description, count: cacheCount }); + } + }); + // Refresh instructions to Service Worker + if (navigator.serviceWorker && navigator.serviceWorker.controller) { + // Create a Message Channel + var channel = new MessageChannel(); + navigator.serviceWorker.controller.postMessage({ + action: { + assetsCache: params.assetsCache ? 'enable' : 'disable', + appCache: params.appCache ? 'enable' : 'disable', + checkCache: window.location.href + } + }, [channel.port2]); + } +} + +/** + * Opens an IndexedDB database and adds or retrieves a key-value pair to it, or performs utility commands + * on the database + * + * @param {String} keyOrCommand The key of the value to be written or read, or commands 'clear' (clears objStore), + * 'count' (counts number of objects in objStore), 'delete' (deletes a record with key passed in valueOrCallback), + * 'deleteNonCurrent' (deletes all databases that do not match CACHEIDB - but only works in Chromium currently) + * @param {Variable} valueOrCallback The value to write, or a callback function for read and command transactions + * @param {Function} callback Callback for write transactions only - mandatory for delete and write transactions + */ +function idxDB (keyOrCommand, valueOrCallback, callback) { + var value = callback ? valueOrCallback : null; + var rtnFn = callback || valueOrCallback; + if (typeof window.indexedDB === 'undefined') { + rtnFn(false); + return; + } + + // Delete all non-curren IdxDB databases (only works in Chromium currently) + if (keyOrCommand === 'deleteNonCurrent') { + if (indexedDB.databases) { + var result = 0; + indexedDB.databases().then(function (dbs) { + dbs.forEach(function (db) { + if (db.name !== CACHEIDB) { + result++; + indexedDB.deleteDatabase(db.name); + } + }); + }).then(function () { + rtnFn(result); + }); + } else { + rtnFn(false); + } + return; + } + + // Open (or create) the database + var open = indexedDB.open(CACHEIDB, 1); + + open.onerror = function (e) { + // Suppress error reporting if testing (older versions of Firefox support indexedDB but cannot use it with + // the file:// protocol, so will report an error) + if (assetsCache.capability !== 'test') { + console.error('IndexedDB failed to open: ' + open.error.message); + } + rtnFn(false); + }; + + // Create the schema + open.onupgradeneeded = function () { + var db = open.result; + // eslint-disable-next-line no-unused-vars + var store = db.createObjectStore(objStore); + }; + + open.onsuccess = function () { + // Start a new transaction + var db = open.result; + + // Set the store to readwrite or read only according to presence or not of value variable + var tx = value !== null || /clear|delete/.test(keyOrCommand) ? db.transaction(objStore, 'readwrite') : db.transaction(objStore); + var store = tx.objectStore(objStore); + + var processData; + // Process commands + if (keyOrCommand === 'clear') { + // Delete all keys and values in the store + processData = store.clear(); + } else if (keyOrCommand === 'count') { + // Count the objects in the store + processData = store.count(); + } else if (keyOrCommand === 'delete') { + // Delete the record with key set to value + processData = store.delete(value); + } else { + // Request addition or retrieval of data + processData = value !== null ? store.put(value, keyOrCommand) : store.get(keyOrCommand); + } + // Call the callback with the result + processData.onsuccess = function (e) { + if (keyOrCommand === 'delete') { + rtnFn(true); + } else { + rtnFn(processData.result); + } + }; + processData.onerror = function (e) { + console.error('IndexedDB command failed: ' + processData.error); + rtnFn(false); + }; + + // Close the db when the transaction is done + tx.oncomplete = function () { + db.close(); + }; + }; +} + +/** + * Opens a CacheAPI cache and adds or retrieves a key-value pair to it, or performs utility commands + * on the cache. This interface also allows the use of callbacks inside the Cache Promise API for ease of + * interoperability with the interface for idxDB code above. + * + * @param {String} keyOrCommand The key of the value to be written or read, or commands 'clear' (clears cache), + * 'delete' (deletes a record with key passed in valueOrCallback) + * @param {Variable} valueOrCallback The value to write, or a callback function for read and command transactions + * @param {Function} callback Callback for write transactions only + * @param {String} mimetype The MIME type of any content to be stored + */ +function cacheAPI (keyOrCommand, valueOrCallback, callback, mimetype) { + var value = callback ? valueOrCallback : null; + var rtnFn = callback || valueOrCallback; + // Process commands + if (keyOrCommand === 'clear') { + caches.delete(CACHEAPI).then(rtnFn); + } else if (keyOrCommand === 'delete') { + caches.open(CACHEAPI).then(function (cache) { + cache.delete(value).then(rtnFn); + }); + } else if (value === null) { + // Request retrieval of data + caches.open(CACHEAPI).then(function (cache) { + cache.match('../' + keyOrCommand).then(function (response) { + if (!response) { + rtnFn(null); + } else { + response.text().then(function (data) { + rtnFn(data); + }); + } + }).catch(function (err) { + console.error('Unable to match assets from Cache API!', err); + rtnFn(null); + }); + }); + } else { + // Request storing of data in cache + caches.open(CACHEAPI).then(function (cache) { + var contentLength; + if (typeof value === 'string') { + var m = encodeURIComponent(value).match(/%[89ABab]/g); + contentLength = value.length + (m ? m.length : 0); + } else { + contentLength = value.byteLength || value.length; + } + var headers = new Headers(); + if (contentLength) headers.set('Content-Length', contentLength); + // Prevent CORS issues in PWAs + // if (contentLength) headers.set('Access-Control-Allow-Origin', '*'); + headers.set('Content-Security-Policy', 'sandbox allow-scripts allow-same-origin allow-modals allow-popups allow-forms'); + if (mimetype) headers.set('Content-Type', mimetype); + var responseInit = { + status: 200, + statusText: 'OK', + headers: headers + }; + var httpResponse = new Response(value, responseInit); + cache.put('../' + keyOrCommand, httpResponse).then(function () { + rtnFn(true); + }).catch(function (err) { + console.error('Unable to store assets in Cache API!', err); + rtnFn(null); + }); + }); + } +} + +/** + * Stores information about the last visited page in a cookie and, if available, in localStorage or indexedDB + * + * @param {String} zimFile The filename (or name of first file in set) of the ZIM archive + * @param {String} article The URL of the article (including namespace) + * @param {String} content The content of the page to be stored + * @param {Function} callback Callback function to report the outcome of the operation + */ +function setArticle (zimFile, article, content, callback) { + // Prevent storage if user has deselected the option in Configuration + if (!params.rememberLastPage) { + callback(-1); + return; + } + settingsStore.setItem(zimFile, article, Infinity); + setItem(zimFile, content, 'text/html', function (response) { + callback(response); + }); +} + +/** + * Retrieves article contents from cache only if the article's key has been stored in settings store + * (since checking the store is synchronous, it prevents unnecessary async cache lookups) + * + * @param {String} zimFile The filename (or name of first file in set) of the ZIM archive + * @param {String} article The URL of the article to be retrieved (including namespace) + * @param {Function} callback The function to call with the result + */ +function getArticle (zimFile, article, callback) { + if (settingsStore.getItem(zimFile) === article) { + getItem(zimFile, callback); + } else { + callback(false); + } +} + +/** + * Caches the contents of an asset in memory or local storage + * + * @param {String} key The database key of the asset to cache + * @param {String} contents The file contents to be stored in the cache + * @param {String} mimetype The MIME type of the contents + * @param {Function} callback Callback function to report outcome of operation + * @param {Boolean} isAsset Optional indicator that a file is an asset + */ +function setItem (key, contents, mimetype, callback, isAsset) { + // Prevent use of storage if user has deselected the option in Configuration + // or if the asset is of the wrong type + if (params.assetsCache === false || !regexpMimeTypes.test(mimetype)) { + callback(-1); + return; + } + // Check if we're actually setting an article + var keyArticle = key.match(/([^/]+)\/([AC]\/.+$)/); + if (keyArticle && !isAsset && /\bx?html\b/i.test(mimetype) && !/\.(png|gif|jpe?g|css|js|mpe?g|webp|webm|woff2?|eot|mp[43])(\?|$)/i.test(key)) { // We're setting an article, so go to setArticle function + setArticle(keyArticle[1], keyArticle[2], contents, callback); + return; + } + if (/^localStorage/.test(assetsCache.capability)) { + localStorage.setItem(key, contents); + } else { + assetsCache.set(key, contents); + } + if (/^indexedDB/.test(assetsCache.capability)) { + idxDB(key, contents, function (result) { + callback(result); + }); + } else if (/^cacheAPI/.test(assetsCache.capability)) { + cacheAPI(key, contents, function (result) { + callback(result); + }, mimetype); + } else { + callback(key); + } +} + +/** + * Retrieves a ZIM file asset that has been cached with the addItem function + * either from the memory cache or local storage + * + * @param {String} key The database key of the asset to retrieve + * @param {Function} callback The function to call with the result + */ +function getItem (key, callback) { + // Only look up assets of the type stored in the cache + if (params.assetsCache === false) { + callback(false); + return; + } + // Check if we're actually calling an article + // DEV: With new ZIM types, we can't know we're retrieving an article... + // var keyArticle = key.match(/([^/]+)\/(A\/.+$)/); + // if (keyArticle) { // We're retrieving an article, so go to getArticle function + // getArticle(keyArticle[1], keyArticle[2], callback); + // return; + // } + var contents = null; + if (assetsCache.has(key)) { + contents = assetsCache.get(key); + callback(contents); + } else if (/^localStorage/.test(assetsCache.capability)) { + contents = localStorage.getItem(key); + callback(contents); + } else if (/^cacheAPI/.test(assetsCache.capability)) { + cacheAPI(key, function (contents) { + callback(contents); + }); + } else if (/^indexedDB/.test(assetsCache.capability)) { + idxDB(key, function (contents) { + if (typeof contents !== 'undefined') { + // Also store in fast memory cache to prevent repaints + assetsCache.set(key, contents); + } + callback(contents); + }); + } else { + callback(contents); + } +} + +/** + * Gets an item from the cache, or extracts it from the ZIM if it is not cached. After extracting + * an item from the ZIM, it is added to the cache if it is of the type specified in regexpKeyTypes. + * + * @param {Object} selectedArchive The ZIM archive picked by the user + * @param {String} key The cache key of the item to retrieve + * @param {Object} dirEntry If the item's dirEntry has already been looked up, it can optionally be + * supplied here (saves a redundant dirEntry lookup) + * @returns {Promise} A Promise for the content + */ +function getItemFromCacheOrZIM (selectedArchive, key, dirEntry) { + return new Promise(function (resolve, reject) { + // First check if the item is already in the cache + var title = key.replace(/^[^/]+\//, ''); + getItem(key, function (result) { + if (result !== null && result !== false && typeof result !== 'undefined') { + // console.debug("Cache supplied " + title); + if (/\.css$/.test(title)) { + assetsCache.cssLoading--; + if (assetsCache.cssLoading <= 0) { + document.getElementById('articleContent').style.display = 'block'; + } + } + resolve(result); + return; + } + // Bypass getting dirEntry if we already have it + var getDirEntry = dirEntry ? Promise.resolve() + : selectedArchive.getDirEntryByPath(title); + // Read data from ZIM + getDirEntry.then(function (resolvedDirEntry) { + if (dirEntry) resolvedDirEntry = dirEntry; + if (resolvedDirEntry === null) { + console.log('Error: asset file not found: ' + title); + resolve(null); + } else { + var mimetype = resolvedDirEntry.getMimetype(); + if (resolvedDirEntry.nullify) { + console.debug('Zimit filter prevented access to ' + resolvedDirEntry.url + '. Storing empty contents in cache.'); + setItem(key, '', mimetype, function () { }); + resolve(''); + return; + } + var shortTitle = key.replace(/[^/]+\//g, '').substring(0, 18); + // Since there was no result, post UI messages and look up asset in ZIM + if (/\bx?html\b/.test(mimetype) && !resolvedDirEntry.isAsset && + !/\.(png|gif|jpe?g|svg|css|js|mpe?g|webp|webm|woff2?|eot|mp[43])(\?|$)/i.test(resolvedDirEntry.url)) { + uiUtil.pollSpinner('Loading ' + shortTitle + '...'); + } else if (/(css|javascript|video|vtt)/i.test(mimetype)) { + uiUtil.pollSpinner('Getting ' + shortTitle + '...'); + } + // Set the read function to use according to filetype + var readFile = /\b(?:x?html|css|javascript)\b/i.test(mimetype) + ? selectedArchive.readUtf8File : selectedArchive.readBinaryFile; + readFile(resolvedDirEntry, function (fileDirEntry, content) { + if (regexpMimeTypes.test(mimetype)) { + console.debug('Cache retrieved ' + title + ' from ZIM'); + // Process any pre-cache transforms + content = transform(content, title.replace(/^.*\.([^.]+)$/, '$1')); + } + // Hide article while it is rendering + if (!fileDirEntry.isAsset && /\bx?html\b/i.test(mimetype) && !/\.(png|gif|jpe?g|svg|css|js|mpe?g|webp|webm|woff2?|eot|mp[34])(\?|$)/i.test(key)) { + // Count CSS so we can attempt to show article before JS/images are fully loaded + var cssCount = content.match(/<(?:link)[^>]+?href=["']([^"']+)[^>]+>/ig); + assetsCache.cssLoading = cssCount ? cssCount.length : 0; + if (assetsCache.cssLoading) document.getElementById('articleContent').style.display = 'none'; + } + if (/\bcss\b/i.test(mimetype)) { + assetsCache.cssLoading--; + if (assetsCache.cssLoading <= 0) { + document.getElementById('articleContent').style.display = 'block'; + } + } + setItem(key, content, mimetype, function (result) { + if (result === -1) { + // Cache rejected item due to user settings + } else if (result) { + console.log('Cache: stored asset ' + title); + } else { + console.error('Cache: failed to store asset ' + title); + } + }, fileDirEntry.isAsset); + resolve(content); + }); + } + }).catch(function (e) { + reject(new Error('Could not find DirEntry for asset ' + title, e)); + }); + }); + }); +} + +/** + * Clears caches (including cookie) according to the scope represented by the 'items' variable + * + * @param {String} items 'lastpages' (last visited pages of various archives), 'all' or 'reset' + * @param {Function} callback Callback function to report the number of items cleared + */ +function clear (items, callback) { + if (!/lastpages|all|reset/.test(items)) { + if (callback) callback(false); + return; + } + // Delete cookie entries with a key containing '.zim' or '.zimaa' etc. followed by article namespace + var itemsCount = 0; + var key; + var capability = assetsCache.capability; + var zimRegExp = /(?:^|;)\s*([^=]+)=([^;]*)/ig; + var currentCookies = document.cookie; + var cookieCrumb = zimRegExp.exec(currentCookies); + while (cookieCrumb !== null) { + if (/\.zim\w{0,2}=/i.test(decodeURIComponent(cookieCrumb[0]))) { + key = cookieCrumb[1]; + // This expiry date will cause the browser to delete the cookie on next page refresh + document.cookie = key + '=;expires=Thu, 21 Sep 1979 00:00:01 UTC;'; + if (items === 'lastpages') { + assetsCache.delete(key); + // See note on loose test below + if (/localStorage/.test(capability)) { + localStorage.removeItem(key); + } + if (/indexedDB/.test(capability)) { + idxDB('delete', key, function () { }); + } + if (/cacheAPI/.test(capability)) { + cacheAPI('delete', key, function () { }); + } + itemsCount++; + } + } + cookieCrumb = zimRegExp.exec(currentCookies); + } + if (items === 'all' || items === 'reset') { + var result; + if (/^(memory|indexedDB|cacheAPI)/.test(capability)) { + itemsCount += assetsCache.size; + result = 'assetsCache'; + } + // Delete and reinitialize assetsCache + assetsCache = new Map(); + assetsCache.capability = capability; + // Loose test here ensures we clear localStorage even if it wasn't being used in this session + if (/localStorage/.test(capability)) { + if (items === 'reset') { + itemsCount += localStorage.length; + localStorage.clear(); + } else { + for (var i = localStorage.length; i--;) { + key = localStorage.key(i); + if (/\.zim\w{0,2}/i.test(key)) { + localStorage.removeItem(key); + itemsCount++; + } + } + } + result = result ? result + ' and localStorage' : 'localStorage'; + } + // Loose test here ensures we clear indexedDB even if it wasn't being used in this session + if (/indexedDB/.test(capability)) { + result = result ? result + ' and indexedDB' : 'indexedDB'; + idxDB('count', function (number) { + itemsCount += number; + idxDB('clear', function () { + result = result ? result + ' (' + itemsCount + ' items deleted)' : 'no assets to delete'; + console.log('cache.clear: ' + result); + if (!/^cacheAPI/.test(capability) && callback) callback(itemsCount); + }); + }); + } + // No need to use loose test here because cacheAPI trumps the others + if (/^cacheAPI/.test(capability)) { + result = result ? result + ' and cacheAPI' : 'cacheAPI'; + count(function (number) { + itemsCount += number[1]; + cacheAPI('clear', function () { + result = result ? result + ' (' + itemsCount + ' items deleted)' : 'no assets to delete'; + console.log('cache.clear: ' + result); + if (callback) callback(itemsCount); + }); + }); + } + } + if (!/^cacheAPI|indexedDB/.test(capability)) { + result = result ? result + ' (' + itemsCount + ' items deleted)' : 'no assets to delete'; + console.log('cache.clear: ' + result); + if (callback) callback(itemsCount); + } +} + +/** + * Replaces all assets that have the given attribute in the html string with inline tags containing content + * from the cache entries corresponding to the given zimFile + * Function is intended for link or script tags, but could be extended + * Returns the substituted html in the callback function (even if no substitutions were made) + * + * @param {String} html The html string to process + * @param {String} tags The html tag or tags ('link|script') containing the asset to replace; + * multiple tags must be separated with a pipe + * @param {String} attribute The attribute that stores the URL to be substituted + * @param {String} zimFile The name of the ZIM file (or first file in the file set) + * @param {Object} selectedArchive The archive selected by the user in app.js + * @param {Function} callback The function to call with the substituted html + */ +function replaceAssetRefsWithUri (html, tags, attribute, zimFile, selectedArchive, callback) { + // Creates an array of all link tags that have the given attribute + var regexpTagsWithAttribute = new RegExp('<(?:' + tags + ')[^>]+?' + attribute + '=["\']([^"\']+)[^>]+>', 'ig'); + var titles = []; + var tagArray = regexpTagsWithAttribute.exec(html); + while (tagArray !== null) { + titles.push([tagArray[0], + decodeURIComponent(tagArray[1])]); + tagArray = regexpTagsWithAttribute.exec(html); + } + if (!titles.length) { + callback(html); + } + // Iterate through the erray of titles, populating the HTML string with substituted tags containing + // a reference to the content from the Cache or from the ZIM + assetsCache.busy = titles.length; + titles.forEach(function (title) { + getItemFromCacheOrZIM(selectedArchive, zimFile + '/' + title[1], function (assetContent) { + assetsCache.busy--; + if (assetContent || assetContent === '') { + var newAssetTag = uiUtil.createNewAssetElement(title[0], attribute, assetContent); + html = html.replace(title[0], newAssetTag); + } + if (!assetsCache.busy) callback(html); + }); + }); +} + +/** + * Provides "Server Side" transformation of textual content "served" to app.js + * For performance reasons, this is only hooked into content extracted from the ZIM: the transformed + * content will then be cached in its transformed state + * + * @param {String} string The string to transform + * @param {String} filter An optional filter: only transforms which match the filter will be executed + * @returns {String} The tranformed content + */ +function transform (string, filter) { + switch (filter) { + case 'html': + // Filter to remove any BOM (causes quirks mode in browser) + string = string.replace(/^[^<]*/, ''); + + // Filter to open all heading sections + string = string.replace(/(class=["'][^"']*?collapsible-(?:heading|block)(?!\s+open-block))/g, + '$1 open-block'); + + break; + } + return string; +} + +/** + * Provide method to verify File System Access API permissions + * + * @param {Object} fileHandle The file handle that we wish to verify with the Native Filesystem API + * @param {Boolean} withWrite Indicates read only or read/write persmissions + * @returns {Promise} A Promise for a Boolean value indicating whether permission has been granted or not + */ +function verifyPermission (fileHandle, withWrite) { + if (params.useOPFS) return Promise.resolve(true); // No permission prompt required for OPFS + var opts = withWrite ? { mode: 'readwrite' } : {}; + return fileHandle.queryPermission(opts).then(function (permission) { + if (permission === 'granted') return true; + return fileHandle.requestPermission(opts).then(function (permission) { + if (permission === 'granted') return true; + console.error('Permission for ' + fileHandle.name + ' was not granted: ' + permission); + return false; + }).catch(function (error) { + console.warn('Cannot use previously picked file handle programmatically (this is normal) ' + fileHandle.name, error); + }); + }); +} + +/** + * Download an archive directly into the picked folder (primarily for use with the Origin Private File System) + * + * @param {String} archiveName The name of the archive to download (will be used as the filename) + * @param {String} archiveUrl An optional URL to download the archive from (if not supplied, will use params.kiwixDownloadLink) + * @param {Function} callback Callback function to report the progress of the download + * @returns {Promise} A Promise for a FileHandle object representing the downloaded file + */ +function downloadArchiveToPickedFolder (archiveName, archiveUrl, callback) { + archiveUrl = archiveUrl || params.kiwixDownloadLink + archiveName; + if (params.pickedFolder && params.pickedFolder.getFileHandle) { + return verifyPermission(params.pickedFolder, true).then(function (permission) { + if (permission) { + return params.pickedFolder.getFileHandle(archiveName, { create: true }).then(function (fileHandle) { + return fileHandle.createWritable().then(function (writer) { + return fetch(archiveUrl).then(function (response) { + if (!response.ok) { + return writer.close().then(function () { + // Delete the file + params.pickedFolder.removeEntry(archiveName).then(function () { + throw new Error('HTTP error, status = ' + response.status); + }); + }); + } + var loaded = 0; + var reported = 0; + return new Response( + new ReadableStream({ + start: function (controller) { + var reader = response.body.getReader(); + var processResult = function (result) { + if (result.done) { + return controller.close(); + } + loaded += result.value.byteLength; + if (loaded - reported >= 1048576) { // 1024 * 1024 + reported = loaded; + if (callback) { + callback(reported); + } else console.debug('Downloaded ' + reported + ' bytes so far...'); + } + controller.enqueue(result.value); + return reader.read().then(processResult); + }; + return reader.read().then(processResult); + } + }) + ).body.pipeTo(writer).then(function () { + if (callback) callback('completed'); + return true; + }).catch(function (err) { + console.error('Error downloading archive', err); + if (callback) callback('error'); + writer.close().then(function () { + // Delete the file + params.pickedFolder.removeEntry(archiveName).then(function () { + throw err; + }); + }); + }); + }); + }); + }); + } else { + throw (new Error('Write permission not granted!')); + } + }).catch(function (err) { + console.error('Error downloading archive', err); + throw err; + }); + } else { + return Promise.reject(new Error('No picked folder available!')); + } +} + +/** + * Imports the picked files into the OPFS file system + * + * @param {Array} files An array of File objects to import + */ +function importOPFSEntries (files) { + return Promise.all(files.map(function (file) { + return params.pickedFolder.getFileHandle(file.name, { create: true }).then(function (fileHandle) { + return fileHandle.createWritable().then(function (writer) { + uiUtil.pollOpsPanel('Please wait: Importing ' + file.name + '...', true); + return writer.write(file).then(function () { + uiUtil.pollOpsPanel('Please wait: Imported ' + file.name + '...', true); + return writer.close(); + }); + }); + }); + })); +} + +/** + * Exports an entry from the OPFS file system to the user-visible file system + * + * @param {String} name The filename of the entry to export + * @returns {Promise} A Promise for a Boolean value indicating whether the export was successful + */ +function exportOPFSEntry (name) { + if (navigator && navigator.storage && 'getDirectory' in navigator.storage) { + return navigator.storage.getDirectory().then(function (dir) { + return dir.getFileHandle(name).then(function (fileHandle) { + try { + // Obtain a file handle to a new file in the user-visible file system + // with the same name as the file in the origin private file system. + return window.showSaveFilePicker({ + suggestedName: fileHandle.name || '' + }).then(function (saveHandle) { + return saveHandle.createWritable().then(function (writable) { + return fileHandle.getFile().then(function (file) { + return writable.write(file).then(function () { + writable.close(); + return true; + }); + }); + }); + }); + } catch (err) { + console.error(err.name, err.message); + return false; + } + }).catch(function (err) { + console.error('Unable to get file handle from OPFS', err); + return false; + }); + }).catch(function (err) { + console.error('Unable to get directory from OPFS', err); + return false; + }); + } +} + +/** + * Deletes an entry from the OPFS file system + * + * @param {String} name The filename of the entry to delete + */ +function deleteOPFSEntry (name) { + if (navigator && navigator.storage && 'getDirectory' in navigator.storage) { + return navigator.storage.getDirectory().then(function (dirHandle) { + return iterateOPFSEntries().then(function (entries) { + var baseName = name.replace(/\.zim[^.]*$/i, ''); + entries.forEach(function (entry) { + if (~entry.name.indexOf(baseName)) { + return dirHandle.removeEntry(entry.name).then(function () { + console.log('Deleted ' + entry.name + ' from OPFS'); + populateOPFSStorageQuota(); + }).catch(function (err) { + console.error('Unable to delete ' + entry.name + ' from OPFS', err); + }); + } + }); + }).catch(function (err) { + console.error('Unable to get directory from OPFS', err); + }); + }).catch(function (err) { + console.error('Unable to get directory from OPFS', err); + }); + } +} + +/** + * Iterates the OPFS file system and returns an array of entries found + * + * @returns {Promise} A Promise for an array of entries in the OPFS file system + */ +function iterateOPFSEntries () { + if (navigator && navigator.storage && 'getDirectory' in navigator.storage) { + return navigator.storage.getDirectory().then(function (dirHandle) { + var archiveEntries = []; + var entries = dirHandle.entries(); + var promisesForEntries = []; + // Push the pormise for each entry to the promises array + var pushPromises = new Promise(function (resolve) { + (function iterate () { + return entries.next().then(function (result) { + if (!result.done) { + // Process the entry, then continue iterating + var entry = result.value[1]; + archiveEntries.push(entry); + promisesForEntries.push(result); + iterate(); + } else { + return resolve(true); + } + }); + })(); + }); + return pushPromises.then(function () { + return Promise.all(promisesForEntries).then(function () { + return archiveEntries; + }).catch(function (err) { + console.error('Unable to iterate OPFS entries', err); + }); + }); + }); + } +} + +/** + * Gets the OPFS storage quota and populates the OPFSQuota panel + * + * @returns {Promise} A Promise that populates the OPFSQuota panel + */ +function populateOPFSStorageQuota () { + if (navigator && navigator.storage && ('estimate' in navigator.storage)) { + return navigator.storage.estimate().then(function (estimate) { + var percent = ((estimate.usage / estimate.quota) * 100).toFixed(2); + appstate.OPFSQuota = estimate.quota - estimate.usage; + document.getElementById('OPFSQuota').innerHTML = + 'OPFS storage quota:
Used: ' + percent + '%; Remaining: ' + + (appstate.OPFSQuota / 1024 / 1024 / 1024).toFixed(2) + ' GB'; + }); + } +} + +/** + * Wraps a semaphor in a Promise. A function can signal that it is done by setting a sempahor to true, + * if it has first set it to false at the outset of the procedure. Ensure no other functions use the same + * sempahor. The semaphor must be an object key of the app-wide assetsCache object. + * + * @param {String} semaphor The name of a semaphor key in the assetsCache object + * @param {String|Object} value An optional value or object to pass in the resolved promise + * @returns {Promise} A promise that resolves when assetsCache[semaphor] is true + */ +function wait (semaphor, value) { + var p = new Promise(function (resolve) { + setTimeout(function awaitCache () { + if (assetsCache[semaphor]) { + return resolve(value); + } + setTimeout(awaitCache, 300); + }, 0); + }); + return p; +} + +export default { + APPCACHE: APPCACHE, + CACHEAPI: CACHEAPI, + test: test, + count: count, + idxDB: idxDB, // only this is used in fileSystem.js + cacheAPI: cacheAPI, + setArticle: setArticle, + getArticle: getArticle, + setItem: setItem, + getItem: getItem, + clear: clear, + wait: wait, + getItemFromCacheOrZIM: getItemFromCacheOrZIM, + replaceAssetRefsWithUri: replaceAssetRefsWithUri, + verifyPermission: verifyPermission, + downloadArchiveToPickedFolder: downloadArchiveToPickedFolder, + importOPFSEntries: importOPFSEntries, + exportOPFSEntry: exportOPFSEntry, + deleteOPFSEntry: deleteOPFSEntry, + iterateOPFSEntries: iterateOPFSEntries, + populateOPFSStorageQuota: populateOPFSStorageQuota +}; diff --git a/www/js/lib/zimfile.js b/www/js/lib/zimfile.js index f84651268..54b7284db 100644 --- a/www/js/lib/zimfile.js +++ b/www/js/lib/zimfile.js @@ -54,14 +54,7 @@ if (!String.prototype.startsWith) { }); } -/** - * A global variable to track the assembler machine type and the last used decompressor (for reporting to the API panel) - * This is populated in the Emscripten wrappers - * @type {Object} - * @property {String} assemblerMachineType The assembler machine type supported and/or loaded by this app: 'ASM' or 'WASM' - * @property {String} decompressorLastUsed The decompressor that was last used to decode a compressed cluster (currently 'XZ' or 'ZSTD') - * @property {String} errorStatus A description of any detected error in loading a decompressor - */ +// to learn more read init.js:57 or search DecompressorAPI in init.js params.decompressorAPI = { assemblerMachineType: null, decompressorLastUsed: null,