Writediraszip refactoring (#1932)

* Make some function in tipFiles async * Fix prettier * Small fix for tools * Fix getConfigFileName * Fix getConfigFileName * Fixed suggestions * Fixed suggestions * Show which ID has error * Save by default unknown files as binary * Types as dev dep in root package * pack lock update * Fix prettier * Play with tests * Fix eslint * Fix tests * Fix tests * Fix tests * Fix tests Co-authored-by: GermanBluefox <dogafox@gmail.com>
ioBroker · Aug 24, 2022 · dc10033 · dc10033
1 parent ab2693a
commit dc10033
Show file tree

Hide file tree

Showing 13 changed files with 126 additions and 156 deletions.
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -13,6 +13,11 @@ on:
   schedule:
     - cron: '0 0 * * 0' # weekly
 
+# Cancel previous PR/branch runs when a new commit is pushed
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   lint:
     name: Eslint

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/cli/src/lib/setup/setupInstall.ts b/packages/cli/src/lib/setup/setupInstall.ts
@@ -1649,7 +1649,7 @@ class Install {
                     console.log(
                         `Info: Can not get current GitHub commit, only remember that we installed from GitHub: ${err.message}`
                     );
-                    // Install using the npm Github URL syntax `npm i user/repo_name`:
+                    // Install using the npm GitHub URL syntax `npm i user/repo_name`:
                     url = `${user}/${repo}`;
                 }
             } else {
@@ -1675,7 +1675,7 @@ class Install {
                 }
             } else {
                 const githubUrlParts = tools.parseShortGithubUrl(url);
-                // Try to extract the adapter name from the github url if possible
+                // Try to extract the adapter name from the GitHub url if possible
                 // Otherwise fall back to the complete URL
                 if (githubUrlParts) {
                     name = githubUrlParts.repo;
@@ -1721,7 +1721,7 @@ class Install {
             await this.upload.uploadAdapter(name, false, true);
             await this.upload.upgradeAdapterObjects(name);
         } else {
-            // Try to find io-package.json with newest date
+            // Try to find io-package.json with the newest date
             const dirs = fs.readdirSync(installDir);
             let date = null;
             let dir = null;

diff --git a/packages/cli/src/lib/setup/setupList.ts b/packages/cli/src/lib/setup/setupList.ts
@@ -309,10 +309,9 @@ export class List {
                                     const id = obj.value._id;
                                     const type = obj.value.type;
 
-                                    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
                                     console.log(`${id.padStart(39)}: ${type.padStart(39)} - ${name || ''}`);
                                 } else {
-                                    // @ts-expect-error if we would have an design object it would have no type
+                                    // @ts-expect-error if we would have a design object it would have no type
                                     console.log(obj.value._id);
                                 }
                             }
@@ -455,7 +454,6 @@ export class List {
                                         host = host.padEnd(40);
                                     }
 
-                                    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
                                     let text = `${id}: ${name || ''}: ${host} - ${
                                         row.value.common.enabled ? ' enabled' : 'disabled'
                                     }`;
@@ -796,7 +794,6 @@ export class List {
                                 );
                                 const id = obj.value._id.substring(5);
 
-                                // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
                                 console.log(`${id.padEnd(19)}(${name})`);
                                 console.log(
                                     '-------------------------------------------------------------------------------------'

diff --git a/packages/common/src/lib/common/maybeCallback.ts b/packages/common/src/lib/common/maybeCallback.ts
@@ -167,7 +167,7 @@ export function maybeCallbackWithRedisError<
  *
  * @param callback - callback function to be executed
  * @param error - error which will be used by the callback function. If callback is not a function and
- * error is given, a rejected Promise is returned. If error is given but it is not an instance of Error, it is converted into one.
+ * error is given, a rejected Promise is returned. If error is given, but it is not an instance of Error, it is converted into one.
  * @param args - as many arguments as needed, which will be returned by the callback function or by the Promise
  * @returns Promise if Promise is resolved with multiple arguments, an array is returned
  */

diff --git a/packages/common/src/lib/common/tools.ts b/packages/common/src/lib/common/tools.ts
@@ -2158,34 +2158,43 @@ export function getDefaultDataDir(): string {
  * Returns the path of the config file
  */
 export function getConfigFileName(): string {
-    let configDir: string | string[] = __dirname.replace(/\\/g, '/');
-    configDir = configDir.split('/');
     const _appName = appName.toLowerCase();
+    let devConfigDir;
 
     if (_isDevInstallation()) {
+        devConfigDir = __dirname.replace(/\\/g, '/');
+        const devConfigParts = devConfigDir.split('/');
+
         // dev install -> Remove /lib
-        configDir.splice(configDir.length - 4, 4);
-        configDir = configDir.join('/');
-        configDir += '/controller'; // go inside controller dir
-        if (fs.existsSync(`${configDir}/conf/${_appName}.json`)) {
-            return `${configDir}/conf/${_appName}.json`;
-        } else {
-            return `${configDir}/data/${_appName}.json`;
+        devConfigParts.splice(devConfigParts.length - 4, 4);
+        devConfigDir = devConfigParts.join('/');
+        devConfigDir += '/controller'; // go inside controller dir
+        if (fs.existsSync(`${devConfigDir}/conf/${_appName}.json`)) {
+            return `${devConfigDir}/conf/${_appName}.json`;
+        } else if (fs.existsSync(`${devConfigDir}/data/${_appName}.json`)) {
+            return `${devConfigDir}/data/${_appName}.json`;
         }
     }
 
+    let configDir = __dirname.replace(/\\/g, '/');
+    const configParts = configDir.split('/');
+
     // if debugging with npm5 -> node_modules on e.g. /opt/node_modules
     if (
         fs.existsSync(`${__dirname}/../../../../../../../../node_modules/${_appName.toLowerCase()}.js-controller`) ||
         fs.existsSync(`${__dirname}/../../../../../../../../node_modules/${_appName}.js-controller`)
     ) {
         // remove /node_modules/' + appName + '.js-controller/lib
-        configDir.splice(configDir.length - 8, 8);
-        configDir = configDir.join('/');
+        configParts.splice(configParts.length - 8, 8);
+        configDir = configParts.join('/');
     } else {
         // If installed with npm -> remove node_modules/@iobroker/js-controller-common/src/lib/common
-        configDir.splice(configDir.length - 6, 6);
-        configDir = configDir.join('/');
+        configParts.splice(configParts.length - 6, 6);
+        configDir = configParts.join('/');
+    }
+
+    if (!fs.existsSync(`${configDir}/${_appName}-data/${_appName}.json`) && devConfigDir) {
+        return `${devConfigDir}/data/${_appName}.json`;
     }
 
     return `${configDir}/${_appName}-data/${_appName}.json`;

diff --git a/packages/controller/lib/zipFiles.js b/packages/controller/lib/zipFiles.js
@@ -168,80 +168,64 @@ function _checkDir(objects, id, root, parts, options, callback) {
     });
 }
 
-async function _writeOneFile(objects, zip, id, name, filename, options, callback) {
-    try {
-        let data = await zip.files[filename].async('nodebuffer');
-        let _err;
-        if (options.parse) {
-            try {
-                data = options.parse(name, filename, data, options ? options.settings : null);
-            } catch (e) {
-                _err = e;
-            }
+async function _writeOneFile(objects, zip, id, name, filename, options) {
+    let data = await zip.files[filename].async('nodebuffer');
+    let _err;
+    if (options.parse) {
+        try {
+            data = options.parse(name, filename, data, options ? options.settings : null);
+        } catch (e) {
+            _err = e;
         }
-        const fName = name + filename;
-        const parts = fName.split('/');
-        parts.pop();
-        _checkDir(objects, id, '', parts, options, () =>
-            objects.writeFile(id, name + filename, data, options, err => callback(_err || err))
-        );
-    } catch (e) {
-        callback(e.message);
     }
+    const fName = name + filename;
+    const parts = fName.split('/');
+    parts.pop();
+    return new Promise((resolve, reject) =>
+        _checkDir(objects, id, '', parts, options, () =>
+            objects.writeFile(id, name + filename, data, options, err =>
+                _err || err ? reject(_err || err) : resolve()
+            )
+        )
+    );
 }
 
-async function writeDirAsZip(objects, id, name, data, options, callback) {
+async function writeDirAsZip(objects, id, name, data, options) {
     JSZip = JSZip || require('jszip');
     const zip = new JSZip();
 
     options = options || {};
 
     let adapter = id;
-    if (adapter.indexOf('.') !== -1) {
+    if (adapter.includes('.')) {
         adapter = id.split('.')[0];
     }
 
     // try to load processor of adapter
     try {
-        options.parse = require(tools.appName + '.' + adapter + '/lib/convert.js').parse;
+        options.parse = require(`${tools.appName}.${adapter}/lib/convert.js`).parse;
     } catch {
         // OK
     }
 
-    try {
-        await zip.loadAsync(data);
-        let count = 0;
-        const error = [];
-        if (name[name.length - 1] !== '/') {
-            name += '/';
-        }
-        for (const filename of Object.keys(zip.files)) {
-            if (!filename || filename[filename.length - 1] === '/') {
-                continue;
-            }
-            count++;
-            try {
-                _writeOneFile(objects, zip, id, name, filename, options, err => {
-                    err && error.push('Cannot write file "' + filename + '":' + err.toString());
-
-                    if (!--count && callback) {
-                        callback(error.length ? error.join(', ') : null);
-                        callback = null;
-                    }
-                });
-            } catch (error) {
-                if (callback) {
-                    callback(error.toString());
-                    callback = null;
-                }
-            }
+    await zip.loadAsync(data);
+    const errors = [];
+    if (name[name.length - 1] !== '/') {
+        name += '/';
+    }
+    for (const filename of Object.keys(zip.files)) {
+        if (!filename || filename[filename.length - 1] === '/') {
+            continue;
         }
-    } catch (error) {
-        if (callback) {
-            callback(error.toString());
-            callback = null;
+        try {
+            await _writeOneFile(objects, zip, id, name, filename, options);
+        } catch (error) {
+            errors.push(`Cannot write file "${filename}": ${error.toString()}`);
         }
     }
+    if (errors.length) {
+        throw new Error(errors.join(', '));
+    }
 }
 
 // pack all files as zip and send it back
@@ -255,7 +239,7 @@ function readObjectsAsZip(objects, rootId, adapter, options, callback) {
     if (adapter) {
         // try to load processor of adapter
         try {
-            options.stringify = require(tools.appName + '.' + adapter + '/lib/convert.js').stringify;
+            options.stringify = require(`${tools.appName}.${adapter}/lib/convert.js`).stringify;
         } catch {
             // OK
         }
@@ -312,13 +296,13 @@ async function _writeOneObject(objects, zip, rootId, filename, options, callback
             try {
                 data.data = JSON.parse(data.data);
             } catch (e) {
-                callback(`Cannot parse "${data.id}": ${e}`);
+                callback(`Cannot parse "${data.id}": ${e.message}`);
                 return;
             }
         }
         if (data && data.id && data.data) {
             options.ts = new Date().getTime();
-            options.from = 'system.host.' + tools.getHostName() + '.cli';
+            options.from = `system.host.${tools.getHostName()}.cli`;
             objects.setObject(data.id, data.data, options, err => callback(err));
         } else {
             if (data && data.error) {
@@ -340,7 +324,7 @@ async function writeObjectsAsZip(objects, rootId, adapter, data, options, callba
     if (adapter) {
         // try to load processor of adapter
         try {
-            options.parse = require(tools.appName + '.' + adapter + '/lib/convert.js').parse;
+            options.parse = require(`${tools.appName}.${adapter}/lib/convert.js`).parse;
         } catch {
             // OK
         }

diff --git a/packages/controller/main.js b/packages/controller/main.js
@@ -2177,7 +2177,7 @@ async function sendTo(objName, command, message, callback) {
     try {
         await states.pushMessage(objName, obj);
     } catch (e) {
-        // do not stringify the object, we had issue invalid string length on serialization
+        // do not stringify the object, we had the issue with the invalid string length on serialization
         logger.error(
             `${hostLogPrefix} [sendTo] Could not push message "${inspect(obj)}" to "${objName}": ${e.message}`
         );
@@ -2872,15 +2872,17 @@ async function processMessage(msg) {
         case 'writeDirAsZip':
             zipFiles = zipFiles || require('./lib/zipFiles');
             try {
-                zipFiles.writeDirAsZip(
+                await zipFiles.writeDirAsZip(
                     objects,
                     msg.message.id,
                     msg.message.name,
                     Buffer.from(msg.message.data, 'base64'),
-                    msg.message.options,
-                    error => msg.callback && msg.from && sendTo(msg.from, msg.command, { error }, msg.callback)
+                    msg.message.options
                 );
+
+                msg.callback && msg.from && sendTo(msg.from, msg.command, {}, msg.callback);
             } catch (error) {
+                logger.error(`${hostLogPrefix} Cannot write zip file as folder: ${error}`);
                 msg.callback && msg.from && sendTo(msg.from, msg.command, { error }, msg.callback);
             }
             break;

diff --git a/packages/controller/test/lib/testFiles.js b/packages/controller/test/lib/testFiles.js
@@ -166,7 +166,7 @@ function register(it, expect, context) {
         /** @type Buffer */
         const { file, mimeType } = await context.adapter.readFileAsync(objId, fileName);
 
-        expect(mimeType).to.be.equal('text/javascript');
+        expect(mimeType).to.be.equal('application/octet-stream');
         expect(file.toString('utf8')).to.be.equal(dataBinary.toString('utf8'));
     });