First steps towards init log reduction

src/backend/doc/extensions/builtins/data.md

@@ -85,7 +85,7 @@ This key will persist for 20 minutes, even if the server restarts.
 ```javascript
 kv.expire({
     key: 'some-key',
-    ttl: 1000 * 60 * 20, // 1 minute
+    ttl: 1000 * 60 * 20, // 20 minutes
 });
 ```
 

src/backend/src/Kernel.js

@@ -569,7 +569,7 @@ class Kernel extends AdvancedBase {
         };
         const data_json = JSON.stringify(data);
 
-        console.log('WRITING TO', path_.join(mod_path, 'package.json'));
+        this.bootLogger.debug('WRITING TO: ' + path_.join(mod_path, 'package.json'));
 
         await fs.promises.writeFile(path_.join(mod_path, 'package.json'), data_json);
         return data;

src/backend/src/boot/BootLogger.js

@@ -28,6 +28,10 @@ class BootLogger {
             ...args,
         );
     }
+    debug (...args) {
+        if ( ! process.env.DEBUG ) return;
+        console.log('\x1B[37m[BOOT/DEBUG]', ...args, '\x1B[0m');
+    }
     error (...args) {
         console.log(
             '\x1B[31;1m[BOOT/ERROR]\x1B[0m',

src/backend/src/boot/RuntimeEnvironment.js

@@ -266,7 +266,7 @@ class RuntimeEnvironment extends AdvancedBase {
             generated_values.private_uid_secret = crypto.randomBytes(24).toString('hex');
             generated_values.private_uid_namespace = crypto.randomUUID();
             if ( using_config ) {
-                this.logger.info(
+                this.logger.debug(
                     `Overwriting ${quot(using_config)} because ` +
                     `${hl('--overwrite-config')} is set`
                 );
@@ -302,7 +302,7 @@ class RuntimeEnvironment extends AdvancedBase {
 
         let config_to_load = 'config.json';
         if ( process.env.PUTER_CONFIG_PROFILE ) {
-            this.logger.info(
+            this.logger.debug(
                 hl('PROFILE') + ' ' +
                 quot(process.env.PUTER_CONFIG_PROFILE) + ' ' +
                 `because $PUTER_CONFIG_PROFILE is set`
@@ -330,7 +330,7 @@ class RuntimeEnvironment extends AdvancedBase {
         if ( ! config.config_name ) {
             throw new Error('config_name is required');
         }
-        this.logger.info(hl(`config name`) + ` ${quot(config.config_name)}`);
+        this.logger.debug(hl(`config name`) + ` ${quot(config.config_name)}`);
 
         const mod_paths = [];
         environment.mod_paths = mod_paths;
@@ -359,18 +359,18 @@ class RuntimeEnvironment extends AdvancedBase {
     get_first_suitable_path_ (meta, paths, last_checks) {
         for ( const entry of paths ) {
             const checks = [...(entry.checks ?? []), ...last_checks];
-            this.logger.info(
+            this.logger.debug(
                 `Checking path ${quot(entry.label ?? entry.path)} for ${meta.pathFor}...`
             );
 
             let checks_pass = true;
             for ( const check of checks ) {
-                this.logger.info(
+                this.logger.debug(
                     `-> doing ${quot(check.name)} on path ${quot(entry.path)}...`
                 );
                 const result = check(entry);
                 if ( result === false ) {
-                    this.logger.info(
+                    this.logger.debug(
                         `-> ${quot(check.name)} doesn't like this path`
                     );
                     checks_pass = false;

src/backend/src/config.js

@@ -123,6 +123,7 @@ config.legacy_token_migrate = true;
 // === OS Information ===
 const os = require('os');
 const fs = require('fs');
+const { Context, context_config } = require('./util/context');
 config.os = {};
 config.os.platform = os.platform();
 
@@ -244,14 +245,32 @@ const config_pointer = {};
     // confusing issues, so we log any time this happens
     config_to_export = new Proxy(config_to_export, {
         set: (target, prop, value, receiver) => {
-            console.log(
+            const logger = Context.get('logger', { allow_fallback: true });
+            logger.debug(
                 '\x1B[36;1mCONFIGURATION MUTATED AT RUNTIME\x1B[0m',
-                prop, 'to', value
+                { prop, value },
             );
             target[prop] = value;
             return true;
         }
     })
 }
 
+// We configure the behavior in context.js from here to avoid a cyclic
+// mutual dependency between it and this file.
+//
+// Previously we had this:
+// context --(are we in "dev" environment?)--> config
+//
+// So we could not add this:
+// config --(where is the logger?) --> context
+//
+// So instead we now have:
+// config --(read this property to determine 'strict' mode)--> context
+// config --(where is the logger?) --> context
+//
+Object.defineProperty(context_config, 'strict', {
+    get: () => config_to_export.env === 'dev',
+});
+
 module.exports = config_to_export;

src/backend/src/config/ConfigLoader.js

@@ -47,7 +47,7 @@ class ConfigLoader extends AdvancedBase {
             delete config_values.$requires;
             this.apply_requires(this.path, config_list, { by: name });
         }
-        this.logger.info(
+        this.logger.debug(
             `Applying config: ${path_.relative(this.path, config_path)}` +
             (meta.by ? ` (required by ${meta.by})` : '')
         );

src/backend/src/extension/RuntimeModuleRegistry.js

@@ -16,7 +16,6 @@ class RuntimeModuleRegistry extends AdvancedBase {
         if ( this.modules_.hasOwnProperty(uniqueName) ) {
             throw new Error(`duplicate runtime module: ${uniqueName}`);
         }
-        console.log(`registering with name... ${uniqueName}`);
         this.modules_[uniqueName] = extensionModule;
         extensionModule.runtimeModuleRegistry = this;
     }

src/backend/src/modules/core/LogService.js

@@ -21,10 +21,10 @@ const logSeverity = (ordinal, label, esc, winst) => ({ ordinal, label, esc, wins
 const LOG_LEVEL_ERRO = logSeverity(0, 'ERRO', '31;1', 'error');
 const LOG_LEVEL_WARN = logSeverity(1, 'WARN', '33;1', 'warn');
 const LOG_LEVEL_INFO = logSeverity(2, 'INFO', '36;1', 'info');
-const LOG_LEVEL_TICK = logSeverity(10, 'TICK', '34;1', 'info');
-const LOG_LEVEL_DEBU = logSeverity(4, 'DEBU', '37;1', 'debug');
 const LOG_LEVEL_NOTICEME = logSeverity(3, 'NOTICE_ME', '33;1', 'error');
 const LOG_LEVEL_SYSTEM = logSeverity(3, 'SYSTEM', '33;1', 'system');
+const LOG_LEVEL_DEBU = logSeverity(4, 'DEBU', '37', 'debug');
+const LOG_LEVEL_TICK = logSeverity(10, 'TICK', '34;1', 'info');
 
 const winston = require('winston');
 const { Context } = require('../../util/context');

src/backend/src/modules/core/ParameterService.js

@@ -47,7 +47,7 @@ class ParameterService extends BaseService {
 
     createParameters(serviceName, parameters, opt_instance) {
         for (const parameter of parameters) {
-            this.log.info(`registering parameter ${serviceName}:${parameter.id}`);
+            this.log.debug(`registering parameter ${serviceName}:${parameter.id}`);
             this.parameters_.push(new Parameter({
                 ...parameter,
                 id: `${serviceName}:${parameter.id}`,

src/backend/src/modules/hostos/ProcessService.js

@@ -69,11 +69,10 @@ class ProcessService extends BaseService {
                 global_config: this.global_config
             });
         }
-        console.log(
+        this.log.debug(
             'command',
-            command,
-            ...args
-        )
+            { command, args },
+        );
         const proc = this.modules.spawn(command, args, {
             shell: true,
             env: {

src/backend/src/modules/selfhosted/MinLogService.js

@@ -7,7 +7,7 @@ class MinLogService extends BaseService {
     `
 
     _construct () {
-        this.on = true;
+        this.on = false;
         this.visible = new Set();
 
         this.widget_ = null;
@@ -44,6 +44,7 @@ class MinLogService extends BaseService {
         if ( ! svc_devConsole ) return;
 
         this.widget_ = () => {
+            if ( ! this.on ) return ['minlog is off'];
             const lines = [
                 `\x1B[31;1mSome logs hidden! Type minlog:off to see all logs.\x1B[0m`
             ];

src/backend/src/modules/selfhosted/SelfHostedModule.js

@@ -86,15 +86,6 @@ class SelfHostedModule extends AdvancedBase {
                             PUTER_JS_URL: ({ global_config: config }) => config.origin + '/sdk/puter.dev.js',
                         },
                     },
-                    {
-                        name: 'git:rollup-watch',
-                        directory: 'src/git',
-                        command: 'npx',
-                        args: ['rollup', '-c', 'rollup.config.js', '--watch'],
-                        env: {
-                            PUTER_JS_URL: ({ global_config: config }) => config.origin + '/sdk/puter.dev.js',
-                        },
-                    },
                     {
                         name: 'emulator:webpack-watch',
                         directory: 'src/emulator',

src/backend/src/services/CommandService.js

@@ -129,7 +129,7 @@ class CommandService extends BaseService {
     registerCommands(serviceName, commands) {
         if ( ! this.log ) process.exit(1);
         for (const command of commands) {
-            this.log.info(`registering command ${serviceName}:${command.id}`);
+            this.log.debug(`registering command ${serviceName}:${command.id}`);
             this.commands_.push(new Command({
                 ...command,
                 id: `${serviceName}:${command.id}`,

src/backend/src/services/Container.js

@@ -175,22 +175,23 @@ class Container {
     async init () {
         for ( const k in this.instances_ ) {
             if ( ! this.instances_[k]._run_as_early_as_possible ) continue;
-            this.logger.info(`very early hooks for ${k}`);
             await this.instances_[k].run_as_early_as_possible();
         }
         for ( const k in this.instances_ ) {
-            this.logger.info(`constructing ${k}`);
             await this.instances_[k].construct();
         }
         const init_failures = [];
+        const promises = [];
+        const PARALLEL = config.experimental_parallel_init;
         for ( const k in this.instances_ ) {
-            this.logger.info(`initializing ${k}`);
             try {
-                await this.instances_[k].init();
+                if ( PARALLEL ) promises.push(this.instances_[k].init());
+                else await this.instances_[k].init();
             } catch (e) {
                 init_failures.push({ k, e });
             }
         }
+        if ( PARALLEL ) await Promise.all(promises);
 
         if ( init_failures.length ) {
             console.error('init failures', init_failures);

src/backend/src/services/repositories/DBKVStore/DBKVStore.mjs

@@ -285,6 +285,7 @@ export class DBKVStore {
         for ( const [path, amount] of Object.entries(pathAndAmountMap) ){
             const pathParts = path.split('.');
             let obj = currVal;
+            if ( obj === null )  continue;
             for ( let i = 0; i < pathParts.length - 1; i++ ){
                 const part = pathParts[i];
                 if ( !obj[part] ){
@@ -295,6 +296,7 @@ export class DBKVStore {
                 }
                 obj = obj[part];
             }
+            if ( obj === null )  continue;
             const lastPart = pathParts[pathParts.length - 1];
             if ( !obj[lastPart] ){
                 obj[lastPart] = 0;

src/backend/src/util/context.js

@@ -17,7 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 const { AsyncLocalStorage } = require('async_hooks');
-const config = require('../config');
+const context_config = {};
 
 class Context {
     static USE_NAME_FALLBACK = {};
@@ -46,7 +46,7 @@ class Context {
     static get (k, { allow_fallback } = {}) {
         let x = this.contextAsyncLocalStorage.getStore()?.get('context');
         if ( ! x ) {
-            if ( config.env === 'dev' && ! allow_fallback ) {
+            if ( context_config.strict && ! allow_fallback ) {
                 throw new Error(
                     'FAILED TO GET THE CORRECT CONTEXT'
                 );
@@ -246,4 +246,5 @@ class ContextExpressMiddleware {
 module.exports = {
     Context,
     ContextExpressMiddleware,
+    context_config,
 };

src/emulator/package.json

@@ -4,7 +4,7 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start-webpack": "webpack --watch --devtool source-map"
+    "start-webpack": "webpack --watch --devtool source-map --stats=errors-only"
   },
   "keywords": [],
   "author": "",

src/gui/package.json

@@ -32,7 +32,7 @@
     "start=gui": "nodemon --exec \"node dev-server.js\" ",
     "build": "node ./build.js",
     "check-translations": "node tools/check-translations.js",
-    "start-webpack": "webpack --watch --devtool source-map"
+    "start-webpack": "webpack --watch --devtool source-map --stats=errors-only"
   },
   "workspaces": [
     "src/*"

src/puter-js/package.json

@@ -25,7 +25,7 @@
     ],
     "scripts": {
         "start-server": "npx http-server --cors -c-1",
-        "start-webpack": "webpack && webpack --output-filename puter.dev.js --watch --devtool source-map",
+        "start-webpack": "webpack --stats=errors-only && webpack --output-filename puter.dev.js --watch --devtool source-map --stats=errors-only",
         "start": "concurrently \"npm run start-server\" \"npm run start-webpack\"",
         "build": "webpack && { echo \"// Copyright 2024-present Puter Technologies Inc. All rights reserved.\"; echo \"// Generated on $(date '+%Y-%m-%d %H:%M')\n\"; cat ./dist/puter.js; } > temp && mv temp ./dist/puter.js",
         "prepublishOnly": "npm run build && mv dist/puter.js dist/puter.cjs && npm version patch"