[server] Introduce /debug/logging

chart/templates/_helpers.tpl

@@ -183,7 +183,13 @@ env:
 - name: GITPOD_INSTALLATION_SHORTNAME
   value: {{ template "gitpod.installation.shortname" . }}
 - name: LOG_LEVEL
-  value: {{ $gp.log.level | default "debug" | lower | quote }}
+  value: {{ template "gitpod.loglevel" }}
+{{- end -}}
+
+{{- define "gitpod.loglevel" -}}
+{{- $ := .root -}}
+{{- $gp := .gp -}}
+{{ $gp.log.level | default "debug" | lower | quote }}
 {{- end -}}
 
 {{- define "gitpod.container.analyticsEnv" -}}

chart/templates/server-configmap.yaml

@@ -23,6 +23,7 @@ data:
 {{- if .Values.devBranch }}
         "devBranch": "{{ .Values.devBranch }}",
 {{- end }}
+        "logLevel": {{ template "gitpod.loglevel" }},
         "license": "{{ .Values.license }}",
         "workspaceHeartbeat": {{ $comp.workspaceHeartbeat | toJson }},
         "workspaceDefaults": {

chart/values.yaml

@@ -299,7 +299,6 @@ components:
     githubApp:
       enabled: false
       authProviderId: "Public-GitHub"
-      logLevel: "trace"
     blockNewUsers:
       enabled: false
       passlist: []
@@ -326,9 +325,12 @@ components:
       metrics:
         expose: true
         containerPort: 9500
-      debug:
+      debugNode:
         expose: false
         containerPort: 9229
+      debug:
+        expose: false
+        containerPort: 6060
     serviceSessionAffinity: None
     serverContainer:
       volumeMounts: null

components/ee/db-sync/src/main.ts

@@ -7,11 +7,11 @@
 require("reflect-metadata")
 
 import { ArgumentParser } from "argparse";
-import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
+import { log, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
 import { ICommand } from "./commands";
 import { Container } from "inversify";
 import { productionContainerModule } from "./container-module";
-log.enableJSONLogging('db-sync', undefined);
+log.enableJSONLogging('db-sync', undefined, LogrusLogLevel.getFromEnv());
 
 const parser = new ArgumentParser({
     version: "0.1.5",

components/ee/payment-endpoint/src/main.ts

@@ -11,10 +11,10 @@ import { Container } from 'inversify';
 import { Server } from "./server";
 import { productionContainerModule } from './container-module';
 
-import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
+import { log, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
 import { dbContainerModule } from '@gitpod/gitpod-db/lib/container-module';
 
-log.enableJSONLogging('payment-endpoint', undefined);
+log.enableJSONLogging('payment-endpoint', undefined, LogrusLogLevel.getFromEnv());
 
 const init = async () => {
     const container = new Container();

components/gitpod-protocol/src/util/logging.ts

@@ -97,24 +97,32 @@ export namespace log {
     /**
      * Do not use in frontend.
      */
-    export function enableJSONLogging(componentArg: string, versionArg: string | undefined): void {
+    export function enableJSONLogging(componentArg: string, versionArg: string | undefined, logLevel?: LogrusLogLevel): void {
         component = componentArg;
         version = versionArg;
 
+        setLogLevel(logLevel);
+    }
+
+    export function setLogLevel(logLevel: LogrusLogLevel | undefined) {
         jsonLogging = true;
 
-        console.error = function (...args: any[]): void {
-            errorLog(true, args);
-        }
-        console.warn = function (...args: any[]): void {
-            warnLog(true, args);
-        }
-        console.info = function (...args: any[]): void {
-            infoLog(true, args);
-        }
-        console.debug = function (...args: any[]): void {
-            debugLog(true, args);
-        }
+        const noop = () => {};
+        const setLog = (logFunc: (calleViaConsole: boolean, args: any[]) => void, funcLvl: LogrusLogLevel): (() => void) => {
+            if (LogrusLogLevel.isGreatherOrEqual(funcLvl, logLevel)) {
+                return function (...args: any[]): void {
+                    logFunc(true, args);
+                };
+            } else {
+                return noop;
+            }
+        };
+
+        console.error = setLog(errorLog, "error");
+        console.warn = setLog(warnLog, "warning");
+        console.info = setLog(infoLog, "info");
+        console.debug = setLog(debugLog, "debug");
+
         console.log = console.info;
         // FIXME wrap also other console methods (e.g. trace())
     }
@@ -150,6 +158,45 @@ function debugLog(calledViaConsole: boolean, args: any[]): void {
     doLog(calledViaConsole, debugConsoleLog, 'DEBUG', args);
 }
 
+// Ref: https://github.com/sirupsen/logrus#level-logging
+export type LogrusLogLevel = keyof (typeof LogrusLogLevels);
+export const LogrusLogLevels = {
+    trace: true,
+    debug: true,
+    info: true,
+    warning: true,
+    error: true,
+    fatal: true,
+    panic: true,
+}
+export namespace LogrusLogLevel {
+    export function isGreatherOrEqual(lvl: LogrusLogLevel | undefined, ref: LogrusLogLevel | undefined): boolean {
+        if (lvl === undefined) {
+            return false;
+        }
+        if (ref === undefined) {
+            return true;
+        }
+        return getLevelArity(lvl) >= getLevelArity(ref);
+    }
+    function getLevelArity(lvl: LogrusLogLevel): number {
+        return Object.keys(LogrusLogLevels)
+            .findIndex((l) => l === lvl);
+    }
+    export function getFromEnv(): LogrusLogLevel | undefined {
+        const lvlStr = process.env.LOG_LEVEL;
+        if (!lvlStr) {
+            return undefined;
+        }
+        const lvl = lvlStr as LogrusLogLevel;
+        const exists = LogrusLogLevels[lvl]
+        if (!exists) {
+            return undefined;
+        }
+        return lvl;
+    }
+}
+
 // Source: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity
 type GoogleLogSeverity = 'EMERGENCY' | 'ALERT' | 'CRITICAL' | 'ERROR' | 'WARNING' | 'INFO' | 'DEBUG';
 namespace GoogleLogSeverity {

components/server/ee/src/prebuilds/github-app.ts

@@ -12,7 +12,7 @@ import { injectable, inject } from 'inversify';
 import { Config } from '../../../src/config';
 import { AppInstallationDB, TracedWorkspaceDB, DBWithTracing, UserDB, WorkspaceDB, ProjectDB, TeamDB } from '@gitpod/gitpod-db/lib';
 import * as express from 'express';
-import { log, LogContext } from '@gitpod/gitpod-protocol/lib/util/logging';
+import { log, LogContext, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
 import { WorkspaceConfig, User, Project, StartPrebuildResult } from '@gitpod/gitpod-protocol';
 import { MessageBusIntegration } from '../../../src/workspace/messagebus-integration';
 import { GithubAppRules } from './github-app-rules';
@@ -56,7 +56,7 @@ export class GithubApp {
                     appId: config.githubApp.appId,
                     privateKey: GithubApp.loadPrivateKey(config.githubApp.certPath),
                     secret: config.githubApp.webhookSecret,
-                    logLevel: config.githubApp.logLevel as Options["logLevel"],
+                    logLevel: GithubApp.mapToGitHubLogLevel(config.logLevel),
                     baseUrl: config.githubApp.baseUrl,
                 })
             });
@@ -404,4 +404,15 @@ export namespace GithubApp {
             }
         }
     }
+
+    export function mapToGitHubLogLevel(logLevel: LogrusLogLevel): Options["logLevel"] {
+        switch (logLevel) {
+            case "warning":
+                return "warn";
+            case "panic":
+                return "fatal";
+            default:
+                return logLevel;
+        }
+    }
 }

components/server/src/config.ts

@@ -13,7 +13,7 @@ import { RateLimiterConfig } from './auth/rate-limiter';
 import { CodeSyncConfig } from './code-sync/code-sync-service';
 import { ChargebeeProviderOptions, readOptionsFromFile } from "@gitpod/gitpod-payment-endpoint/lib/chargebee";
 import * as fs from 'fs';
-import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
+import { log, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
 import { filePathTelepresenceAware, KubeStage, translateLegacyStagename } from '@gitpod/gitpod-protocol/lib/env';
 import { BrandingParser } from './branding-parser';
 
@@ -51,6 +51,7 @@ export interface ConfigSerialized {
     stage: string;
     devBranch?: string;
     insecureNoDomain: boolean;
+    logLevel: LogrusLogLevel;
 
     license?: string;
 
@@ -74,7 +75,6 @@ export interface ConfigSerialized {
         authProviderId: string;
         certPath: string;
         marketplaceName: string;
-        logLevel?: string;
     };
 
     definitelyGpDisabled: boolean;

components/server/src/container-module.ts

@@ -83,6 +83,7 @@ import { defaultGRPCOptions } from '@gitpod/gitpod-protocol/lib/util/grpc';
 import { IDEConfigService } from './ide-config';
 import { PrometheusClientCallMetrics } from "@gitpod/gitpod-protocol/lib/messaging/client-call-metrics";
 import { IClientCallMetrics } from '@gitpod/content-service/lib/client-call-metrics';
+import { DebugApp } from './debug-app';
 
 export const productionContainerModule = new ContainerModule((bind, unbind, isBound, rebind) => {
     bind(Config).toConstantValue(ConfigFile.fromFile());
@@ -103,6 +104,7 @@ export const productionContainerModule = new ContainerModule((bind, unbind, isBo
 
     bind(SessionHandlerProvider).toSelf().inSingletonScope();
     bind(Server).toSelf().inSingletonScope();
+    bind(DebugApp).toSelf().inSingletonScope();
 
     bind(GitpodFileParser).toSelf().inSingletonScope();
 

components/server/src/debug-app.ts

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import * as http from 'http';
+import * as express from 'express';
+import { injectable, postConstruct } from "inversify";
+import { log, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
+
+
+export interface SetLogLevelRequest {
+    level: LogrusLogLevel,
+}
+export namespace SetLogLevelRequest {
+    export function is(o: any): o is SetLogLevelRequest {
+        return typeof o === 'object'
+            && 'level' in o;
+    }
+}
+
+@injectable()
+export class DebugApp {
+    protected _app: express.Application;
+    protected httpServer: http.Server | undefined = undefined;
+
+    @postConstruct()
+    public ctor() {
+        this._app = this.create();
+    }
+
+    create(): express.Application {
+        const app = express();
+        app.post('/debug/logging', (req, res) => {
+	        try {
+                const parsed = JSON.parse(req.body);
+                if (!SetLogLevelRequest.is(parsed)) {
+                    res.status(400).end("not a SetLogLevelRequest");
+                    return;
+                }
+
+                const newLogLevel = parsed.level;
+                log.setLogLevel(newLogLevel);
+                log.info("set log level", { newLogLevel });
+	        } catch (err) {
+    		    res.status(500).end(err);
+	        }
+        });
+        return app;
+    }
+
+    public start(port: number) {
+        this.httpServer = this._app.listen(port, 'localhost', () => {
+            log.info(`debug server listening on port: ${port}`);
+        });
+    }
+
+    public async stop() {
+        const server = this.httpServer;
+        if (!server) {
+            return;
+        }
+        return new Promise<void>((resolve) => server.close((err: any) => {
+            if (err) {
+                log.warn(`error while closing http server`, { err });
+            }
+            this.httpServer = undefined;
+            resolve();
+        }));
+    }
+
+    public get app(): express.Application {
+        return this._app;
+    }
+}

components/server/src/init.ts

@@ -59,13 +59,13 @@ if (typeof (Symbol as any).asyncIterator === 'undefined') {
 import * as express from 'express';
 import { Container } from 'inversify';
 import { Server } from "./server"
-import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
+import { log, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging';
 import { TracingManager } from '@gitpod/gitpod-protocol/lib/util/tracing';
 if (process.env.NODE_ENV === 'development') {
     require('longjohn');
 }
 
-log.enableJSONLogging('server', process.env.VERSION);
+log.enableJSONLogging('server', process.env.VERSION, LogrusLogLevel.getFromEnv());
 
 export async function start(container: Container) {
     const tracing = container.get(TracingManager);

components/server/src/server.ts

@@ -41,6 +41,7 @@ import { OAuthController } from './oauth-server/oauth-controller';
 import { HeadlessLogController, HEADLESS_LOGS_PATH_PREFIX, HEADLESS_LOG_DOWNLOAD_PATH_PREFIX } from './workspace/headless-log-controller';
 import { NewsletterSubscriptionController } from './user/newsletter-subscription-controller';
 import { Config } from './config';
+import { DebugApp } from './debug-app';
 
 @injectable()
 export class Server<C extends GitpodClient, S extends GitpodServer> {
@@ -58,6 +59,7 @@ export class Server<C extends GitpodClient, S extends GitpodServer> {
     @inject(MonitoringEndpointsApp) protected readonly monitoringEndpointsApp: MonitoringEndpointsApp;
     @inject(CodeSyncService) private readonly codeSyncService: CodeSyncService;
     @inject(HeadlessLogController) protected readonly headlessLogController: HeadlessLogController;
+    @inject(DebugApp) protected readonly debugApp: DebugApp;
 
     @inject(RabbitMQConsensusLeaderMessenger) protected readonly consensusMessenger: RabbitMQConsensusLeaderMessenger;
     @inject(ConsensusLeaderQorum) protected readonly qorum: ConsensusLeaderQorum;
@@ -76,8 +78,8 @@ export class Server<C extends GitpodClient, S extends GitpodServer> {
     protected readonly eventEmitter = new EventEmitter();
     protected app?: express.Application;
     protected httpServer?: http.Server;
-    protected monApp?: express.Application;
-    protected monHttpServer?: http.Server;
+    protected monitoringApp?: express.Application;
+    protected monitoringHttpServer?: http.Server;
 
     public async init(app: express.Application) {
         log.setVersion(this.config.version);
@@ -210,7 +212,7 @@ export class Server<C extends GitpodClient, S extends GitpodServer> {
 
 
         // Health check + metrics endpoints
-        this.monApp = this.monitoringEndpointsApp.create();
+        this.monitoringApp = this.monitoringEndpointsApp.create();
 
         // Report current websocket connections
         this.installWebsocketConnectionGauge();
@@ -274,15 +276,18 @@ export class Server<C extends GitpodClient, S extends GitpodServer> {
         })
         this.httpServer = httpServer;
 
-        if (this.monApp) {
-            this.monHttpServer = this.monApp.listen(9500, 'localhost', () => {
-                log.info(`monitoring app listening on port: ${(<AddressInfo>this.monHttpServer!.address()).port}`);
+        if (this.monitoringApp) {
+            this.monitoringHttpServer = this.monitoringApp.listen(9500, 'localhost', () => {
+                log.info(`monitoring app listening on port: ${(<AddressInfo>this.monitoringHttpServer!.address()).port}`);
             });
         }
+
+        this.debugApp.start(6060);
     }
 
     public async stop() {
-        await this.stopServer(this.monHttpServer);
+        await this.debugApp.stop();
+        await this.stopServer(this.monitoringHttpServer);
         await this.stopServer(this.httpServer);
         log.info('server stopped.');
     }