[server] Single source of configuration I/II

components/server/ee/src/container-module.ts

@@ -87,6 +87,7 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is
     // various
     bind(EnvEE).toSelf().inSingletonScope();
     rebind(Env).to(EnvEE).inSingletonScope();
+
     rebind(MessageBusIntegration).to(MessageBusIntegrationEE).inSingletonScope();
     rebind(HostContainerMapping).to(HostContainerMappingEE).inSingletonScope();
     bind(EMailDomainService).to(EMailDomainServiceImpl).inSingletonScope();

components/server/ee/src/env.ts

@@ -10,8 +10,6 @@ import { getEnvVar, filePathTelepresenceAware } from '@gitpod/gitpod-protocol/li
 import { readOptionsFromFile } from '@gitpod/gitpod-payment-endpoint/lib/chargebee';
 import { Env } from '../../src/env';
 
-
-
 @injectable()
 export class EnvEE extends Env {
     readonly chargebeeProviderOptions = readOptionsFromFile(filePathTelepresenceAware('/chargebee/providerOptions'));

components/server/ee/src/prebuilds/github-app.ts

@@ -55,14 +55,15 @@ export class GithubApp {
                     appId: env.githubAppAppID,
                     privateKey: GithubApp.loadPrivateKey(env.githubAppCertPath),
                     secret: env.githubAppWebhookSecret,
-                    logLevel: env.githubAppLogLevel as Options["logLevel"]
+                    logLevel: env.githubAppLogLevel as Options["logLevel"],
+                    baseUrl: env.githubAppGHEHost,
                 })
-            })
+            });
             log.debug("Starting GitHub app integration", {
                 appId: env.githubAppAppID,
                 cert: env.githubAppCertPath,
                 secret: env.githubAppWebhookSecret
-            })
+            });
             this.server.load(this.buildApp.bind(this));
         }
     }

components/server/src/auth/rate-limiter.ts

@@ -24,12 +24,12 @@ type FunctionsConfig = {
         points: number,
     }
 }
-type RateLimiterConfig = {
+export type RateLimiterConfig = {
     groups: GroupsConfig,
     functions: FunctionsConfig,
 };
 
-function readConfig(): RateLimiterConfig {
+function getConfig(config: RateLimiterConfig): RateLimiterConfig {
     const defaultGroups: GroupsConfig = {
         default: {
             points: 60000, // 1,000 calls per user per second
@@ -167,11 +167,9 @@ function readConfig(): RateLimiterConfig {
         "trackEvent":  { group: "default", points: 1 },
     };
 
-    const fromEnv = JSON.parse(process.env.RATE_LIMITER_CONFIG || "{}")
-
     return {
-        groups: { ...defaultGroups, ...fromEnv.groups },
-        functions: { ...defaultFunctions, ...fromEnv.functions }
+        groups: { ...defaultGroups, ...config.groups },
+        functions: { ...defaultFunctions, ...config.functions }
     };
 }
 
@@ -184,19 +182,18 @@ export class UserRateLimiter {
 
     private static instance_: UserRateLimiter;
 
-    public static instance(): UserRateLimiter {
+    public static instance(config: RateLimiterConfig): UserRateLimiter {
         if (!UserRateLimiter.instance_) {
-            UserRateLimiter.instance_ = new UserRateLimiter();
+            UserRateLimiter.instance_ = new UserRateLimiter(config);
         }
         return UserRateLimiter.instance_;
     }
 
     private readonly config: RateLimiterConfig;
     private readonly limiters: { [key: string]: RateLimiterMemory };
 
-    private constructor() {
-        this.config = readConfig();
-
+    private constructor(config: RateLimiterConfig) {
+        this.config = getConfig(config);
         this.limiters = {};
         Object.keys(this.config.groups).forEach(group => {
             this.limiters[group] = new RateLimiterMemory({

components/server/src/code-sync/code-sync-service.ts

@@ -21,20 +21,21 @@ import uuid = require('uuid');
 import { accessCodeSyncStorage, UserRateLimiter } from '../auth/rate-limiter';
 import { increaseApiCallUserCounter } from '../prometheus-metrics';
 import { TheiaPluginService } from '../theia-plugin/theia-plugin-service';
+import { Env } from '../env';
 
 // By default: 5 kind of resources * 20 revs * 1Mb = 100Mb max in the content service for user data.
 const defautltRevLimit = 20;
 // It should keep it aligned with client_max_body_size for /code-sync location.
 const defaultContentLimit = '1Mb';
-const codeSyncConfig: Partial<{
+export type CodeSyncConfig = Partial<{
     revLimit: number
     contentLimit: number
     resources: {
         [resource: string]: {
             revLimit?: number
         }
     }
-}> = JSON.parse(process.env.CODE_SYNC_CONFIG || "{}");
+}>;
 
 const objectPrefix = 'code-sync/';
 function toObjectName(resource: ServerResource, rev: string): string {
@@ -55,6 +56,9 @@ const userSettingsUri = 'user_storage:settings.json';
 @injectable()
 export class CodeSyncService {
 
+    @inject(Env)
+    private readonly env: Env;
+
     @inject(BearerAuth)
     private readonly auth: BearerAuth;
 
@@ -71,6 +75,7 @@ export class CodeSyncService {
     private readonly userStorageResourcesDB: UserStorageResourcesDB;
 
     get apiRouter(): express.Router {
+        const config = this.env.codeSyncConfig;
         const router = express.Router();
         router.use((_, res, next) => {
             // to correlate errors reported by users with errors logged by the server
@@ -87,7 +92,7 @@ export class CodeSyncService {
             const id = req.user.id;
             increaseApiCallUserCounter(accessCodeSyncStorage, id);
             try {
-                await UserRateLimiter.instance().consume(id, accessCodeSyncStorage);
+                await UserRateLimiter.instance(this.env.rateLimiter).consume(id, accessCodeSyncStorage);
             } catch (e) {
                 if (e instanceof Error) {
                     throw e;
@@ -207,7 +212,7 @@ export class CodeSyncService {
             res.send(content);
         });
         router.post('/v1/resource/:resource', bodyParser.text({
-            limit: codeSyncConfig?.contentLimit || defaultContentLimit
+            limit: config?.contentLimit || defaultContentLimit
         }), async (req, res) => {
             if (!User.is(req.user)) {
                 res.sendStatus(204);
@@ -222,7 +227,7 @@ export class CodeSyncService {
             if (latestRev === fromTheiaRev) {
                 latestRev = undefined;
             }
-            const revLimit = resourceKey === 'machines' ? 1 : codeSyncConfig.resources?.[resourceKey]?.revLimit || codeSyncConfig?.revLimit || defautltRevLimit;
+            const revLimit = resourceKey === 'machines' ? 1 : config.resources?.[resourceKey]?.revLimit || config?.revLimit || defautltRevLimit;
             const userId = req.user.id;
             let oldObject: string | undefined;
             const contentType = req.headers['content-type'] || '*/*';

components/server/src/config.ts

@@ -0,0 +1,218 @@
+/**
+ * Copyright (c) 2020 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { GitpodHostUrl } from '@gitpod/gitpod-protocol/lib/util/gitpod-host-url';
+import { AuthProviderParams } from './auth/auth-provider';
+
+import { Branding, NamedWorkspaceFeatureFlag } from '@gitpod/gitpod-protocol';
+
+import { RateLimiterConfig } from './auth/rate-limiter';
+import { Env } from './env';
+import { CodeSyncConfig } from './code-sync/code-sync-service';
+import { ChargebeeProviderOptions } from "@gitpod/gitpod-payment-endpoint/lib/chargebee";
+import { EnvEE } from '../ee/src/env';
+
+export const Config = Symbol("Config");
+export interface Config {
+    version: string;
+    hostUrl: GitpodHostUrl;
+
+    license: string | undefined;
+    trialLicensePrivateKey: string | undefined;
+
+    heartbeat: {
+        intervalSeconds: number;
+        timeoutSeconds: number,
+    };
+
+    workspaceDefaults: {
+        ideVersion: string;
+        ideImageRepo: string;
+        ideImage: string;
+        ideImageAliases: { [index: string]: string };
+        workspaceImage: string;
+        previewFeatureFlags: NamedWorkspaceFeatureFlag[];
+        defaultFeatureFlags: NamedWorkspaceFeatureFlag[];
+    };
+
+    session: {
+        maxAgeMs: number;
+        secret: string;
+    };
+
+    githubApp?: {
+        enabled: boolean;
+        appId: number;
+        webhookSecret: string;
+        authProviderId: string;
+        certPath: string;
+        marketplaceName: string;
+        logLevel?: string;
+    };
+
+    definitelyGpDisabled: boolean;
+
+    workspaceGarbageCollection: {
+        disabled: boolean;
+        startDate: number;
+        chunkLimit: number;
+        minAgeDays: number;
+        minAgePrebuildDays: number;
+        contentRetentionPeriodDays: number;
+        contentChunkLimit: number;
+    };
+
+    enableLocalApp: boolean;
+
+    authProviderConfigs: AuthProviderParams[];
+    disableDynamicAuthProviderLogin: boolean;
+
+    // TODO(gpl) Can we remove this?
+    brandingConfig: Branding;
+
+    /**
+     * The maximum number of environment variables a user can have configured in their list at any given point in time.
+     * Note: This limit should be so high that no regular user ever reaches it.
+     */
+    maxEnvvarPerUserCount: number;
+
+    /** maxConcurrentPrebuildsPerRef is the maximum number of prebuilds we allow per ref type at any given time */
+    maxConcurrentPrebuildsPerRef: number;
+
+    incrementalPrebuilds: {
+        repositoryPassList: string[];
+        commitHistory: number;
+    };
+
+    blockNewUsers: {
+        enabled: boolean;
+        passlist: string[];
+    }
+
+    makeNewUsersAdmin: boolean;
+
+    /** this value - if present - overrides the default naming scheme for the GCloud bucket that Theia Plugins are stored in */
+    theiaPluginsBucketNameOverride: string | undefined;
+
+    /** defaultBaseImageRegistryWhitelist is the list of registryies users get acces to by default */
+    defaultBaseImageRegistryWhitelist: string[];
+
+    // TODO(gpl): can we remove this? We never set the value anywhere it seems
+    insecureNoDomain: boolean;
+
+    runDbDeleter: boolean;
+
+    oauthServer: {
+        enabled: boolean;
+        jwtSecret: string;
+    }
+
+    /**
+     * The configuration for the rate limiter we (mainly) use for the websocket API
+     */
+    rateLimiter: RateLimiterConfig;
+
+    /**
+     * The address content service clients connect to
+     * Example: content-service:8080
+    */
+    contentServiceAddress: string;
+
+    /**
+     * TODO(gpl) Looks like this is not used anymore! Verify and remove
+     */
+    serverProxyApiKey?: string;
+
+    codeSyncConfig: CodeSyncConfig;
+
+    /**
+     * Payment related options
+     */
+    chargebeeProviderOptions?: ChargebeeProviderOptions;
+    enablePayment?: boolean;
+}
+
+export namespace EnvConfig {
+    export function fromEnv(env: Env): Config {
+        return {
+            version: env.version,
+            hostUrl: env.hostUrl,
+            license: env.gitpodLicense,
+            trialLicensePrivateKey: env.trialLicensePrivateKey,
+            heartbeat: {
+                intervalSeconds: env.theiaHeartbeatInterval,
+                timeoutSeconds: env.workspaceUserTimeout,
+            },
+            workspaceDefaults: {
+                ideVersion: env.theiaVersion,
+                ideImageRepo: env.theiaImageRepo,
+                ideImage: env.ideDefaultImage,
+                ideImageAliases: env.ideImageAliases,
+                workspaceImage: env.workspaceDefaultImage,
+                previewFeatureFlags: env.previewFeatureFlags,
+                defaultFeatureFlags: env.defaultFeatureFlags,
+            },
+            session: {
+                maxAgeMs: env.sessionMaxAgeMs,
+                secret: env.sessionSecret,
+            },
+            githubApp: {
+                enabled: env.githubAppEnabled,
+                appId: env.githubAppAppID,
+                webhookSecret: env.githubAppWebhookSecret,
+                authProviderId: env.githubAppAuthProviderId,
+                certPath: env.githubAppCertPath,
+                marketplaceName: env.githubAppMarketplaceName,
+                logLevel: env.githubAppLogLevel,
+            },
+            definitelyGpDisabled: env.definitelyGpDisabled,
+            workspaceGarbageCollection: {
+                disabled: env.garbageCollectionDisabled,
+                startDate: env.garbageCollectionStartDate,
+                chunkLimit: env.garbageCollectionLimit,
+                minAgeDays: env.daysBeforeGarbageCollection,
+                minAgePrebuildDays: env.daysBeforeGarbageCollectingPrebuilds,
+                contentRetentionPeriodDays: env.workspaceDeletionRetentionPeriodDays,
+                contentChunkLimit: env.workspaceDeletionLimit,
+            },
+            enableLocalApp: env.enableLocalApp,
+            authProviderConfigs: env.authProviderConfigs,
+            disableDynamicAuthProviderLogin: env.disableDynamicAuthProviderLogin,
+            brandingConfig: env.brandingConfig,
+            maxEnvvarPerUserCount: env.maxUserEnvvarCount,
+            maxConcurrentPrebuildsPerRef: env.maxConcurrentPrebuildsPerRef,
+            incrementalPrebuilds: {
+                repositoryPassList: env.incrementalPrebuildsRepositoryPassList,
+                commitHistory: env.incrementalPrebuildsCommitHistory,
+            },
+            blockNewUsers: {
+                enabled: env.blockNewUsers,
+                passlist: env.blockNewUsersPassList,
+            },
+            makeNewUsersAdmin: env.makeNewUsersAdmin,
+            theiaPluginsBucketNameOverride: env.theiaPluginsBucketNameOverride,
+            defaultBaseImageRegistryWhitelist: env.defaultBaseImageRegistryWhitelist,
+            insecureNoDomain: env.insecureNoDomain,
+            runDbDeleter: env.runDbDeleter,
+            oauthServer: {
+                enabled: env.enableOAuthServer,
+                jwtSecret: env.oauthServerJWTSecret,
+            },
+            rateLimiter: env.rateLimiter,
+            contentServiceAddress: env.contentServiceAddress,
+            serverProxyApiKey: env.serverProxyApiKey,
+            codeSyncConfig: env.codeSyncConfig,
+        };
+    }
+    export function fromEnvEE(env: EnvEE): Config {
+        const config = EnvConfig.fromEnv(env);
+        return {
+            ...config,
+            chargebeeProviderOptions: env.chargebeeProviderOptions,
+            enablePayment: env.enablePayment,
+        }
+    }
+}