Skip to content

Commit

Permalink
[server] Introduce Config (without using it)
Browse files Browse the repository at this point in the history
To be able to easily map Env+EnvEE into config there was some minor refactoring/cleanup necessary. This has be done in a way to be as straigth forward as possible to minimize the risk of breaking things, while making it possible to easily write an alternative Config parser.
  • Loading branch information
geropl authored and roboquat committed Jul 27, 2021
1 parent 311936b commit ae0698e
Show file tree
Hide file tree
Showing 12 changed files with 290 additions and 77 deletions.
1 change: 1 addition & 0 deletions components/server/ee/src/container-module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is
// various
bind(EnvEE).toSelf().inSingletonScope();
rebind(Env).to(EnvEE).inSingletonScope();

rebind(MessageBusIntegration).to(MessageBusIntegrationEE).inSingletonScope();
rebind(HostContainerMapping).to(HostContainerMappingEE).inSingletonScope();
bind(EMailDomainService).to(EMailDomainServiceImpl).inSingletonScope();
Expand Down
2 changes: 0 additions & 2 deletions components/server/ee/src/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ import { getEnvVar, filePathTelepresenceAware } from '@gitpod/gitpod-protocol/li
import { readOptionsFromFile } from '@gitpod/gitpod-payment-endpoint/lib/chargebee';
import { Env } from '../../src/env';



@injectable()
export class EnvEE extends Env {
readonly chargebeeProviderOptions = readOptionsFromFile(filePathTelepresenceAware('/chargebee/providerOptions'));
Expand Down
7 changes: 4 additions & 3 deletions components/server/ee/src/prebuilds/github-app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,15 @@ export class GithubApp {
appId: env.githubAppAppID,
privateKey: GithubApp.loadPrivateKey(env.githubAppCertPath),
secret: env.githubAppWebhookSecret,
logLevel: env.githubAppLogLevel as Options["logLevel"]
logLevel: env.githubAppLogLevel as Options["logLevel"],
baseUrl: env.githubAppGHEHost,
})
})
});
log.debug("Starting GitHub app integration", {
appId: env.githubAppAppID,
cert: env.githubAppCertPath,
secret: env.githubAppWebhookSecret
})
});
this.server.load(this.buildApp.bind(this));
}
}
Expand Down
19 changes: 8 additions & 11 deletions components/server/src/auth/rate-limiter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ type FunctionsConfig = {
points: number,
}
}
type RateLimiterConfig = {
export type RateLimiterConfig = {
groups: GroupsConfig,
functions: FunctionsConfig,
};

function readConfig(): RateLimiterConfig {
function getConfig(config: RateLimiterConfig): RateLimiterConfig {
const defaultGroups: GroupsConfig = {
default: {
points: 60000, // 1,000 calls per user per second
Expand Down Expand Up @@ -167,11 +167,9 @@ function readConfig(): RateLimiterConfig {
"trackEvent": { group: "default", points: 1 },
};

const fromEnv = JSON.parse(process.env.RATE_LIMITER_CONFIG || "{}")

return {
groups: { ...defaultGroups, ...fromEnv.groups },
functions: { ...defaultFunctions, ...fromEnv.functions }
groups: { ...defaultGroups, ...config.groups },
functions: { ...defaultFunctions, ...config.functions }
};
}

Expand All @@ -184,19 +182,18 @@ export class UserRateLimiter {

private static instance_: UserRateLimiter;

public static instance(): UserRateLimiter {
public static instance(config: RateLimiterConfig): UserRateLimiter {
if (!UserRateLimiter.instance_) {
UserRateLimiter.instance_ = new UserRateLimiter();
UserRateLimiter.instance_ = new UserRateLimiter(config);
}
return UserRateLimiter.instance_;
}

private readonly config: RateLimiterConfig;
private readonly limiters: { [key: string]: RateLimiterMemory };

private constructor() {
this.config = readConfig();

private constructor(config: RateLimiterConfig) {
this.config = getConfig(config);
this.limiters = {};
Object.keys(this.config.groups).forEach(group => {
this.limiters[group] = new RateLimiterMemory({
Expand Down
15 changes: 10 additions & 5 deletions components/server/src/code-sync/code-sync-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,21 @@ import uuid = require('uuid');
import { accessCodeSyncStorage, UserRateLimiter } from '../auth/rate-limiter';
import { increaseApiCallUserCounter } from '../prometheus-metrics';
import { TheiaPluginService } from '../theia-plugin/theia-plugin-service';
import { Env } from '../env';

// By default: 5 kind of resources * 20 revs * 1Mb = 100Mb max in the content service for user data.
const defautltRevLimit = 20;
// It should keep it aligned with client_max_body_size for /code-sync location.
const defaultContentLimit = '1Mb';
const codeSyncConfig: Partial<{
export type CodeSyncConfig = Partial<{
revLimit: number
contentLimit: number
resources: {
[resource: string]: {
revLimit?: number
}
}
}> = JSON.parse(process.env.CODE_SYNC_CONFIG || "{}");
}>;

const objectPrefix = 'code-sync/';
function toObjectName(resource: ServerResource, rev: string): string {
Expand All @@ -55,6 +56,9 @@ const userSettingsUri = 'user_storage:settings.json';
@injectable()
export class CodeSyncService {

@inject(Env)
private readonly env: Env;

@inject(BearerAuth)
private readonly auth: BearerAuth;

Expand All @@ -71,6 +75,7 @@ export class CodeSyncService {
private readonly userStorageResourcesDB: UserStorageResourcesDB;

get apiRouter(): express.Router {
const config = this.env.codeSyncConfig;
const router = express.Router();
router.use((_, res, next) => {
// to correlate errors reported by users with errors logged by the server
Expand All @@ -87,7 +92,7 @@ export class CodeSyncService {
const id = req.user.id;
increaseApiCallUserCounter(accessCodeSyncStorage, id);
try {
await UserRateLimiter.instance().consume(id, accessCodeSyncStorage);
await UserRateLimiter.instance(this.env.rateLimiter).consume(id, accessCodeSyncStorage);
} catch (e) {
if (e instanceof Error) {
throw e;
Expand Down Expand Up @@ -207,7 +212,7 @@ export class CodeSyncService {
res.send(content);
});
router.post('/v1/resource/:resource', bodyParser.text({
limit: codeSyncConfig?.contentLimit || defaultContentLimit
limit: config?.contentLimit || defaultContentLimit
}), async (req, res) => {
if (!User.is(req.user)) {
res.sendStatus(204);
Expand All @@ -222,7 +227,7 @@ export class CodeSyncService {
if (latestRev === fromTheiaRev) {
latestRev = undefined;
}
const revLimit = resourceKey === 'machines' ? 1 : codeSyncConfig.resources?.[resourceKey]?.revLimit || codeSyncConfig?.revLimit || defautltRevLimit;
const revLimit = resourceKey === 'machines' ? 1 : config.resources?.[resourceKey]?.revLimit || config?.revLimit || defautltRevLimit;
const userId = req.user.id;
let oldObject: string | undefined;
const contentType = req.headers['content-type'] || '*/*';
Expand Down
218 changes: 218 additions & 0 deletions components/server/src/config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
/**
* Copyright (c) 2020 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

import { GitpodHostUrl } from '@gitpod/gitpod-protocol/lib/util/gitpod-host-url';
import { AuthProviderParams } from './auth/auth-provider';

import { Branding, NamedWorkspaceFeatureFlag } from '@gitpod/gitpod-protocol';

import { RateLimiterConfig } from './auth/rate-limiter';
import { Env } from './env';
import { CodeSyncConfig } from './code-sync/code-sync-service';
import { ChargebeeProviderOptions } from "@gitpod/gitpod-payment-endpoint/lib/chargebee";
import { EnvEE } from '../ee/src/env';

export const Config = Symbol("Config");
export interface Config {
version: string;
hostUrl: GitpodHostUrl;

license: string | undefined;
trialLicensePrivateKey: string | undefined;

heartbeat: {
intervalSeconds: number;
timeoutSeconds: number,
};

workspaceDefaults: {
ideVersion: string;
ideImageRepo: string;
ideImage: string;
ideImageAliases: { [index: string]: string };
workspaceImage: string;
previewFeatureFlags: NamedWorkspaceFeatureFlag[];
defaultFeatureFlags: NamedWorkspaceFeatureFlag[];
};

session: {
maxAgeMs: number;
secret: string;
};

githubApp?: {
enabled: boolean;
appId: number;
webhookSecret: string;
authProviderId: string;
certPath: string;
marketplaceName: string;
logLevel?: string;
};

definitelyGpDisabled: boolean;

workspaceGarbageCollection: {
disabled: boolean;
startDate: number;
chunkLimit: number;
minAgeDays: number;
minAgePrebuildDays: number;
contentRetentionPeriodDays: number;
contentChunkLimit: number;
};

enableLocalApp: boolean;

authProviderConfigs: AuthProviderParams[];
disableDynamicAuthProviderLogin: boolean;

// TODO(gpl) Can we remove this?
brandingConfig: Branding;

/**
* The maximum number of environment variables a user can have configured in their list at any given point in time.
* Note: This limit should be so high that no regular user ever reaches it.
*/
maxEnvvarPerUserCount: number;

/** maxConcurrentPrebuildsPerRef is the maximum number of prebuilds we allow per ref type at any given time */
maxConcurrentPrebuildsPerRef: number;

incrementalPrebuilds: {
repositoryPassList: string[];
commitHistory: number;
};

blockNewUsers: {
enabled: boolean;
passlist: string[];
}

makeNewUsersAdmin: boolean;

/** this value - if present - overrides the default naming scheme for the GCloud bucket that Theia Plugins are stored in */
theiaPluginsBucketNameOverride: string | undefined;

/** defaultBaseImageRegistryWhitelist is the list of registryies users get acces to by default */
defaultBaseImageRegistryWhitelist: string[];

// TODO(gpl): can we remove this? We never set the value anywhere it seems
insecureNoDomain: boolean;

runDbDeleter: boolean;

oauthServer: {
enabled: boolean;
jwtSecret: string;
}

/**
* The configuration for the rate limiter we (mainly) use for the websocket API
*/
rateLimiter: RateLimiterConfig;

/**
* The address content service clients connect to
* Example: content-service:8080
*/
contentServiceAddress: string;

/**
* TODO(gpl) Looks like this is not used anymore! Verify and remove
*/
serverProxyApiKey?: string;

codeSyncConfig: CodeSyncConfig;

/**
* Payment related options
*/
chargebeeProviderOptions?: ChargebeeProviderOptions;
enablePayment?: boolean;
}

export namespace EnvConfig {
export function fromEnv(env: Env): Config {
return {
version: env.version,
hostUrl: env.hostUrl,
license: env.gitpodLicense,
trialLicensePrivateKey: env.trialLicensePrivateKey,
heartbeat: {
intervalSeconds: env.theiaHeartbeatInterval,
timeoutSeconds: env.workspaceUserTimeout,
},
workspaceDefaults: {
ideVersion: env.theiaVersion,
ideImageRepo: env.theiaImageRepo,
ideImage: env.ideDefaultImage,
ideImageAliases: env.ideImageAliases,
workspaceImage: env.workspaceDefaultImage,
previewFeatureFlags: env.previewFeatureFlags,
defaultFeatureFlags: env.defaultFeatureFlags,
},
session: {
maxAgeMs: env.sessionMaxAgeMs,
secret: env.sessionSecret,
},
githubApp: {
enabled: env.githubAppEnabled,
appId: env.githubAppAppID,
webhookSecret: env.githubAppWebhookSecret,
authProviderId: env.githubAppAuthProviderId,
certPath: env.githubAppCertPath,
marketplaceName: env.githubAppMarketplaceName,
logLevel: env.githubAppLogLevel,
},
definitelyGpDisabled: env.definitelyGpDisabled,
workspaceGarbageCollection: {
disabled: env.garbageCollectionDisabled,
startDate: env.garbageCollectionStartDate,
chunkLimit: env.garbageCollectionLimit,
minAgeDays: env.daysBeforeGarbageCollection,
minAgePrebuildDays: env.daysBeforeGarbageCollectingPrebuilds,
contentRetentionPeriodDays: env.workspaceDeletionRetentionPeriodDays,
contentChunkLimit: env.workspaceDeletionLimit,
},
enableLocalApp: env.enableLocalApp,
authProviderConfigs: env.authProviderConfigs,
disableDynamicAuthProviderLogin: env.disableDynamicAuthProviderLogin,
brandingConfig: env.brandingConfig,
maxEnvvarPerUserCount: env.maxUserEnvvarCount,
maxConcurrentPrebuildsPerRef: env.maxConcurrentPrebuildsPerRef,
incrementalPrebuilds: {
repositoryPassList: env.incrementalPrebuildsRepositoryPassList,
commitHistory: env.incrementalPrebuildsCommitHistory,
},
blockNewUsers: {
enabled: env.blockNewUsers,
passlist: env.blockNewUsersPassList,
},
makeNewUsersAdmin: env.makeNewUsersAdmin,
theiaPluginsBucketNameOverride: env.theiaPluginsBucketNameOverride,
defaultBaseImageRegistryWhitelist: env.defaultBaseImageRegistryWhitelist,
insecureNoDomain: env.insecureNoDomain,
runDbDeleter: env.runDbDeleter,
oauthServer: {
enabled: env.enableOAuthServer,
jwtSecret: env.oauthServerJWTSecret,
},
rateLimiter: env.rateLimiter,
contentServiceAddress: env.contentServiceAddress,
serverProxyApiKey: env.serverProxyApiKey,
codeSyncConfig: env.codeSyncConfig,
};
}
export function fromEnvEE(env: EnvEE): Config {
const config = EnvConfig.fromEnv(env);
return {
...config,
chargebeeProviderOptions: env.chargebeeProviderOptions,
enablePayment: env.enablePayment,
}
}
}

0 comments on commit ae0698e

Please sign in to comment.