From 638fc62601b58eb2a711f710e3cd80636af15798 Mon Sep 17 00:00:00 2001 From: forehalo Date: Tue, 28 May 2024 06:43:53 +0000 Subject: [PATCH] feat(server): runtime setting support (#5602) ---
Generated summary (powered by Graphite) > ## TL;DR > This pull request adds a new migration file, a new model, and new modules related to runtime settings. It also introduces a new `Runtime` service that allows getting, setting, and updating runtime configurations. > > ## What changed > - Added a new migration file `migration.sql` that creates a table called `application_settings` with columns `key` and `value`. > - Added a new model `ApplicationSetting` with properties `key` and `value`. > - Added a new module `RuntimeSettingModule` that exports the `Runtime` service. > - Added a new service `Runtime` that provides methods for getting, setting, and updating runtime configurations. > - Modified the `app.module.ts` file to import the `RuntimeSettingModule`. > - Modified the `index.ts` file in the `fundamentals` directory to export the `Runtime` service. > - Added a new file `def.ts` in the `runtime` directory that defines the runtime configurations and provides a default implementation. > - Added a new file `service.ts` in the `runtime` directory that implements the `Runtime` service. > > ## How to test > 1. Run the migration script to create the `application_settings` table. > 2. Use the `Runtime` service to get, set, and update runtime configurations. > 3. Verify that the runtime configurations are stored correctly in the database and can be retrieved and modified using the `Runtime` service. > > ## Why make this change > This change introduces a new feature related to runtime settings. The `Runtime` service allows the application to dynamically manage and modify runtime configurations without requiring a restart. This provides flexibility and allows for easier customization and configuration of the application.
--- .../migration.sql | 23 ++ packages/backend/server/schema.prisma | 44 ++- packages/backend/server/src/app.module.ts | 17 +- packages/backend/server/src/app.ts | 10 +- .../backend/server/src/config/affine.env.ts | 41 +- .../backend/server/src/config/affine.self.ts | 46 ++- packages/backend/server/src/config/affine.ts | 65 ++-- .../backend/server/src/core/auth/config.ts | 81 ++++ .../server/src/core/auth/controller.ts | 16 +- .../backend/server/src/core/auth/index.ts | 2 + .../backend/server/src/core/auth/resolver.ts | 45 +-- .../backend/server/src/core/auth/service.ts | 2 +- packages/backend/server/src/core/config.ts | 102 ----- .../backend/server/src/core/config/config.ts | 23 ++ .../backend/server/src/core/config/index.ts | 12 + .../server/src/core/config/resolver.ts | 207 ++++++++++ .../backend/server/src/core/config/types.ts | 5 + .../backend/server/src/core/doc/config.ts | 71 ++++ packages/backend/server/src/core/doc/index.ts | 2 + .../backend/server/src/core/doc/manager.ts | 10 +- .../server/src/core/features/management.ts | 6 +- .../backend/server/src/core/storage/config.ts | 30 ++ .../backend/server/src/core/storage/index.ts | 2 + .../src/core/storage/wrappers/avatar.ts | 16 +- .../server/src/core/storage/wrappers/blob.ts | 4 +- .../src/core/sync/events/events.gateway.ts | 14 +- .../server/src/core/utils/validators.ts | 45 +-- .../server/src/fundamentals/config/def.ts | 353 ++---------------- .../server/src/fundamentals/config/default.ts | 258 +++++-------- .../server/src/fundamentals/config/index.ts | 36 +- .../server/src/fundamentals/config/module.ts | 58 --- .../src/fundamentals/config/provider.ts | 19 + .../src/fundamentals/config/register.ts | 66 ++++ .../src/fundamentals/config/runtime/event.ts | 22 ++ .../fundamentals/config/runtime/service.ts | 242 ++++++++++++ .../server/src/fundamentals/config/types.ts | 127 +++++++ .../server/src/fundamentals/event/types.ts | 41 +- .../server/src/fundamentals/graphql/config.ts | 17 + .../server/src/fundamentals/graphql/index.ts | 4 +- .../helpers/__tests__/crypto.spec.ts | 8 +- .../helpers/__tests__/url.spec.ts | 8 +- .../server/src/fundamentals/helpers/config.ts | 53 +++ .../server/src/fundamentals/helpers/crypto.ts | 8 +- .../server/src/fundamentals/helpers/index.ts | 2 + .../server/src/fundamentals/helpers/url.ts | 33 +- .../backend/server/src/fundamentals/index.ts | 3 +- .../server/src/fundamentals/mailer/config.ts | 16 + .../server/src/fundamentals/mailer/index.ts | 4 +- .../server/src/fundamentals/metrics/config.ts | 33 ++ .../server/src/fundamentals/metrics/index.ts | 2 + .../server/src/fundamentals/nestjs/config.ts | 42 +++ .../server/src/fundamentals/nestjs/index.ts | 1 + .../fundamentals/nestjs/optional-module.ts | 4 +- .../storage/index.ts => storage/config.ts} | 16 +- .../server/src/fundamentals/storage/index.ts | 7 +- .../src/fundamentals/storage/providers/fs.ts | 2 +- .../fundamentals/storage/providers/index.ts | 13 +- .../storage/providers/provider.ts | 2 +- .../src/fundamentals/throttler/config.ts | 27 ++ .../src/fundamentals/throttler/decorators.ts | 4 +- .../src/fundamentals/throttler/index.ts | 26 +- .../server/src/fundamentals/utils/promise.ts | 10 +- .../server/src/fundamentals/utils/types.ts | 17 +- packages/backend/server/src/global.d.ts | 8 +- packages/backend/server/src/index.ts | 13 +- packages/backend/server/src/plugins/config.ts | 38 +- .../server/src/plugins/copilot/config.ts | 26 ++ .../server/src/plugins/copilot/index.ts | 4 +- .../src/plugins/copilot/providers/index.ts | 16 +- .../server/src/plugins/copilot/storage.ts | 8 +- .../server/src/plugins/copilot/types.ts | 9 - .../server/src/plugins/gcloud/config.ts | 15 +- .../server/src/plugins/gcloud/index.ts | 2 + packages/backend/server/src/plugins/index.ts | 6 +- .../src/plugins/oauth/{types.ts => config.ts} | 13 +- .../server/src/plugins/oauth/controller.ts | 2 +- .../backend/server/src/plugins/oauth/index.ts | 3 +- .../server/src/plugins/oauth/providers/def.ts | 2 +- .../src/plugins/oauth/providers/github.ts | 2 +- .../src/plugins/oauth/providers/google.ts | 2 +- .../src/plugins/oauth/providers/oidc.ts | 6 +- .../server/src/plugins/oauth/register.ts | 2 +- .../server/src/plugins/oauth/resolver.ts | 2 +- .../server/src/plugins/oauth/service.ts | 2 +- .../server/src/plugins/payment/config.ts | 20 + .../server/src/plugins/payment/index.ts | 4 +- .../server/src/plugins/payment/resolver.ts | 11 +- .../server/src/plugins/payment/stripe.ts | 2 +- .../server/src/plugins/payment/types.ts | 10 - .../server/src/plugins/payment/webhook.ts | 2 +- .../server/src/plugins/redis/config.ts | 11 + .../backend/server/src/plugins/redis/index.ts | 5 +- .../server/src/plugins/redis/instances.ts | 6 +- .../backend/server/src/plugins/redis/types.ts | 3 - .../backend/server/src/plugins/registry.ts | 16 +- .../server/src/plugins/storage/config.ts | 27 ++ .../server/src/plugins/storage/index.ts | 4 +- .../src/plugins/storage/providers/r2.ts | 5 +- .../src/plugins/storage/providers/s3.ts | 2 +- .../server/src/plugins/storage/types.ts | 16 - packages/backend/server/src/prelude.ts | 14 +- packages/backend/server/src/schema.gql | 42 ++- .../server/tests/auth/controller.spec.ts | 2 +- packages/backend/server/tests/cache.spec.ts | 10 +- packages/backend/server/tests/config.spec.ts | 19 +- packages/backend/server/tests/doc.spec.ts | 3 +- packages/backend/server/tests/feature.spec.ts | 11 +- packages/backend/server/tests/graphql.spec.ts | 21 +- .../server/tests/nestjs/throttler.spec.ts | 8 +- .../server/tests/oauth/controller.spec.ts | 7 +- .../server/tests/payment/service.spec.ts | 2 + packages/backend/server/tests/utils/utils.ts | 3 +- .../plans/ai/actions/subscribe.tsx | 2 +- .../general-setting/plans/plan-card.tsx | 2 +- .../frontend/core/src/pages/subscribe.tsx | 5 +- packages/frontend/graphql/src/schema.ts | 10 +- 116 files changed, 1906 insertions(+), 1105 deletions(-) create mode 100644 packages/backend/server/migrations/20240520055805_runtime_setting/migration.sql create mode 100644 packages/backend/server/src/core/auth/config.ts delete mode 100644 packages/backend/server/src/core/config.ts create mode 100644 packages/backend/server/src/core/config/config.ts create mode 100644 packages/backend/server/src/core/config/index.ts create mode 100644 packages/backend/server/src/core/config/resolver.ts create mode 100644 packages/backend/server/src/core/config/types.ts create mode 100644 packages/backend/server/src/core/doc/config.ts create mode 100644 packages/backend/server/src/core/storage/config.ts delete mode 100644 packages/backend/server/src/fundamentals/config/module.ts create mode 100644 packages/backend/server/src/fundamentals/config/provider.ts create mode 100644 packages/backend/server/src/fundamentals/config/register.ts create mode 100644 packages/backend/server/src/fundamentals/config/runtime/event.ts create mode 100644 packages/backend/server/src/fundamentals/config/runtime/service.ts create mode 100644 packages/backend/server/src/fundamentals/config/types.ts create mode 100644 packages/backend/server/src/fundamentals/graphql/config.ts create mode 100644 packages/backend/server/src/fundamentals/helpers/config.ts create mode 100644 packages/backend/server/src/fundamentals/mailer/config.ts create mode 100644 packages/backend/server/src/fundamentals/metrics/config.ts create mode 100644 packages/backend/server/src/fundamentals/nestjs/config.ts rename packages/backend/server/src/fundamentals/{config/storage/index.ts => storage/config.ts} (81%) create mode 100644 packages/backend/server/src/fundamentals/throttler/config.ts create mode 100644 packages/backend/server/src/plugins/copilot/config.ts rename packages/backend/server/src/plugins/oauth/{types.ts => config.ts} (74%) create mode 100644 packages/backend/server/src/plugins/payment/config.ts create mode 100644 packages/backend/server/src/plugins/redis/config.ts delete mode 100644 packages/backend/server/src/plugins/redis/types.ts create mode 100644 packages/backend/server/src/plugins/storage/config.ts delete mode 100644 packages/backend/server/src/plugins/storage/types.ts diff --git a/packages/backend/server/migrations/20240520055805_runtime_setting/migration.sql b/packages/backend/server/migrations/20240520055805_runtime_setting/migration.sql new file mode 100644 index 0000000000000..d32546a29fbba --- /dev/null +++ b/packages/backend/server/migrations/20240520055805_runtime_setting/migration.sql @@ -0,0 +1,23 @@ +-- CreateEnum +CREATE TYPE "RuntimeConfigType" AS ENUM ('String', 'Number', 'Boolean', 'Object', 'Array'); + +-- CreateTable +CREATE TABLE "app_runtime_settings" ( + "id" VARCHAR NOT NULL, + "type" "RuntimeConfigType" NOT NULL, + "module" VARCHAR NOT NULL, + "key" VARCHAR NOT NULL, + "value" JSON NOT NULL, + "description" TEXT NOT NULL, + "updated_at" TIMESTAMPTZ(6) NOT NULL, + "deleted_at" TIMESTAMPTZ(6), + "last_updated_by" VARCHAR(36), + + CONSTRAINT "app_runtime_settings_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "app_runtime_settings_module_key_key" ON "app_runtime_settings"("module", "key"); + +-- AddForeignKey +ALTER TABLE "app_runtime_settings" ADD CONSTRAINT "app_runtime_settings_last_updated_by_fkey" FOREIGN KEY ("last_updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prisma index 0f1e2801ce0fd..3d5502282f36a 100644 --- a/packages/backend/server/schema.prisma +++ b/packages/backend/server/schema.prisma @@ -22,15 +22,16 @@ model User { /// for example, the value will be false if user never registered and invited into a workspace by others. registered Boolean @default(true) - features UserFeatures[] - customer UserStripeCustomer? - subscriptions UserSubscription[] - invoices UserInvoice[] - workspacePermissions WorkspaceUserPermission[] - pagePermissions WorkspacePageUserPermission[] - connectedAccounts ConnectedAccount[] - sessions UserSession[] - aiSessions AiSession[] + features UserFeatures[] + customer UserStripeCustomer? + subscriptions UserSubscription[] + invoices UserInvoice[] + workspacePermissions WorkspaceUserPermission[] + pagePermissions WorkspacePageUserPermission[] + connectedAccounts ConnectedAccount[] + sessions UserSession[] + aiSessions AiSession[] + updatedRuntimeConfigs RuntimeConfig[] @@index([email]) @@map("users") @@ -505,3 +506,28 @@ model DataMigration { @@map("_data_migrations") } + +enum RuntimeConfigType { + String + Number + Boolean + Object + Array +} + +model RuntimeConfig { + id String @id @db.VarChar + type RuntimeConfigType + module String @db.VarChar + key String @db.VarChar + value Json @db.Json + description String @db.Text + updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz(6) + deletedAt DateTime? @map("deleted_at") @db.Timestamptz(6) + lastUpdatedBy String? @map("last_updated_by") @db.VarChar(36) + + lastUpdatedByUser User? @relation(fields: [lastUpdatedBy], references: [id]) + + @@unique([module, key]) + @@map("app_runtime_settings") +} diff --git a/packages/backend/server/src/app.module.ts b/packages/backend/server/src/app.module.ts index 3948a0b282c04..d6e2a7c9d6ce7 100644 --- a/packages/backend/server/src/app.module.ts +++ b/packages/backend/server/src/app.module.ts @@ -17,8 +17,11 @@ import { UserModule } from './core/user'; import { WorkspaceModule } from './core/workspaces'; import { getOptionalModuleMetadata } from './fundamentals'; import { CacheModule } from './fundamentals/cache'; -import type { AvailablePlugins } from './fundamentals/config'; -import { Config, ConfigModule } from './fundamentals/config'; +import { + AFFiNEConfig, + ConfigModule, + mergeConfigOverride, +} from './fundamentals/config'; import { EventModule } from './fundamentals/event'; import { GqlModule } from './fundamentals/graphql'; import { HelpersModule } from './fundamentals/helpers'; @@ -30,6 +33,7 @@ import { StorageProviderModule } from './fundamentals/storage'; import { RateLimiterModule } from './fundamentals/throttler'; import { WebSocketModule } from './fundamentals/websocket'; import { REGISTERED_PLUGINS } from './plugins'; +import { ENABLED_PLUGINS } from './plugins/registry'; export const FunctionalityModules = [ ConfigModule.forRoot(), @@ -47,7 +51,7 @@ export const FunctionalityModules = [ export class AppModuleBuilder { private readonly modules: AFFiNEModule[] = []; - constructor(private readonly config: Config) {} + constructor(private readonly config: AFFiNEConfig) {} use(...modules: AFFiNEModule[]): this { modules.forEach(m => { @@ -90,7 +94,7 @@ export class AppModuleBuilder { } useIf( - predicator: (config: Config) => boolean, + predicator: (config: AFFiNEConfig) => boolean, ...modules: AFFiNEModule[] ): this { if (predicator(this.config)) { @@ -112,6 +116,7 @@ export class AppModuleBuilder { } function buildAppModule() { + AFFiNE = mergeConfigOverride(AFFiNE); const factor = new AppModuleBuilder(AFFiNE); factor @@ -147,8 +152,8 @@ function buildAppModule() { ); // plugin modules - AFFiNE.plugins.enabled.forEach(name => { - const plugin = REGISTERED_PLUGINS.get(name as AvailablePlugins); + ENABLED_PLUGINS.forEach(name => { + const plugin = REGISTERED_PLUGINS.get(name); if (!plugin) { throw new Error(`Unknown plugin ${name}`); } diff --git a/packages/backend/server/src/app.ts b/packages/backend/server/src/app.ts index c5cfca00f9960..369b5745c988b 100644 --- a/packages/backend/server/src/app.ts +++ b/packages/backend/server/src/app.ts @@ -50,11 +50,13 @@ export async function createApp() { app.useWebSocketAdapter(adapter); } - if (AFFiNE.isSelfhosted && AFFiNE.telemetry.enabled) { + if (AFFiNE.isSelfhosted && AFFiNE.metrics.telemetry.enabled) { const mixpanel = await import('mixpanel'); - mixpanel.init(AFFiNE.telemetry.token).track('selfhost-server-started', { - version: AFFiNE.version, - }); + mixpanel + .init(AFFiNE.metrics.telemetry.token) + .track('selfhost-server-started', { + version: AFFiNE.version, + }); } return app; diff --git a/packages/backend/server/src/config/affine.env.ts b/packages/backend/server/src/config/affine.env.ts index d0c447a6a1ae1..aa75011d133f3 100644 --- a/packages/backend/server/src/config/affine.env.ts +++ b/packages/backend/server/src/config/affine.env.ts @@ -1,31 +1,22 @@ // Convenient way to map environment variables to config values. AFFiNE.ENV_MAP = { - AFFINE_SERVER_PORT: ['port', 'int'], - AFFINE_SERVER_HOST: 'host', - AFFINE_SERVER_SUB_PATH: 'path', - AFFINE_SERVER_HTTPS: ['https', 'boolean'], - DATABASE_URL: 'db.url', - ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'], - CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'], - OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId', - OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret', - OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId', - OAUTH_GITHUB_CLIENT_SECRET: 'plugins.oauth.providers.github.clientSecret', - OAUTH_OIDC_ISSUER: 'plugins.oauth.providers.oidc.issuer', - OAUTH_OIDC_CLIENT_ID: 'plugins.oauth.providers.oidc.clientId', - OAUTH_OIDC_CLIENT_SECRET: 'plugins.oauth.providers.oidc.clientSecret', - OAUTH_OIDC_SCOPE: 'plugins.oauth.providers.oidc.args.scope', - OAUTH_OIDC_CLAIM_MAP_USERNAME: 'plugins.oauth.providers.oidc.args.claim_id', - OAUTH_OIDC_CLAIM_MAP_EMAIL: 'plugins.oauth.providers.oidc.args.claim_email', - OAUTH_OIDC_CLAIM_MAP_NAME: 'plugins.oauth.providers.oidc.args.claim_name', + AFFINE_SERVER_PORT: ['server.port', 'int'], + AFFINE_SERVER_HOST: 'server.host', + AFFINE_SERVER_SUB_PATH: 'server.path', + AFFINE_SERVER_HTTPS: ['server.https', 'boolean'], + ENABLE_TELEMETRY: ['metrics.telemetry.enabled', 'boolean'], MAILER_HOST: 'mailer.host', MAILER_PORT: ['mailer.port', 'int'], MAILER_USER: 'mailer.auth.user', MAILER_PASSWORD: 'mailer.auth.pass', MAILER_SENDER: 'mailer.from.address', MAILER_SECURE: ['mailer.secure', 'boolean'], - THROTTLE_TTL: ['rateLimiter.ttl', 'int'], - THROTTLE_LIMIT: ['rateLimiter.limit', 'int'], + OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId', + OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret', + OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId', + OAUTH_GITHUB_CLIENT_SECRET: 'plugins.oauth.providers.github.clientSecret', + OAUTH_OIDC_CLIENT_ID: 'plugins.oauth.providers.oidc.clientId', + OAUTH_OIDC_CLIENT_SECRET: 'plugins.oauth.providers.oidc.clientSecret', METRICS_CUSTOMER_IO_TOKEN: ['metrics.customerIo.token', 'string'], COPILOT_OPENAI_API_KEY: 'plugins.copilot.openai.apiKey', COPILOT_FAL_API_KEY: 'plugins.copilot.fal.apiKey', @@ -36,16 +27,6 @@ AFFiNE.ENV_MAP = { REDIS_SERVER_PASSWORD: 'plugins.redis.password', REDIS_SERVER_DATABASE: ['plugins.redis.db', 'int'], DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'], - DOC_MERGE_USE_JWST_CODEC: [ - 'doc.manager.experimentalMergeWithYOcto', - 'boolean', - ], STRIPE_API_KEY: 'plugins.payment.stripe.keys.APIKey', STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey', - FEATURES_EARLY_ACCESS_PREVIEW: ['featureFlags.earlyAccessPreview', 'boolean'], - FEATURES_SYNC_CLIENT_VERSION_CHECK: [ - 'featureFlags.syncClientVersionCheck', - 'boolean', - ], - TELEMETRY_ENABLE: ['telemetry.enabled', 'boolean'], }; diff --git a/packages/backend/server/src/config/affine.self.ts b/packages/backend/server/src/config/affine.self.ts index 43597d5250325..1e191cea4707a 100644 --- a/packages/backend/server/src/config/affine.self.ts +++ b/packages/backend/server/src/config/affine.self.ts @@ -20,35 +20,47 @@ const env = process.env; AFFiNE.metrics.enabled = !AFFiNE.node.test; if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) { - AFFiNE.plugins.use('cloudflare-r2', { + AFFiNE.use('cloudflare-r2', { accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID, credentials: { accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!, secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!, }, }); - AFFiNE.storage.storages.avatar.provider = 'cloudflare-r2'; - AFFiNE.storage.storages.avatar.bucket = 'account-avatar'; - AFFiNE.storage.storages.avatar.publicLinkFactory = key => + AFFiNE.storages.avatar.provider = 'cloudflare-r2'; + AFFiNE.storages.avatar.bucket = 'account-avatar'; + AFFiNE.storages.avatar.publicLinkFactory = key => `https://avatar.affineassets.com/${key}`; - AFFiNE.storage.storages.blob.provider = 'cloudflare-r2'; - AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${ + AFFiNE.storages.blob.provider = 'cloudflare-r2'; + AFFiNE.storages.blob.bucket = `workspace-blobs-${ AFFiNE.affine.canary ? 'canary' : 'prod' }`; - AFFiNE.storage.storages.copilot.provider = 'cloudflare-r2'; - AFFiNE.storage.storages.copilot.bucket = `workspace-copilot-${ - AFFiNE.affine.canary ? 'canary' : 'prod' - }`; + AFFiNE.use('copilot', { + storage: { + provider: 'cloudflare-r2', + bucket: `workspace-copilot-${AFFiNE.affine.canary ? 'canary' : 'prod'}`, + }, + }); } -AFFiNE.plugins.use('copilot', { - openai: {}, - fal: {}, +AFFiNE.use('copilot', { + openai: { + apiKey: '', + }, + fal: { + apiKey: '', + }, +}); +AFFiNE.use('redis', { + host: env.REDIS_SERVER_HOST, + db: 0, + port: 6379, + username: env.REDIS_SERVER_USER, + password: env.REDIS_SERVER_PASSWORD, }); -AFFiNE.plugins.use('redis'); -AFFiNE.plugins.use('payment', { +AFFiNE.use('payment', { stripe: { keys: { // fake the key to ensure the server generate full GraphQL Schema even env vars are not set @@ -57,7 +69,7 @@ AFFiNE.plugins.use('payment', { }, }, }); -AFFiNE.plugins.use('oauth'); +AFFiNE.use('oauth'); if (AFFiNE.deploy) { AFFiNE.mailer = { @@ -68,5 +80,5 @@ if (AFFiNE.deploy) { }, }; - AFFiNE.plugins.use('gcloud'); + AFFiNE.use('gcloud'); } diff --git a/packages/backend/server/src/config/affine.ts b/packages/backend/server/src/config/affine.ts index 107b426dd584a..0f3748f41206c 100644 --- a/packages/backend/server/src/config/affine.ts +++ b/packages/backend/server/src/config/affine.ts @@ -26,22 +26,14 @@ // AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud'; // // /* Whether the server is deployed behind a HTTPS proxied environment */ -AFFiNE.https = false; +AFFiNE.server.https = false; // /* Domain of your server that your server will be available at */ -AFFiNE.host = 'localhost'; +AFFiNE.server.host = 'localhost'; // /* The local port of your server that will listen on */ -AFFiNE.port = 3010; +AFFiNE.server.port = 3010; // /* The sub path of your server */ -// /* For example, if you set `AFFiNE.path = '/affine'`, then the server will be available at `${domain}/affine` */ -// AFFiNE.path = '/affine'; -// -// -// ############################################################### -// ## Database settings ## -// ############################################################### -// -// /* The URL of the database where most of AFFiNE server data will be stored in */ -// AFFiNE.db.url = 'postgres://user:passsword@localhost:5432/affine'; +// /* For example, if you set `AFFiNE.server.path = '/affine'`, then the server will be available at `${domain}/affine` */ +// AFFiNE.server.path = '/affine'; // // // ############################################################### @@ -52,19 +44,12 @@ AFFiNE.port = 3010; // /* The metrics will be available at `http://localhost:9464/metrics` with [Prometheus] format exported */ // AFFiNE.metrics.enabled = true; // -// /* Authentication Settings */ -// /* Whether allow anyone signup */ -// AFFiNE.auth.allowSignup = true; -// -// /* User Signup password limitation */ -// AFFiNE.auth.password = { -// minLength: 8, -// maxLength: 32, -// }; // -// /* How long the login session would last by default */ // AFFiNE.auth.session = { +// /* How long the login session would last by default */ // ttl: 15 * 24 * 60 * 60, // 15 days +// /* How long we should refresh the token before it getting expired */ +// ttr: 7 * 24 * 60 * 60, // 7 days // }; // // /* GraphQL configurations that control the behavior of the Apollo Server behind */ @@ -85,9 +70,6 @@ AFFiNE.port = 3010; // /* How long the buffer time of creating a new history snapshot when doc get updated */ // AFFiNE.doc.history.interval = 1000 * 60 * 10; // 10 minutes // -// /* Use `y-octo` to merge updates at the same time when merging using Yjs */ -// AFFiNE.doc.manager.experimentalMergeWithYOcto = true; -// // /* How often the manager will start a new turn of merging pending updates into doc snapshot */ // AFFiNE.doc.manager.updatePollInterval = 1000 * 3; // @@ -99,20 +81,20 @@ AFFiNE.port = 3010; // /* Redis Plugin */ // /* Provide caching and session storing backed by Redis. */ // /* Useful when you deploy AFFiNE server in a cluster. */ -// AFFiNE.plugins.use('redis', { +// AFFiNE.use('redis', { // /* override options */ // }); // // // /* Payment Plugin */ -// AFFiNE.plugins.use('payment', { +// AFFiNE.use('payment', { // stripe: { keys: {}, apiVersion: '2023-10-16' }, // }); // // // /* Cloudflare R2 Plugin */ // /* Enable if you choose to store workspace blobs or user avatars in Cloudflare R2 Storage Service */ -// AFFiNE.plugins.use('cloudflare-r2', { +// AFFiNE.use('cloudflare-r2', { // accountId: '', // credentials: { // accessKeyId: '', @@ -122,17 +104,17 @@ AFFiNE.port = 3010; // // /* AWS S3 Plugin */ // /* Enable if you choose to store workspace blobs or user avatars in AWS S3 Storage Service */ -// AFFiNE.plugins.use('aws-s3', { +// AFFiNE.use('aws-s3', { // credentials: { // accessKeyId: '', // secretAccessKey: '', // }) // /* Update the provider of storages */ -// AFFiNE.storage.storages.blob.provider = 'r2'; -// AFFiNE.storage.storages.avatar.provider = 'r2'; +// AFFiNE.storages.blob.provider = 'cloudflare-r2'; +// AFFiNE.storages.avatar.provider = 'cloudflare-r2'; // -/* OAuth Plugin */ -// AFFiNE.plugins.use('oauth', { +// /* OAuth Plugin */ +// AFFiNE.use('oauth', { // providers: { // github: { // clientId: '', @@ -166,3 +148,18 @@ AFFiNE.port = 3010; // }, // }, // }); +// +// /* Copilot Plugin */ +// AFFiNE.use('copilot', { +// openai: { +// apiKey: 'your-key', +// }, +// fal: { +// apiKey: 'your-key', +// }, +// unsplashKey: 'your-key', +// storage: { +// provider: 'cloudflare-r2', +// bucket: 'copilot', +// } +// }) diff --git a/packages/backend/server/src/core/auth/config.ts b/packages/backend/server/src/core/auth/config.ts new file mode 100644 index 0000000000000..b63b888a8e3fb --- /dev/null +++ b/packages/backend/server/src/core/auth/config.ts @@ -0,0 +1,81 @@ +import { + defineRuntimeConfig, + defineStartupConfig, + ModuleConfig, +} from '../../fundamentals/config'; + +export interface AuthStartupConfigurations { + /** + * auth session config + */ + session: { + /** + * Application auth expiration time in seconds + */ + ttl: number; + /** + * Application auth time to refresh in seconds + */ + ttr: number; + }; + + /** + * Application access token config + */ + accessToken: { + /** + * Application access token expiration time in seconds + */ + ttl: number; + /** + * Application refresh token expiration time in seconds + */ + refreshTokenTtl: number; + }; +} + +export interface AuthRuntimeConfigurations { + /** + * Whether allow anonymous users to sign up + */ + allowSignup: boolean; + /** + * The minimum and maximum length of the password when registering new users + */ + password: { + min: number; + max: number; + }; +} + +declare module '../../fundamentals/config' { + interface AppConfig { + auth: ModuleConfig; + } +} + +defineStartupConfig('auth', { + session: { + ttl: 60 * 60 * 24 * 15, // 15 days + ttr: 60 * 60 * 24 * 7, // 7 days + }, + accessToken: { + ttl: 60 * 60 * 24 * 7, // 7 days + refreshTokenTtl: 60 * 60 * 24 * 30, // 30 days + }, +}); + +defineRuntimeConfig('auth', { + allowSignup: { + desc: 'Whether allow new registrations', + default: true, + }, + 'password.min': { + desc: 'The minimum length of user password', + default: 8, + }, + 'password.max': { + desc: 'The maximum length of user password', + default: 32, + }, +}); diff --git a/packages/backend/server/src/core/auth/controller.ts b/packages/backend/server/src/core/auth/controller.ts index 26a24da87250a..47e1635d89ad5 100644 --- a/packages/backend/server/src/core/auth/controller.ts +++ b/packages/backend/server/src/core/auth/controller.ts @@ -14,12 +14,7 @@ import { } from '@nestjs/common'; import type { Request, Response } from 'express'; -import { - Config, - PaymentRequiredException, - Throttle, - URLHelper, -} from '../../fundamentals'; +import { Config, Throttle, URLHelper } from '../../fundamentals'; import { UserService } from '../user'; import { validators } from '../utils/validators'; import { CurrentUser } from './current-user'; @@ -60,7 +55,7 @@ export class AuthController { validators.assertValidEmail(credential.email); const canSignIn = await this.auth.canSignIn(credential.email); if (!canSignIn) { - throw new PaymentRequiredException( + throw new BadRequestException( `You don't have early access permission\nVisit https://community.affine.pro/c/insider-general/ for more information` ); } @@ -76,8 +71,11 @@ export class AuthController { } else { // send email magic link const user = await this.user.findUserByEmail(credential.email); - if (!user && !this.config.auth.allowSignup) { - throw new BadRequestException('You are not allows to sign up.'); + if (!user) { + const allowSignup = await this.config.runtime.fetch('auth/allowSignup'); + if (!allowSignup) { + throw new BadRequestException('You are not allows to sign up.'); + } } const result = await this.sendSignInEmail( diff --git a/packages/backend/server/src/core/auth/index.ts b/packages/backend/server/src/core/auth/index.ts index 6e5dcbc2d2d6d..c1551d67528ee 100644 --- a/packages/backend/server/src/core/auth/index.ts +++ b/packages/backend/server/src/core/auth/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { Module } from '@nestjs/common'; import { FeatureModule } from '../features'; diff --git a/packages/backend/server/src/core/auth/resolver.ts b/packages/backend/server/src/core/auth/resolver.ts index 3e18e961e4667..3cfa3ddbb6cc6 100644 --- a/packages/backend/server/src/core/auth/resolver.ts +++ b/packages/backend/server/src/core/auth/resolver.ts @@ -10,7 +10,7 @@ import { Resolver, } from '@nestjs/graphql'; -import { Config, SkipThrottle, Throttle } from '../../fundamentals'; +import { Config, SkipThrottle, Throttle, URLHelper } from '../../fundamentals'; import { UserService } from '../user'; import { UserType } from '../user/types'; import { validators } from '../utils/validators'; @@ -36,6 +36,7 @@ export class ClientTokenType { export class AuthResolver { constructor( private readonly config: Config, + private readonly url: URLHelper, private readonly auth: AuthService, private readonly user: UserService, private readonly token: TokenService @@ -83,7 +84,14 @@ export class AuthResolver { @Args('token') token: string, @Args('newPassword') newPassword: string ) { - validators.assertValidPassword(newPassword); + const config = await this.config.runtime.fetchAll({ + 'auth/password.max': true, + 'auth/password.min': true, + }); + validators.assertValidPassword(newPassword, { + min: config['auth/password.min'], + max: config['auth/password.max'], + }); // NOTE: Set & Change password are using the same token type. const valid = await this.token.verifyToken( TokenType.ChangePassword, @@ -144,13 +152,9 @@ export class AuthResolver { user.id ); - const url = new URL(callbackUrl, this.config.baseUrl); - url.searchParams.set('token', token); + const url = this.url.link(callbackUrl, { token }); - const res = await this.auth.sendChangePasswordEmail( - user.email, - url.toString() - ); + const res = await this.auth.sendChangePasswordEmail(user.email, url); return !res.rejected.length; } @@ -170,13 +174,9 @@ export class AuthResolver { user.id ); - const url = new URL(callbackUrl, this.config.baseUrl); - url.searchParams.set('token', token); + const url = this.url.link(callbackUrl, { token }); - const res = await this.auth.sendSetPasswordEmail( - user.email, - url.toString() - ); + const res = await this.auth.sendSetPasswordEmail(user.email, url); return !res.rejected.length; } @@ -200,10 +200,9 @@ export class AuthResolver { const token = await this.token.createToken(TokenType.ChangeEmail, user.id); - const url = new URL(callbackUrl, this.config.baseUrl); - url.searchParams.set('token', token); + const url = this.url.link(callbackUrl, { token }); - const res = await this.auth.sendChangeEmail(user.email, url.toString()); + const res = await this.auth.sendChangeEmail(user.email, url); return !res.rejected.length; } @@ -240,11 +239,8 @@ export class AuthResolver { user.id ); - const url = new URL(callbackUrl, this.config.baseUrl); - url.searchParams.set('token', verifyEmailToken); - url.searchParams.set('email', email); - - const res = await this.auth.sendVerifyChangeEmail(email, url.toString()); + const url = this.url.link(callbackUrl, { token: verifyEmailToken, email }); + const res = await this.auth.sendVerifyChangeEmail(email, url); return !res.rejected.length; } @@ -256,10 +252,9 @@ export class AuthResolver { ) { const token = await this.token.createToken(TokenType.VerifyEmail, user.id); - const url = new URL(callbackUrl, this.config.baseUrl); - url.searchParams.set('token', token); + const url = this.url.link(callbackUrl, { token }); - const res = await this.auth.sendVerifyEmail(user.email, url.toString()); + const res = await this.auth.sendVerifyEmail(user.email, url); return !res.rejected.length; } diff --git a/packages/backend/server/src/core/auth/service.ts b/packages/backend/server/src/core/auth/service.ts index 59c1f95d1e6ad..672250942273c 100644 --- a/packages/backend/server/src/core/auth/service.ts +++ b/packages/backend/server/src/core/auth/service.ts @@ -61,7 +61,7 @@ export class AuthService implements OnApplicationBootstrap { sameSite: 'lax', httpOnly: true, path: '/', - secure: this.config.https, + secure: this.config.server.https, }; static readonly sessionCookieName = 'affine_session'; static readonly authUserSeqHeaderName = 'x-auth-user'; diff --git a/packages/backend/server/src/core/config.ts b/packages/backend/server/src/core/config.ts deleted file mode 100644 index 2d09d92c3f154..0000000000000 --- a/packages/backend/server/src/core/config.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { Module } from '@nestjs/common'; -import { Field, ObjectType, Query, registerEnumType } from '@nestjs/graphql'; - -import { DeploymentType } from '../fundamentals'; -import { Public } from './auth'; - -export enum ServerFeature { - Copilot = 'copilot', - Payment = 'payment', - OAuth = 'oauth', -} - -registerEnumType(ServerFeature, { - name: 'ServerFeature', -}); - -registerEnumType(DeploymentType, { - name: 'ServerDeploymentType', -}); - -const ENABLED_FEATURES: Set = new Set(); -export function ADD_ENABLED_FEATURES(feature: ServerFeature) { - ENABLED_FEATURES.add(feature); -} - -@ObjectType() -export class PasswordLimitsType { - @Field() - minLength!: number; - @Field() - maxLength!: number; -} - -@ObjectType() -export class CredentialsRequirementType { - @Field() - password!: PasswordLimitsType; -} - -@ObjectType() -export class ServerConfigType { - @Field({ - description: - 'server identical name could be shown as badge on user interface', - }) - name!: string; - - @Field({ description: 'server version' }) - version!: string; - - @Field({ description: 'server base url' }) - baseUrl!: string; - - @Field(() => DeploymentType, { description: 'server type' }) - type!: DeploymentType; - - /** - * @deprecated - */ - @Field({ description: 'server flavor', deprecationReason: 'use `features`' }) - flavor!: string; - - @Field(() => [ServerFeature], { description: 'enabled server features' }) - features!: ServerFeature[]; - - @Field(() => CredentialsRequirementType, { - description: 'credentials requirement', - }) - credentialsRequirement!: CredentialsRequirementType; - - @Field({ description: 'enable telemetry' }) - enableTelemetry!: boolean; -} - -export class ServerConfigResolver { - @Public() - @Query(() => ServerConfigType, { - description: 'server config', - }) - serverConfig(): ServerConfigType { - return { - name: AFFiNE.serverName, - version: AFFiNE.version, - baseUrl: AFFiNE.baseUrl, - type: AFFiNE.type, - // BACKWARD COMPATIBILITY - // the old flavors contains `selfhosted` but it actually not flavor but deployment type - // this field should be removed after frontend feature flags implemented - flavor: AFFiNE.type, - features: Array.from(ENABLED_FEATURES), - credentialsRequirement: { - password: AFFiNE.auth.password, - }, - enableTelemetry: AFFiNE.telemetry.enabled, - }; - } -} - -@Module({ - providers: [ServerConfigResolver], -}) -export class ServerConfigModule {} diff --git a/packages/backend/server/src/core/config/config.ts b/packages/backend/server/src/core/config/config.ts new file mode 100644 index 0000000000000..8a0ba601025e9 --- /dev/null +++ b/packages/backend/server/src/core/config/config.ts @@ -0,0 +1,23 @@ +import { defineRuntimeConfig, ModuleConfig } from '../../fundamentals/config'; + +export interface ServerFlags { + earlyAccessControl: boolean; + syncClientVersionCheck: boolean; +} + +declare module '../../fundamentals/config' { + interface AppConfig { + flags: ModuleConfig; + } +} + +defineRuntimeConfig('flags', { + earlyAccessControl: { + desc: 'Only allow users with early access features to access the app', + default: false, + }, + syncClientVersionCheck: { + desc: 'Only allow client with exact the same version with server to establish sync connections', + default: false, + }, +}); diff --git a/packages/backend/server/src/core/config/index.ts b/packages/backend/server/src/core/config/index.ts new file mode 100644 index 0000000000000..15dc42ae39770 --- /dev/null +++ b/packages/backend/server/src/core/config/index.ts @@ -0,0 +1,12 @@ +import './config'; + +import { Module } from '@nestjs/common'; + +import { ServerConfigResolver, ServerRuntimeConfigResolver } from './resolver'; + +@Module({ + providers: [ServerConfigResolver, ServerRuntimeConfigResolver], +}) +export class ServerConfigModule {} +export { ADD_ENABLED_FEATURES, ServerConfigType } from './resolver'; +export { ServerFeature } from './types'; diff --git a/packages/backend/server/src/core/config/resolver.ts b/packages/backend/server/src/core/config/resolver.ts new file mode 100644 index 0000000000000..63df8fdce07d1 --- /dev/null +++ b/packages/backend/server/src/core/config/resolver.ts @@ -0,0 +1,207 @@ +import { + Args, + Field, + GraphQLISODateTime, + Mutation, + ObjectType, + Query, + registerEnumType, + ResolveField, + Resolver, +} from '@nestjs/graphql'; +import { RuntimeConfig, RuntimeConfigType } from '@prisma/client'; +import { GraphQLJSON, GraphQLJSONObject } from 'graphql-scalars'; + +import { Config, DeploymentType, URLHelper } from '../../fundamentals'; +import { Public } from '../auth'; +import { Admin } from '../common'; +import { ServerFlags } from './config'; +import { ServerFeature } from './types'; + +const ENABLED_FEATURES: Set = new Set(); +export function ADD_ENABLED_FEATURES(feature: ServerFeature) { + ENABLED_FEATURES.add(feature); +} + +registerEnumType(ServerFeature, { + name: 'ServerFeature', +}); + +registerEnumType(DeploymentType, { + name: 'ServerDeploymentType', +}); + +@ObjectType() +export class PasswordLimitsType { + @Field() + minLength!: number; + @Field() + maxLength!: number; +} + +@ObjectType() +export class CredentialsRequirementType { + @Field() + password!: PasswordLimitsType; +} + +@ObjectType() +export class ServerConfigType { + @Field({ + description: + 'server identical name could be shown as badge on user interface', + }) + name!: string; + + @Field({ description: 'server version' }) + version!: string; + + @Field({ description: 'server base url' }) + baseUrl!: string; + + @Field(() => DeploymentType, { description: 'server type' }) + type!: DeploymentType; + + /** + * @deprecated + */ + @Field({ description: 'server flavor', deprecationReason: 'use `features`' }) + flavor!: string; + + @Field(() => [ServerFeature], { description: 'enabled server features' }) + features!: ServerFeature[]; + + @Field({ description: 'enable telemetry' }) + enableTelemetry!: boolean; +} + +registerEnumType(RuntimeConfigType, { + name: 'RuntimeConfigType', +}); +@ObjectType() +export class ServerRuntimeConfigType implements Partial { + @Field() + id!: string; + + @Field() + module!: string; + + @Field() + key!: string; + + @Field() + description!: string; + + @Field(() => GraphQLJSON) + value!: any; + + @Field(() => RuntimeConfigType) + type!: RuntimeConfigType; + + @Field(() => GraphQLISODateTime) + updatedAt!: Date; +} + +@ObjectType() +export class ServerFlagsType implements ServerFlags { + @Field() + earlyAccessControl!: boolean; + + @Field() + syncClientVersionCheck!: boolean; +} + +@Resolver(() => ServerConfigType) +export class ServerConfigResolver { + constructor( + private readonly config: Config, + private readonly url: URLHelper + ) {} + + @Public() + @Query(() => ServerConfigType, { + description: 'server config', + }) + serverConfig(): ServerConfigType { + return { + name: this.config.serverName, + version: this.config.version, + baseUrl: this.url.home, + type: this.config.type, + // BACKWARD COMPATIBILITY + // the old flavors contains `selfhosted` but it actually not flavor but deployment type + // this field should be removed after frontend feature flags implemented + flavor: this.config.type, + features: Array.from(ENABLED_FEATURES), + enableTelemetry: this.config.metrics.telemetry.enabled, + }; + } + + @ResolveField(() => CredentialsRequirementType, { + description: 'credentials requirement', + }) + async credentialsRequirement() { + const config = await this.config.runtime.fetchAll({ + 'auth/password.max': true, + 'auth/password.min': true, + }); + + return { + password: { + minLength: config['auth/password.min'], + maxLength: config['auth/password.max'], + }, + }; + } + + @ResolveField(() => ServerFlagsType, { + description: 'server flags', + }) + async flags(): Promise { + const records = await this.config.runtime.list('flags'); + + return records.reduce((flags, record) => { + flags[record.key as keyof ServerFlagsType] = record.value as any; + return flags; + }, {} as ServerFlagsType); + } +} + +@Resolver(() => ServerRuntimeConfigType) +export class ServerRuntimeConfigResolver { + constructor(private readonly config: Config) {} + + @Admin() + @Query(() => [ServerRuntimeConfigType], { + description: 'get all server runtime configurable settings', + }) + serverRuntimeConfig(): Promise { + return this.config.runtime.list(); + } + + @Admin() + @Mutation(() => ServerRuntimeConfigType, { + description: 'update server runtime configurable setting', + }) + async updateRuntimeConfig( + @Args('id') id: string, + @Args({ type: () => GraphQLJSON, name: 'value' }) value: any + ): Promise { + return await this.config.runtime.set(id as any, value); + } + + @Admin() + @Mutation(() => [ServerRuntimeConfigType], { + description: 'update multiple server runtime configurable settings', + }) + async updateRuntimeConfigs( + @Args({ type: () => GraphQLJSONObject, name: 'updates' }) updates: any + ): Promise { + const keys = Object.keys(updates); + const results = await Promise.all( + keys.map(key => this.config.runtime.set(key as any, updates[key])) + ); + + return results; + } +} diff --git a/packages/backend/server/src/core/config/types.ts b/packages/backend/server/src/core/config/types.ts new file mode 100644 index 0000000000000..c8745b7f8ac8f --- /dev/null +++ b/packages/backend/server/src/core/config/types.ts @@ -0,0 +1,5 @@ +export enum ServerFeature { + Copilot = 'copilot', + Payment = 'payment', + OAuth = 'oauth', +} diff --git a/packages/backend/server/src/core/doc/config.ts b/packages/backend/server/src/core/doc/config.ts new file mode 100644 index 0000000000000..a0dcc7ae2981c --- /dev/null +++ b/packages/backend/server/src/core/doc/config.ts @@ -0,0 +1,71 @@ +import { + defineRuntimeConfig, + defineStartupConfig, + ModuleConfig, +} from '../../fundamentals/config'; + +interface DocStartupConfigurations { + manager: { + /** + * Whether auto merge updates into doc snapshot. + */ + enableUpdateAutoMerging: boolean; + + /** + * How often the [DocManager] will start a new turn of merging pending updates into doc snapshot. + * + * This is not the latency a new joint client will take to see the latest doc, + * but the buffer time we introduced to reduce the load of our service. + * + * in {ms} + */ + updatePollInterval: number; + + /** + * The maximum number of updates that will be pulled from the server at once. + * Existing for avoiding the server to be overloaded when there are too many updates for one doc. + */ + maxUpdatesPullCount: number; + }; + history: { + /** + * How long the buffer time of creating a new history snapshot when doc get updated. + * + * in {ms} + */ + interval: number; + }; +} + +interface DocRuntimeConfigurations { + /** + * Use `y-octo` to merge updates at the same time when merging using Yjs. + * + * This is an experimental feature, and aimed to check the correctness of JwstCodec. + */ + experimentalMergeWithYOcto: boolean; +} + +declare module '../../fundamentals/config' { + interface AppConfig { + doc: ModuleConfig; + } +} + +defineStartupConfig('doc', { + manager: { + enableUpdateAutoMerging: true, + updatePollInterval: 1000, + maxUpdatesPullCount: 100, + }, + history: { + interval: 1000, + }, +}); + +defineRuntimeConfig('doc', { + experimentalMergeWithYOcto: { + desc: 'Use `y-octo` to merge updates at the same time when merging using Yjs.', + default: false, + }, +}); diff --git a/packages/backend/server/src/core/doc/index.ts b/packages/backend/server/src/core/doc/index.ts index 3630ddc94332b..0fe915483d784 100644 --- a/packages/backend/server/src/core/doc/index.ts +++ b/packages/backend/server/src/core/doc/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { Module } from '@nestjs/common'; import { QuotaModule } from '../quota'; diff --git a/packages/backend/server/src/core/doc/manager.ts b/packages/backend/server/src/core/doc/manager.ts index 59691a75eea8c..7701944cb39b9 100644 --- a/packages/backend/server/src/core/doc/manager.ts +++ b/packages/backend/server/src/core/doc/manager.ts @@ -133,8 +133,11 @@ export class DocManager implements OnModuleInit, OnModuleDestroy { private async applyUpdates(guid: string, ...updates: Buffer[]): Promise { const doc = await this.recoverDoc(...updates); + const useYocto = await this.config.runtime.fetch( + 'doc/experimentalMergeWithYOcto' + ); // test jwst codec - if (this.config.doc.manager.experimentalMergeWithYOcto) { + if (useYocto) { metrics.jwst.counter('codec_merge_counter').add(1); const yjsResult = Buffer.from(encodeStateAsUpdate(doc)); let log = false; @@ -185,11 +188,6 @@ export class DocManager implements OnModuleInit, OnModuleDestroy { }, this.config.doc.manager.updatePollInterval); this.logger.log('Automation started'); - if (this.config.doc.manager.experimentalMergeWithYOcto) { - this.logger.warn( - 'Experimental feature enabled: merge updates with jwst codec is enabled' - ); - } } /** diff --git a/packages/backend/server/src/core/features/management.ts b/packages/backend/server/src/core/features/management.ts index c41476b734e58..3cd304d4d8a8b 100644 --- a/packages/backend/server/src/core/features/management.ts +++ b/packages/backend/server/src/core/features/management.ts @@ -95,7 +95,11 @@ export class FeatureManagementService { email: string, type: EarlyAccessType = EarlyAccessType.App ) { - if (this.config.featureFlags.earlyAccessPreview && !this.isStaff(email)) { + const earlyAccessControlEnabled = await this.config.runtime.fetch( + 'flags/earlyAccessControl' + ); + + if (earlyAccessControlEnabled && !this.isStaff(email)) { const user = await this.user.findUserByEmail(email); if (!user) { return false; diff --git a/packages/backend/server/src/core/storage/config.ts b/packages/backend/server/src/core/storage/config.ts new file mode 100644 index 0000000000000..57b739f4a7988 --- /dev/null +++ b/packages/backend/server/src/core/storage/config.ts @@ -0,0 +1,30 @@ +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; +import { StorageProviderType } from '../../fundamentals/storage'; + +export type StorageConfig = { + provider: StorageProviderType; + bucket: string; +} & Ext; + +export interface StorageStartupConfigurations { + avatar: StorageConfig<{ publicLinkFactory: (key: string) => string }>; + blob: StorageConfig; +} + +declare module '../../fundamentals/config' { + interface AppConfig { + storages: ModuleConfig; + } +} + +defineStartupConfig('storages', { + avatar: { + provider: 'fs', + bucket: 'avatars', + publicLinkFactory: key => `/api/avatars/${key}`, + }, + blob: { + provider: 'fs', + bucket: 'blobs', + }, +}); diff --git a/packages/backend/server/src/core/storage/index.ts b/packages/backend/server/src/core/storage/index.ts index 071931e24fdc6..0147e9260bc2d 100644 --- a/packages/backend/server/src/core/storage/index.ts +++ b/packages/backend/server/src/core/storage/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { Module } from '@nestjs/common'; import { AvatarStorage, WorkspaceBlobStorage } from './wrappers'; diff --git a/packages/backend/server/src/core/storage/wrappers/avatar.ts b/packages/backend/server/src/core/storage/wrappers/avatar.ts index cd41ea2f719f1..538a9c8e76035 100644 --- a/packages/backend/server/src/core/storage/wrappers/avatar.ts +++ b/packages/backend/server/src/core/storage/wrappers/avatar.ts @@ -6,19 +6,25 @@ import type { PutObjectMetadata, StorageProvider, } from '../../../fundamentals'; -import { Config, OnEvent, StorageProviderFactory } from '../../../fundamentals'; +import { + Config, + OnEvent, + StorageProviderFactory, + URLHelper, +} from '../../../fundamentals'; @Injectable() export class AvatarStorage { public readonly provider: StorageProvider; - private readonly storageConfig: Config['storage']['storages']['avatar']; + private readonly storageConfig: Config['storages']['avatar']; constructor( private readonly config: Config, + private readonly url: URLHelper, private readonly storageFactory: StorageProviderFactory ) { - this.provider = this.storageFactory.create('avatar'); - this.storageConfig = this.config.storage.storages.avatar; + this.storageConfig = this.config.storages.avatar; + this.provider = this.storageFactory.create(this.storageConfig); } async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) { @@ -26,7 +32,7 @@ export class AvatarStorage { let link = this.storageConfig.publicLinkFactory(key); if (link.startsWith('/')) { - link = this.config.baseUrl + link; + link = this.url.link(link); } return link; diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index b60b52759e76d..9aa991ab4b924 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common'; import { type BlobInputType, Cache, + Config, EventEmitter, type EventPayload, type ListObjectsMetadata, @@ -16,11 +17,12 @@ export class WorkspaceBlobStorage { public readonly provider: StorageProvider; constructor( + private readonly config: Config, private readonly event: EventEmitter, private readonly storageFactory: StorageProviderFactory, private readonly cache: Cache ) { - this.provider = this.storageFactory.create('blob'); + this.provider = this.storageFactory.create(this.config.storages.blob); } async put(workspaceId: string, key: string, blob: BlobInputType) { diff --git a/packages/backend/server/src/core/sync/events/events.gateway.ts b/packages/backend/server/src/core/sync/events/events.gateway.ts index 0064ca34b879f..516311205e0dc 100644 --- a/packages/backend/server/src/core/sync/events/events.gateway.ts +++ b/packages/backend/server/src/core/sync/events/events.gateway.ts @@ -11,7 +11,7 @@ import { import { Server, Socket } from 'socket.io'; import { encodeStateAsUpdate, encodeStateVector } from 'yjs'; -import { CallTimer, metrics } from '../../../fundamentals'; +import { CallTimer, Config, metrics } from '../../../fundamentals'; import { Auth, CurrentUser } from '../../auth'; import { DocManager } from '../../doc'; import { DocID } from '../../utils/doc'; @@ -98,6 +98,7 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect { private connectionCount = 0; constructor( + private readonly config: Config, private readonly docManager: DocManager, private readonly permissions: PermissionService ) {} @@ -115,10 +116,13 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect { metrics.socketio.gauge('realtime_connections').record(this.connectionCount); } - assertVersion(client: Socket, version?: string) { + async assertVersion(client: Socket, version?: string) { + const shouldCheckClientVersion = await this.config.runtime.fetch( + 'flags/syncClientVersionCheck' + ); if ( // @todo(@darkskygit): remove this flag after 0.12 goes stable - AFFiNE.featureFlags.syncClientVersionCheck && + shouldCheckClientVersion && version !== AFFiNE.version ) { client.emit('server-version-rejected', { @@ -180,7 +184,7 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect { @MessageBody('version') version: string | undefined, @ConnectedSocket() client: Socket ): Promise> { - this.assertVersion(client, version); + await this.assertVersion(client, version); await this.assertWorkspaceAccessible( workspaceId, user.id, @@ -203,7 +207,7 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect { @MessageBody('version') version: string | undefined, @ConnectedSocket() client: Socket ): Promise> { - this.assertVersion(client, version); + await this.assertVersion(client, version); await this.assertWorkspaceAccessible( workspaceId, user.id, diff --git a/packages/backend/server/src/core/utils/validators.ts b/packages/backend/server/src/core/utils/validators.ts index 3621bd9b7d1f3..6b5934b189224 100644 --- a/packages/backend/server/src/core/utils/validators.ts +++ b/packages/backend/server/src/core/utils/validators.ts @@ -1,26 +1,6 @@ import { BadRequestException } from '@nestjs/common'; import z from 'zod'; -function getAuthCredentialValidator() { - const email = z.string().email({ message: 'Invalid email address' }); - let password = z.string(); - - password = password - .min(AFFiNE.auth.password.minLength, { - message: `Password must be ${AFFiNE.auth.password.minLength} or more charactors long`, - }) - .max(AFFiNE.auth.password.maxLength, { - message: `Password must be ${AFFiNE.auth.password.maxLength} or fewer charactors long`, - }); - - return z - .object({ - email, - password, - }) - .required(); -} - function assertValid(z: z.ZodType, value: unknown) { const result = z.safeParse(value); @@ -35,22 +15,25 @@ function assertValid(z: z.ZodType, value: unknown) { } export function assertValidEmail(email: string) { - assertValid(getAuthCredentialValidator().shape.email, email); -} - -export function assertValidPassword(password: string) { - assertValid(getAuthCredentialValidator().shape.password, password); + assertValid(z.string().email({ message: 'Invalid email address' }), email); } -export function assertValidCredential(credential: { - email: string; - password: string; -}) { - assertValid(getAuthCredentialValidator(), credential); +export function assertValidPassword( + password: string, + { min, max }: { min: number; max: number } +) { + assertValid( + z + .string() + .min(min, { message: `Password must be ${min} or more charactors long` }) + .max(max, { + message: `Password must be ${max} or fewer charactors long`, + }), + password + ); } export const validators = { assertValidEmail, assertValidPassword, - assertValidCredential, }; diff --git a/packages/backend/server/src/fundamentals/config/def.ts b/packages/backend/server/src/fundamentals/config/def.ts index 3f7f306048571..a565c36dac5b0 100644 --- a/packages/backend/server/src/fundamentals/config/def.ts +++ b/packages/backend/server/src/fundamentals/config/def.ts @@ -1,16 +1,5 @@ -import type { ApolloDriverConfig } from '@nestjs/apollo'; -import SMTPTransport from 'nodemailer/lib/smtp-transport'; - import type { LeafPaths } from '../utils/types'; -import type { AFFiNEStorageConfig } from './storage'; - -declare global { - // eslint-disable-next-line @typescript-eslint/no-namespace - namespace globalThis { - // eslint-disable-next-line no-var - var AFFiNE: AFFiNEConfig; - } -} +import { AppStartupConfig } from './types'; export type EnvConfigType = 'string' | 'int' | 'float' | 'boolean'; export type ServerFlavor = 'allinone' | 'graphql' | 'sync'; @@ -22,333 +11,33 @@ export enum DeploymentType { Selfhosted = 'selfhosted', } -export type ConfigPaths = LeafPaths< - Omit< - AFFiNEConfig, - | 'ENV_MAP' - | 'version' - | 'type' - | 'isSelfhosted' - | 'flavor' - | 'env' - | 'affine' - | 'deploy' - | 'node' - | 'baseUrl' - | 'origin' - >, - '', - '......' ->; +export type ConfigPaths = LeafPaths; -/** - * All Configurations that would control AFFiNE server behaviors - * - */ -export interface AFFiNEConfig { +export interface PreDefinedAFFiNEConfig { ENV_MAP: Record; - /** - * Server Identity - */ serverId: string; - - /** - * Name may show on the UI - */ serverName: string; - - /** - * System version - */ - readonly version: string; - - /** - * Deployment type, AFFiNE Cloud, or Selfhosted - */ - get type(): DeploymentType; - - /** - * Fast detect whether currently deployed in a selfhosted environment - */ - get isSelfhosted(): boolean; - - /** - * Server flavor - */ - get flavor(): { - type: string; - graphql: boolean; - sync: boolean; - }; - - /** - * Application secrets for authentication and data encryption - */ - secrets: { - /** - * Application public key - * - */ - publicKey: string; - /** - * Application private key - * - */ - privateKey: string; - }; - - /** - * Deployment environment - */ readonly AFFINE_ENV: AFFINE_ENV; - /** - * alias to `process.env.NODE_ENV` - * - * @default 'development' - * @env NODE_ENV - */ readonly NODE_ENV: NODE_ENV; + readonly version: string; + readonly type: DeploymentType; + readonly isSelfhosted: boolean; + readonly flavor: { type: string; graphql: boolean; sync: boolean }; + readonly affine: { canary: boolean; beta: boolean; stable: boolean }; + readonly node: { prod: boolean; dev: boolean; test: boolean }; + readonly deploy: boolean; +} - /** - * fast AFFiNE environment judge - */ - get affine(): { - canary: boolean; - beta: boolean; - stable: boolean; - }; - /** - * fast environment judge - */ - get node(): { - prod: boolean; - dev: boolean; - test: boolean; - }; - - get deploy(): boolean; - - /** - * Whether the server is hosted on a ssl enabled domain - */ - https: boolean; - /** - * where the server get deployed. - * - * @default 'localhost' - * @env AFFINE_SERVER_HOST - */ - host: string; - /** - * which port the server will listen on - * - * @default 3010 - * @env AFFINE_SERVER_PORT - */ - port: number; - /** - * subpath where the server get deployed if there is. - * - * @default '' // empty string - * @env AFFINE_SERVER_SUB_PATH - */ - path: string; - - /** - * Readonly property `baseUrl` is the full url of the server consists of `https://HOST:PORT/PATH`. - * - * if `host` is not `localhost` then the port will be ignored - */ - get baseUrl(): string; - - /** - * Readonly property `origin` is domain origin in the form of `https://HOST:PORT` without subpath. - * - * if `host` is not `localhost` then the port will be ignored - */ - get origin(): string; - - /** - * the database config - */ - db: { - url: string; - }; - - /** - * the apollo driver config - */ - graphql: ApolloDriverConfig; - /** - * app features flag - */ - featureFlags: { - earlyAccessPreview: boolean; - syncClientVersionCheck: boolean; - }; - - /** - * Configuration for Object Storage, which defines how blobs and avatar assets are stored. - */ - storage: AFFiNEStorageConfig; - - /** - * Rate limiter config - */ - rateLimiter: { - /** - * How long each request will be throttled (seconds) - * @default 60 - * @env THROTTLE_TTL - */ - ttl: number; - /** - * How many requests can be made in the given time frame - * @default 120 - * @env THROTTLE_LIMIT - */ - limit: number; - }; - - /** - * authentication config - */ - auth: { - allowSignup: boolean; - - /** - * The minimum and maximum length of the password when registering new users - * - * @default [8,32] - */ - password: { - /** - * The minimum length of the password - * - * @default 8 - */ - minLength: number; - /** - * The maximum length of the password - * - * @default 32 - */ - maxLength: number; - }; - session: { - /** - * Application auth expiration time in seconds - * - * @default 15 days - */ - ttl: number; - - /** - * Application auth time to refresh in seconds - * - * @default 7 days - */ - ttr: number; - }; - - /** - * Application access token config - */ - accessToken: { - /** - * Application access token expiration time in seconds - * - * @default 7 days - */ - ttl: number; - /** - * Application refresh token expiration time in seconds - * - * @default 30 days - */ - refreshTokenTtl: number; - }; - captcha: { - /** - * whether to enable captcha - */ - enable: boolean; - turnstile: { - /** - * Cloudflare Turnstile CAPTCHA secret - * default value is demo api key, witch always return success - */ - secret: string; - }; - challenge: { - /** - * challenge bits length - * default value is 20, which can resolve in 0.5-3 second in M2 MacBook Air in single thread - * @default 20 - */ - bits: number; - }; - }; - }; - - /** - * Configurations for mail service used to post auth or bussiness mails. - * - * @see https://nodemailer.com/smtp/ - */ - mailer?: SMTPTransport.Options; - - doc: { - manager: { - /** - * Whether auto merge updates into doc snapshot. - */ - enableUpdateAutoMerging: boolean; - - /** - * How often the [DocManager] will start a new turn of merging pending updates into doc snapshot. - * - * This is not the latency a new joint client will take to see the latest doc, - * but the buffer time we introduced to reduce the load of our service. - * - * in {ms} - */ - updatePollInterval: number; - - /** - * The maximum number of updates that will be pulled from the server at once. - * Existing for avoiding the server to be overloaded when there are too many updates for one doc. - */ - maxUpdatesPullCount: number; - - /** - * Use `y-octo` to merge updates at the same time when merging using Yjs. - * - * This is an experimental feature, and aimed to check the correctness of JwstCodec. - */ - experimentalMergeWithYOcto: boolean; - }; - history: { - /** - * How long the buffer time of creating a new history snapshot when doc get updated. - * - * in {ms} - */ - interval: number; - }; - }; +export interface AppPluginsConfig {} - metrics: { - enabled: boolean; - customerIo: { - token: string; - }; - }; +export type AFFiNEConfig = PreDefinedAFFiNEConfig & + AppStartupConfig & + AppPluginsConfig; - telemetry: { - enabled: boolean; - token: string; - }; +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace globalThis { + // eslint-disable-next-line no-var + var AFFiNE: AFFiNEConfig; + } } - -export * from './storage'; diff --git a/packages/backend/server/src/fundamentals/config/default.ts b/packages/backend/server/src/fundamentals/config/default.ts index db38ef14e6069..ac12f7fff7f6c 100644 --- a/packages/backend/server/src/fundamentals/config/default.ts +++ b/packages/backend/server/src/fundamentals/config/default.ts @@ -1,55 +1,16 @@ -/// - -import { createPrivateKey, createPublicKey } from 'node:crypto'; - -import { merge } from 'lodash-es'; - import pkg from '../../../package.json' assert { type: 'json' }; -import type { AFFINE_ENV, NODE_ENV, ServerFlavor } from './def'; -import { AFFiNEConfig, DeploymentType } from './def'; +import { + AFFINE_ENV, + AFFiNEConfig, + DeploymentType, + NODE_ENV, + PreDefinedAFFiNEConfig, + ServerFlavor, +} from './def'; import { readEnv } from './env'; -import { getDefaultAFFiNEStorageConfig } from './storage'; - -// Don't use this in production -const examplePrivateKey = `-----BEGIN EC PRIVATE KEY----- -MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49 -AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI -3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg== ------END EC PRIVATE KEY-----`; - -const ONE_DAY_IN_SEC = 60 * 60 * 24; - -const keyPair = (function () { - const AFFINE_PRIVATE_KEY = - process.env.AFFINE_PRIVATE_KEY ?? examplePrivateKey; - const privateKey = createPrivateKey({ - key: Buffer.from(AFFINE_PRIVATE_KEY), - format: 'pem', - type: 'sec1', - }) - .export({ - format: 'pem', - type: 'pkcs8', - }) - .toString('utf8'); - const publicKey = createPublicKey({ - key: Buffer.from(AFFINE_PRIVATE_KEY), - format: 'pem', - type: 'spki', - }) - .export({ - format: 'pem', - type: 'spki', - }) - .toString('utf8'); +import { defaultStartupConfig } from './register'; - return { - publicKey, - privateKey, - }; -})(); - -export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { +function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig { const NODE_ENV = readEnv('NODE_ENV', 'development', [ 'development', 'test', @@ -83,127 +44,84 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { dev: NODE_ENV === 'development', test: NODE_ENV === 'test', }; - const defaultConfig = { - serverId: 'affine-nestjs-server', - serverName: isSelfhosted ? 'Self-Host Cloud' : 'AFFiNE Cloud', - version: pkg.version, - get type() { - return deploymentType; - }, - get isSelfhosted() { - return isSelfhosted; - }, - get flavor() { - return { - type: flavor, - graphql: flavor === 'graphql' || flavor === 'allinone', - sync: flavor === 'sync' || flavor === 'allinone', - }; - }, + + return { ENV_MAP: {}, - AFFINE_ENV, - get affine() { - return affine; - }, NODE_ENV, - get node() { - return node; - }, - get deploy() { - return !this.node.dev && !this.node.test; - }, - secrets: { - privateKey: keyPair.privateKey, - publicKey: keyPair.publicKey, - }, - featureFlags: { - earlyAccessPreview: false, - syncClientVersionCheck: false, - }, - https: false, - host: 'localhost', - port: 3010, - path: '', - db: { - url: '', - }, - get origin() { - return this.node.dev - ? 'http://localhost:8080' - : `${this.https ? 'https' : 'http'}://${this.host}${ - this.host === 'localhost' || this.host === '0.0.0.0' - ? `:${this.port}` - : '' - }`; - }, - get baseUrl() { - return `${this.origin}${this.path}`; - }, - graphql: { - buildSchemaOptions: { - numberScalarMode: 'integer', - }, - introspection: true, - playground: true, - }, - auth: { - allowSignup: true, - password: { - minLength: node.prod ? 8 : 1, - maxLength: 32, - }, - session: { - ttl: 15 * ONE_DAY_IN_SEC, - ttr: 7 * ONE_DAY_IN_SEC, - }, - accessToken: { - ttl: 7 * ONE_DAY_IN_SEC, - refreshTokenTtl: 30 * ONE_DAY_IN_SEC, - }, - captcha: { - enable: false, - turnstile: { - secret: '1x0000000000000000000000000000000AA', - }, - challenge: { - bits: 20, - }, - }, - }, - storage: getDefaultAFFiNEStorageConfig(), - rateLimiter: { - ttl: 60, - limit: 120, - }, - doc: { - manager: { - enableUpdateAutoMerging: flavor !== 'sync', - updatePollInterval: 3000, - maxUpdatesPullCount: 500, - experimentalMergeWithYOcto: false, - }, - history: { - interval: 1000 * 60 * 10 /* 10 mins */, - }, - }, - metrics: { - enabled: false, - customerIo: { - token: '', - }, - }, - telemetry: { - enabled: isSelfhosted, - token: '389c0615a69b57cca7d3fa0a4824c930', - }, - plugins: { - enabled: new Set(), - use(plugin, config) { - this[plugin] = merge(this[plugin], config || {}); - this.enabled.add(plugin); - }, - }, - } satisfies AFFiNEConfig; + AFFINE_ENV, + serverId: 'some-randome-uuid', + serverName: isSelfhosted ? 'Self-Host Cloud' : 'AFFiNE Cloud', + version: pkg.version, + type: deploymentType, + isSelfhosted, + flavor: { + type: flavor, + graphql: flavor === 'graphql' || flavor === 'allinone', + sync: flavor === 'sync' || flavor === 'allinone', + }, + affine, + node, + deploy: !node.dev && !node.test, + }; +} + +export function getAFFiNEConfigModifier(): AFFiNEConfig { + const predefined = getPredefinedAFFiNEConfig() as AFFiNEConfig; + + return chainableProxy(predefined); +} + +function merge(a: any, b: any) { + if (typeof b !== 'object' || b instanceof Map || b instanceof Set) { + return b; + } + + if (Array.isArray(b)) { + if (Array.isArray(a)) { + return a.concat(b); + } + return b; + } + + const result = { ...a }; + Object.keys(b).forEach(key => { + result[key] = merge(result[key], b[key]); + }); + + return result; +} + +export function mergeConfigOverride(override: any) { + return merge(defaultStartupConfig, override); +} - return defaultConfig; -}; +function chainableProxy(obj: any) { + const keys: Set = new Set(Object.keys(obj)); + return new Proxy(obj, { + get(target, prop) { + if (!(prop in target)) { + keys.add(prop as string); + target[prop] = chainableProxy({}); + } + return target[prop]; + }, + set(target, prop, value) { + keys.add(prop as string); + if ( + typeof value === 'object' && + !( + value instanceof Map || + value instanceof Set || + value instanceof Array + ) + ) { + value = chainableProxy(value); + } + target[prop] = value; + return true; + }, + ownKeys() { + return Array.from(keys); + }, + }); +} diff --git a/packages/backend/server/src/fundamentals/config/index.ts b/packages/backend/server/src/fundamentals/config/index.ts index 9936b06f66d93..826d20b5be6dc 100644 --- a/packages/backend/server/src/fundamentals/config/index.ts +++ b/packages/backend/server/src/fundamentals/config/index.ts @@ -1,4 +1,38 @@ +import { DynamicModule, FactoryProvider } from '@nestjs/common'; +import { merge } from 'lodash-es'; + +import { AFFiNEConfig } from './def'; +import { Config } from './provider'; +import { Runtime } from './runtime/service'; + export * from './def'; export * from './default'; export { applyEnvToConfig, parseEnvValue } from './env'; -export * from './module'; +export * from './provider'; +export { defineRuntimeConfig, defineStartupConfig } from './register'; +export type { AppConfig, ConfigItem, ModuleConfig } from './types'; + +function createConfigProvider( + override?: DeepPartial +): FactoryProvider { + return { + provide: Config, + useFactory: (runtime: Runtime) => { + return Object.freeze(merge({}, globalThis.AFFiNE, override, { runtime })); + }, + inject: [Runtime], + }; +} + +export class ConfigModule { + static forRoot = (override?: DeepPartial): DynamicModule => { + const provider = createConfigProvider(override); + + return { + global: true, + module: ConfigModule, + providers: [provider, Runtime], + exports: [provider], + }; + }; +} diff --git a/packages/backend/server/src/fundamentals/config/module.ts b/packages/backend/server/src/fundamentals/config/module.ts deleted file mode 100644 index 0a05597167341..0000000000000 --- a/packages/backend/server/src/fundamentals/config/module.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { DynamicModule, FactoryProvider } from '@nestjs/common'; -import { merge } from 'lodash-es'; - -import { ApplyType } from '../utils/types'; -import { AFFiNEConfig } from './def'; - -/** - * @example - * - * import { Config } from '@affine/server' - * - * class TestConfig { - * constructor(private readonly config: Config) {} - * test() { - * return this.config.env - * } - * } - */ -export class Config extends ApplyType() {} - -function createConfigProvider( - override?: DeepPartial -): FactoryProvider { - return { - provide: Config, - useFactory: () => { - const wrapper = new Config(); - const config = merge({}, globalThis.AFFiNE, override); - - const proxy: Config = new Proxy(wrapper, { - get: (_target, property: keyof Config) => { - const desc = Object.getOwnPropertyDescriptor( - globalThis.AFFiNE, - property - ); - if (desc?.get) { - return desc.get.call(proxy); - } - return config[property]; - }, - }); - return proxy; - }, - }; -} - -export class ConfigModule { - static forRoot = (override?: DeepPartial): DynamicModule => { - const provider = createConfigProvider(override); - - return { - global: true, - module: ConfigModule, - providers: [provider], - exports: [provider], - }; - }; -} diff --git a/packages/backend/server/src/fundamentals/config/provider.ts b/packages/backend/server/src/fundamentals/config/provider.ts new file mode 100644 index 0000000000000..abc0988d6ddcd --- /dev/null +++ b/packages/backend/server/src/fundamentals/config/provider.ts @@ -0,0 +1,19 @@ +import { ApplyType } from '../utils/types'; +import { AFFiNEConfig } from './def'; +import type { Runtime } from './runtime/service'; + +/** + * @example + * + * import { Config } from '@affine/server' + * + * class TestConfig { + * constructor(private readonly config: Config) {} + * test() { + * return this.config.env + * } + * } + */ +export class Config extends ApplyType() { + runtime!: Runtime; +} diff --git a/packages/backend/server/src/fundamentals/config/register.ts b/packages/backend/server/src/fundamentals/config/register.ts new file mode 100644 index 0000000000000..9bece010655c5 --- /dev/null +++ b/packages/backend/server/src/fundamentals/config/register.ts @@ -0,0 +1,66 @@ +import { Prisma, RuntimeConfigType } from '@prisma/client'; +import { get, merge, set } from 'lodash-es'; + +import { + AppModulesConfigDef, + AppStartupConfig, + ModuleRuntimeConfigDescriptions, + ModuleStartupConfigDescriptions, +} from './types'; + +export const defaultStartupConfig: AppStartupConfig = {} as any; +export const defaultRuntimeConfig: Record< + string, + Prisma.RuntimeConfigCreateInput +> = {} as any; + +export function runtimeConfigType(val: any): RuntimeConfigType { + if (Array.isArray(val)) { + return RuntimeConfigType.Array; + } + + switch (typeof val) { + case 'string': + return RuntimeConfigType.String; + case 'number': + return RuntimeConfigType.Number; + case 'boolean': + return RuntimeConfigType.Boolean; + default: + return RuntimeConfigType.Object; + } +} + +function registerRuntimeConfig( + module: T, + configs: ModuleRuntimeConfigDescriptions +) { + Object.entries(configs).forEach(([key, value]) => { + defaultRuntimeConfig[`${module}/${key}`] = { + id: `${module}/${key}`, + module, + key, + description: value.desc, + value: value.default, + type: runtimeConfigType(value.default), + }; + }); +} + +export function defineStartupConfig( + module: T, + configs: ModuleStartupConfigDescriptions +) { + set( + defaultStartupConfig, + module, + merge(get(defaultStartupConfig, module, {}), configs) + ); +} + +export function defineRuntimeConfig( + module: T, + configs: ModuleRuntimeConfigDescriptions +) { + registerRuntimeConfig(module, configs); +} diff --git a/packages/backend/server/src/fundamentals/config/runtime/event.ts b/packages/backend/server/src/fundamentals/config/runtime/event.ts new file mode 100644 index 0000000000000..b7964b3d216ce --- /dev/null +++ b/packages/backend/server/src/fundamentals/config/runtime/event.ts @@ -0,0 +1,22 @@ +import { OnEvent } from '../../event'; +import { Payload } from '../../event/def'; +import { FlattenedAppRuntimeConfig } from '../types'; + +declare module '../../event/def' { + interface EventDefinitions { + runtimeConfig: { + [K in keyof FlattenedAppRuntimeConfig]: { + changed: Payload; + }; + }; + } +} + +/** + * not implemented yet + */ +export const OnRuntimeConfigChange_DO_NOT_USE = ( + nameWithModule: keyof FlattenedAppRuntimeConfig +) => { + return OnEvent(`runtimeConfig.${nameWithModule}.changed`); +}; diff --git a/packages/backend/server/src/fundamentals/config/runtime/service.ts b/packages/backend/server/src/fundamentals/config/runtime/service.ts new file mode 100644 index 0000000000000..345872c5de44a --- /dev/null +++ b/packages/backend/server/src/fundamentals/config/runtime/service.ts @@ -0,0 +1,242 @@ +import { + BadRequestException, + forwardRef, + Inject, + Injectable, + Logger, + OnApplicationBootstrap, +} from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { difference, keyBy } from 'lodash-es'; + +import { Cache } from '../../cache'; +import { defer } from '../../utils/promise'; +import { defaultRuntimeConfig, runtimeConfigType } from '../register'; +import { AppRuntimeConfigModules, FlattenedAppRuntimeConfig } from '../types'; + +function validateConfigType( + key: K, + value: any +) { + const want = defaultRuntimeConfig[key].type; + const get = runtimeConfigType(value); + if (get !== want) { + throw new BadRequestException( + `Invalid runtime config type for '${key}', want '${want}', but get '${get}'` + ); + } +} + +/** + * runtime.fetch(k) // v1 + * runtime.fetchAll(k1, k2, k3) // [v1, v2, v3] + * runtime.set(k, v) + * runtime.update(k, (v) => { + * v.xxx = 'yyy'; + * return v + * }) + */ +@Injectable() +export class Runtime implements OnApplicationBootstrap { + private readonly logger = new Logger('App:RuntimeConfig'); + + constructor( + private readonly db: PrismaClient, + // circular deps: runtime => cache => redis(maybe) => config => runtime + @Inject(forwardRef(() => Cache)) private readonly cache: Cache + ) {} + + async onApplicationBootstrap() { + await this.upgradeDB(); + } + + async fetch( + k: K + ): Promise { + const cached = await this.loadCache(k); + + if (cached !== undefined) { + return cached; + } + + const dbValue = await this.loadDb(k); + + if (dbValue === undefined) { + throw new Error(`Runtime config ${k} not found`); + } + + await this.setCache(k, dbValue); + + return dbValue; + } + + async fetchAll< + Selector extends { [Key in keyof FlattenedAppRuntimeConfig]?: true }, + >( + selector: Selector + ): Promise<{ + // @ts-expect-error allow + [Key in keyof Selector]: FlattenedAppRuntimeConfig[Key]; + }> { + const keys = Object.keys(selector); + + if (keys.length === 0) { + return {} as any; + } + + const records = await this.db.runtimeConfig.findMany({ + select: { + id: true, + value: true, + }, + where: { + id: { + in: keys, + }, + deletedAt: null, + }, + }); + + const keyed = keyBy(records, 'id'); + return keys.reduce((ret, key) => { + ret[key] = keyed[key]?.value ?? defaultRuntimeConfig[key].value; + return ret; + }, {} as any); + } + + async list(module?: AppRuntimeConfigModules) { + return await this.db.runtimeConfig.findMany({ + where: module ? { module, deletedAt: null } : { deletedAt: null }, + }); + } + + async set< + K extends keyof FlattenedAppRuntimeConfig, + V = FlattenedAppRuntimeConfig[K], + >(key: K, value: V) { + validateConfigType(key, value); + const config = await this.db.runtimeConfig.update({ + where: { + id: key, + deletedAt: null, + }, + data: { + value: value as any, + }, + }); + + await this.setCache(key, config.value as FlattenedAppRuntimeConfig[K]); + return config; + } + + async update< + K extends keyof FlattenedAppRuntimeConfig, + V = FlattenedAppRuntimeConfig[K], + >(k: K, modifier: (v: V) => V | Promise) { + const data = await this.fetch(k); + + const updated = await modifier(data as V); + + await this.set(k, updated); + + return updated; + } + + async loadDb( + k: K + ): Promise { + const v = await this.db.runtimeConfig.findFirst({ + where: { + id: k, + deletedAt: null, + }, + }); + + if (v) { + return v.value as FlattenedAppRuntimeConfig[K]; + } else { + const record = await this.db.runtimeConfig.create({ + data: defaultRuntimeConfig[k], + }); + + return record.value as any; + } + } + + async loadCache( + k: K + ): Promise { + return this.cache.get(`SERVER_RUNTIME:${k}`); + } + + async setCache( + k: K, + v: FlattenedAppRuntimeConfig[K] + ): Promise { + return this.cache.set( + `SERVER_RUNTIME:${k}`, + v, + { ttl: 60 * 1000 } + ); + } + + /** + * Upgrade the DB with latest runtime configs + */ + private async upgradeDB() { + const existingConfig = await this.db.runtimeConfig.findMany({ + select: { + id: true, + }, + where: { + deletedAt: null, + }, + }); + + const defined = Object.keys(defaultRuntimeConfig); + const existing = existingConfig.map(c => c.id); + const newConfigs = difference(defined, existing); + const deleteConfigs = difference(existing, defined); + + if (!newConfigs.length && !deleteConfigs.length) { + return; + } + + this.logger.log(`Found runtime config changes, upgrading...`); + const acquired = await this.cache.setnx('runtime:upgrade', 1, { + ttl: 10 * 60 * 1000, + }); + await using _ = defer(async () => { + await this.cache.delete('runtime:upgrade'); + }); + + if (acquired) { + for (const key of newConfigs) { + await this.db.runtimeConfig.upsert({ + create: defaultRuntimeConfig[key], + // old deleted setting should be restored + update: { + ...defaultRuntimeConfig[key], + deletedAt: null, + }, + where: { + id: key, + }, + }); + } + + await this.db.runtimeConfig.updateMany({ + where: { + id: { + in: deleteConfigs, + }, + }, + data: { + deletedAt: new Date(), + }, + }); + } + + this.logger.log('Upgrade completed'); + } +} diff --git a/packages/backend/server/src/fundamentals/config/types.ts b/packages/backend/server/src/fundamentals/config/types.ts new file mode 100644 index 0000000000000..0ed543cc8e281 --- /dev/null +++ b/packages/backend/server/src/fundamentals/config/types.ts @@ -0,0 +1,127 @@ +import { Join, PathType } from '../utils/types'; + +export type ConfigItem = T & { __type: 'ConfigItem' }; + +type ConfigDef = Record | never; + +export interface ModuleConfig< + Startup extends ConfigDef = never, + Runtime extends ConfigDef = never, +> { + startup: Startup; + runtime: Runtime; +} + +export type RuntimeConfigDescription = { + desc: string; + default: T; +}; + +type ConfigItemLeaves = + T extends Record + ? { + [K in keyof T]: K extends string + ? T[K] extends { __type: 'ConfigItem' } + ? K + : T[K] extends PrimitiveType + ? K + : Join> + : never; + }[keyof T] + : never; + +type StartupConfigDescriptions = { + [K in keyof T]: T[K] extends Record + ? T[K] extends ConfigItem + ? V + : T[K] + : T[K]; +}; + +type ModuleConfigLeaves = + T extends Record + ? { + [K in keyof T]: K extends string + ? T[K] extends ModuleConfig + ? K + : Join> + : never; + }[keyof T] + : never; + +type FlattenModuleConfigs> = { + // @ts-expect-error allow + [K in ModuleConfigLeaves]: PathType; +}; + +type _AppStartupConfig> = { + [K in keyof T]: T[K] extends ModuleConfig + ? S + : _AppStartupConfig; +}; + +// for extending +export interface AppConfig {} +export type AppModulesConfigDef = FlattenModuleConfigs; +export type AppConfigModules = keyof AppModulesConfigDef; +export type AppStartupConfig = _AppStartupConfig; + +// app runtime config keyed by module names +export type AppRuntimeConfigByModules = { + [Module in keyof AppModulesConfigDef]: AppModulesConfigDef[Module] extends ModuleConfig< + any, + infer Runtime + > + ? Runtime extends never + ? never + : { + // @ts-expect-error allow + [K in ConfigItemLeaves]: PathType< + Runtime, + K + > extends infer Config + ? Config extends ConfigItem + ? V + : Config + : never; + } + : never; +}; + +// names of modules that have runtime config +export type AppRuntimeConfigModules = { + [Module in keyof AppRuntimeConfigByModules]: AppRuntimeConfigByModules[Module] extends never + ? never + : Module; +}[keyof AppRuntimeConfigByModules]; + +// runtime config keyed by module names flattened into config names +// { auth: { allowSignup: boolean } } => { 'auth/allowSignup': boolean } +export type FlattenedAppRuntimeConfig = UnionToIntersection< + { + [Module in keyof AppRuntimeConfigByModules]: AppModulesConfigDef[Module] extends never + ? never + : { + [K in keyof AppRuntimeConfigByModules[Module] as K extends string + ? `${Module}/${K}` + : never]: AppRuntimeConfigByModules[Module][K]; + }; + }[keyof AppRuntimeConfigByModules] +>; + +export type ModuleStartupConfigDescriptions> = + T extends ModuleConfig + ? S extends never + ? undefined + : StartupConfigDescriptions + : never; + +export type ModuleRuntimeConfigDescriptions< + Module extends keyof AppRuntimeConfigByModules, +> = AppModulesConfigDef[Module] extends never + ? never + : { + [K in keyof AppRuntimeConfigByModules[Module]]: RuntimeConfigDescription< + AppRuntimeConfigByModules[Module][K] + >; + }; diff --git a/packages/backend/server/src/fundamentals/event/types.ts b/packages/backend/server/src/fundamentals/event/types.ts index b16c846030a8e..94e66e3756205 100644 --- a/packages/backend/server/src/fundamentals/event/types.ts +++ b/packages/backend/server/src/fundamentals/event/types.ts @@ -1,35 +1,22 @@ +import type { Join, PathType } from '../utils/types'; + export type Payload = { __payload: true; data: T; }; -export type Join = A extends '' - ? B - : `${A}.${B}`; - -export type PathType = string extends Path - ? unknown - : Path extends keyof T - ? T[Path] - : Path extends `${infer K}.${infer R}` - ? K extends keyof T - ? PathType - : unknown - : unknown; - export type Leaves = - T extends Payload - ? P - : T extends Record - ? { - [K in keyof T]: K extends string ? Leaves> : never; - }[keyof T] - : never; - -export type Flatten = - Leaves extends infer R + T extends Record ? { - // @ts-expect-error yo, ts can't make it - [K in R]: PathType extends Payload ? U : never; - } + [K in keyof T]: K extends string + ? T[K] extends Payload + ? K + : Join> + : never; + }[keyof T] : never; + +export type Flatten> = { + // @ts-expect-error allow + [K in Leaves]: PathType extends Payload ? U : never; +}; diff --git a/packages/backend/server/src/fundamentals/graphql/config.ts b/packages/backend/server/src/fundamentals/graphql/config.ts new file mode 100644 index 0000000000000..7985e29eedd68 --- /dev/null +++ b/packages/backend/server/src/fundamentals/graphql/config.ts @@ -0,0 +1,17 @@ +import { ApolloDriverConfig } from '@nestjs/apollo'; + +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; + +declare module '../../fundamentals/config' { + interface AppConfig { + graphql: ModuleConfig; + } +} + +defineStartupConfig('graphql', { + buildSchemaOptions: { + numberScalarMode: 'integer', + }, + introspection: true, + playground: true, +}); diff --git a/packages/backend/server/src/fundamentals/graphql/index.ts b/packages/backend/server/src/fundamentals/graphql/index.ts index 04b5c6a1d2f11..a4f10609d6331 100644 --- a/packages/backend/server/src/fundamentals/graphql/index.ts +++ b/packages/backend/server/src/fundamentals/graphql/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -25,7 +27,7 @@ export type GraphqlContext = { useFactory: (config: Config) => { return { ...config.graphql, - path: `${config.path}/graphql`, + path: `${config.server.path}/graphql`, csrfPrevention: { requestHeaders: ['content-type'], }, diff --git a/packages/backend/server/src/fundamentals/helpers/__tests__/crypto.spec.ts b/packages/backend/server/src/fundamentals/helpers/__tests__/crypto.spec.ts index 5ab3448ad268e..55676d4aabca7 100644 --- a/packages/backend/server/src/fundamentals/helpers/__tests__/crypto.spec.ts +++ b/packages/backend/server/src/fundamentals/helpers/__tests__/crypto.spec.ts @@ -42,9 +42,11 @@ test.beforeEach(async t => { const module = await Test.createTestingModule({ imports: [ ConfigModule.forRoot({ - secrets: { - publicKey, - privateKey, + crypto: { + secret: { + publicKey, + privateKey, + }, }, }), ], diff --git a/packages/backend/server/src/fundamentals/helpers/__tests__/url.spec.ts b/packages/backend/server/src/fundamentals/helpers/__tests__/url.spec.ts index 7a8e8cb40c620..fc644c6700b37 100644 --- a/packages/backend/server/src/fundamentals/helpers/__tests__/url.spec.ts +++ b/packages/backend/server/src/fundamentals/helpers/__tests__/url.spec.ts @@ -13,9 +13,11 @@ test.beforeEach(async t => { const module = await Test.createTestingModule({ imports: [ ConfigModule.forRoot({ - host: 'app.affine.local', - port: 3010, - https: true, + server: { + host: 'app.affine.local', + port: 3010, + https: true, + }, }), ], providers: [URLHelper], diff --git a/packages/backend/server/src/fundamentals/helpers/config.ts b/packages/backend/server/src/fundamentals/helpers/config.ts new file mode 100644 index 0000000000000..481df28593872 --- /dev/null +++ b/packages/backend/server/src/fundamentals/helpers/config.ts @@ -0,0 +1,53 @@ +import { createPrivateKey, createPublicKey } from 'node:crypto'; + +import { defineStartupConfig, ModuleConfig } from '../config'; + +declare module '../config' { + interface AppConfig { + crypto: ModuleConfig<{ + secret: { + publicKey: string; + privateKey: string; + }; + }>; + } +} + +// Don't use this in production +const examplePrivateKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49 +AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI +3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg== +-----END EC PRIVATE KEY-----`; + +defineStartupConfig('crypto', { + secret: (function () { + const AFFINE_PRIVATE_KEY = + process.env.AFFINE_PRIVATE_KEY ?? examplePrivateKey; + const privateKey = createPrivateKey({ + key: Buffer.from(AFFINE_PRIVATE_KEY), + format: 'pem', + type: 'sec1', + }) + .export({ + format: 'pem', + type: 'pkcs8', + }) + .toString('utf8'); + const publicKey = createPublicKey({ + key: Buffer.from(AFFINE_PRIVATE_KEY), + format: 'pem', + type: 'spki', + }) + .export({ + format: 'pem', + type: 'spki', + }) + .toString('utf8'); + + return { + publicKey, + privateKey, + }; + })(), +}); diff --git a/packages/backend/server/src/fundamentals/helpers/crypto.ts b/packages/backend/server/src/fundamentals/helpers/crypto.ts index fdd868cf806f3..d9835794f5877 100644 --- a/packages/backend/server/src/fundamentals/helpers/crypto.ts +++ b/packages/backend/server/src/fundamentals/helpers/crypto.ts @@ -32,11 +32,11 @@ export class CryptoHelper { constructor(config: Config) { this.keyPair = { - publicKey: Buffer.from(config.secrets.publicKey, 'utf8'), - privateKey: Buffer.from(config.secrets.privateKey, 'utf8'), + publicKey: Buffer.from(config.crypto.secret.publicKey, 'utf8'), + privateKey: Buffer.from(config.crypto.secret.privateKey, 'utf8'), sha256: { - publicKey: this.sha256(config.secrets.publicKey), - privateKey: this.sha256(config.secrets.privateKey), + publicKey: this.sha256(config.crypto.secret.publicKey), + privateKey: this.sha256(config.crypto.secret.privateKey), }, }; } diff --git a/packages/backend/server/src/fundamentals/helpers/index.ts b/packages/backend/server/src/fundamentals/helpers/index.ts index 1d03b06af42e1..e98b755c7cbc1 100644 --- a/packages/backend/server/src/fundamentals/helpers/index.ts +++ b/packages/backend/server/src/fundamentals/helpers/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { Global, Module } from '@nestjs/common'; import { CryptoHelper } from './crypto'; diff --git a/packages/backend/server/src/fundamentals/helpers/url.ts b/packages/backend/server/src/fundamentals/helpers/url.ts index 38f8896507cdf..e460c9a0d34b1 100644 --- a/packages/backend/server/src/fundamentals/helpers/url.ts +++ b/packages/backend/server/src/fundamentals/helpers/url.ts @@ -5,35 +5,44 @@ import { Config } from '../config'; @Injectable() export class URLHelper { - redirectAllowHosts: string[]; + private readonly redirectAllowHosts: string[]; + readonly origin = this.config.node.dev + ? 'http://localhost:8080' + : `${this.config.server.https ? 'https' : 'http'}://${this.config.server.host}${ + this.config.server.host === 'localhost' || + this.config.server.host === '0.0.0.0' + ? `:${this.config.server.port}` + : '' + }`; - constructor(private readonly config: Config) { - this.redirectAllowHosts = [this.config.baseUrl]; - } + readonly baseUrl = `${this.origin}${this.config.server.path}`; + readonly home = this.baseUrl; - get home() { - return this.config.baseUrl; + constructor(private readonly config: Config) { + this.redirectAllowHosts = [this.baseUrl]; } stringify(query: Record) { return new URLSearchParams(query).toString(); } - link(path: string, query: Record = {}) { - const url = new URL( - this.config.baseUrl + (path.startsWith('/') ? path : '/' + path) - ); + url(path: string, query: Record = {}) { + const url = new URL(path, this.origin); for (const key in query) { url.searchParams.set(key, query[key]); } - return url.toString(); + return url; + } + + link(path: string, query: Record = {}) { + return this.url(path, query).toString(); } safeRedirect(res: Response, to: string) { try { - const finalTo = new URL(decodeURIComponent(to), this.config.baseUrl); + const finalTo = new URL(decodeURIComponent(to), this.baseUrl); for (const host of this.redirectAllowHosts) { const hostURL = new URL(host); diff --git a/packages/backend/server/src/fundamentals/index.ts b/packages/backend/server/src/fundamentals/index.ts index 5060d3543265d..6404ac647606b 100644 --- a/packages/backend/server/src/fundamentals/index.ts +++ b/packages/backend/server/src/fundamentals/index.ts @@ -6,11 +6,12 @@ export { SessionCache, } from './cache'; export { + type AFFiNEConfig, applyEnvToConfig, Config, type ConfigPaths, DeploymentType, - getDefaultAFFiNEStorageConfig, + getAFFiNEConfigModifier, } from './config'; export * from './error'; export { EventEmitter, type EventPayload, OnEvent } from './event'; diff --git a/packages/backend/server/src/fundamentals/mailer/config.ts b/packages/backend/server/src/fundamentals/mailer/config.ts new file mode 100644 index 0000000000000..a65891bbeda3b --- /dev/null +++ b/packages/backend/server/src/fundamentals/mailer/config.ts @@ -0,0 +1,16 @@ +import SMTPTransport from 'nodemailer/lib/smtp-transport'; + +import { defineStartupConfig, ModuleConfig } from '../config'; + +declare module '../config' { + interface AppConfig { + /** + * Configurations for mail service used to post auth or bussiness mails. + * + * @see https://nodemailer.com/smtp/ + */ + mailer: ModuleConfig; + } +} + +defineStartupConfig('mailer', {}); diff --git a/packages/backend/server/src/fundamentals/mailer/index.ts b/packages/backend/server/src/fundamentals/mailer/index.ts index d6412fa381fec..d6de678a55121 100644 --- a/packages/backend/server/src/fundamentals/mailer/index.ts +++ b/packages/backend/server/src/fundamentals/mailer/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { Global, Module } from '@nestjs/common'; import { OptionalModule } from '../nestjs'; @@ -8,7 +10,7 @@ import { MAILER } from './mailer'; @OptionalModule({ providers: [MAILER], exports: [MAILER], - requires: ['mailer.auth.user'], + requires: ['mailer.host'], }) class MailerModule {} diff --git a/packages/backend/server/src/fundamentals/metrics/config.ts b/packages/backend/server/src/fundamentals/metrics/config.ts new file mode 100644 index 0000000000000..7494089ac35e9 --- /dev/null +++ b/packages/backend/server/src/fundamentals/metrics/config.ts @@ -0,0 +1,33 @@ +import { defineStartupConfig, ModuleConfig } from '../config'; + +declare module '../config' { + interface AppConfig { + metrics: ModuleConfig<{ + /** + * Enable metric and tracing collection + */ + enabled: boolean; + /** + * Enable telemetry + */ + telemetry: { + enabled: boolean; + token: string; + }; + customerIo: { + token: string; + }; + }>; + } +} + +defineStartupConfig('metrics', { + enabled: false, + telemetry: { + enabled: false, + token: '', + }, + customerIo: { + token: '', + }, +}); diff --git a/packages/backend/server/src/fundamentals/metrics/index.ts b/packages/backend/server/src/fundamentals/metrics/index.ts index 9685b58d49a75..ee2d98f29210f 100644 --- a/packages/backend/server/src/fundamentals/metrics/index.ts +++ b/packages/backend/server/src/fundamentals/metrics/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { Global, Module, diff --git a/packages/backend/server/src/fundamentals/nestjs/config.ts b/packages/backend/server/src/fundamentals/nestjs/config.ts new file mode 100644 index 0000000000000..80e888eadcab3 --- /dev/null +++ b/packages/backend/server/src/fundamentals/nestjs/config.ts @@ -0,0 +1,42 @@ +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; + +export interface ServerStartupConfigurations { + /** + * Whether the server is hosted on a ssl enabled domain + */ + https: boolean; + /** + * where the server get deployed. + * + * @default 'localhost' + * @env AFFINE_SERVER_HOST + */ + host: string; + /** + * which port the server will listen on + * + * @default 3010 + * @env AFFINE_SERVER_PORT + */ + port: number; + /** + * subpath where the server get deployed if there is. + * + * @default '' // empty string + * @env AFFINE_SERVER_SUB_PATH + */ + path: string; +} + +declare module '../../fundamentals/config' { + interface AppConfig { + server: ModuleConfig; + } +} + +defineStartupConfig('server', { + https: false, + host: 'localhost', + port: 3010, + path: '', +}); diff --git a/packages/backend/server/src/fundamentals/nestjs/index.ts b/packages/backend/server/src/fundamentals/nestjs/index.ts index 7404efcb5cdd0..dc26fb62cf7d8 100644 --- a/packages/backend/server/src/fundamentals/nestjs/index.ts +++ b/packages/backend/server/src/fundamentals/nestjs/index.ts @@ -1,2 +1,3 @@ +import './config'; export * from './exception'; export * from './optional-module'; diff --git a/packages/backend/server/src/fundamentals/nestjs/optional-module.ts b/packages/backend/server/src/fundamentals/nestjs/optional-module.ts index b53d2408f6eba..d7be4f5f90b9f 100644 --- a/packages/backend/server/src/fundamentals/nestjs/optional-module.ts +++ b/packages/backend/server/src/fundamentals/nestjs/optional-module.ts @@ -7,7 +7,7 @@ import { } from '@nestjs/common'; import { omit } from 'lodash-es'; -import { Config, ConfigPaths } from '../config'; +import type { AFFiNEConfig, ConfigPaths } from '../config'; export interface OptionalModuleMetadata extends ModuleMetadata { /** @@ -18,7 +18,7 @@ export interface OptionalModuleMetadata extends ModuleMetadata { /** * Only install module if the predication returns true. */ - if?: (config: Config) => boolean; + if?: (config: AFFiNEConfig) => boolean; /** * Defines which feature will be enabled if the module installed. diff --git a/packages/backend/server/src/fundamentals/config/storage/index.ts b/packages/backend/server/src/fundamentals/storage/config.ts similarity index 81% rename from packages/backend/server/src/fundamentals/config/storage/index.ts rename to packages/backend/server/src/fundamentals/storage/config.ts index a56404277f04a..fde09ec80326c 100644 --- a/packages/backend/server/src/fundamentals/config/storage/index.ts +++ b/packages/backend/server/src/fundamentals/storage/config.ts @@ -1,14 +1,28 @@ import { homedir } from 'node:os'; import { join } from 'node:path'; +import { defineStartupConfig, ModuleConfig } from '../config'; + export interface FsStorageConfig { path: string; } export interface StorageProvidersConfig { - fs: FsStorageConfig; + fs?: FsStorageConfig; +} + +declare module '../config' { + interface AppConfig { + storageProviders: ModuleConfig; + } } +defineStartupConfig('storageProviders', { + fs: { + path: join(homedir(), '.affine/storage'), + }, +}); + export type StorageProviderType = keyof StorageProvidersConfig; export type StorageConfig = { diff --git a/packages/backend/server/src/fundamentals/storage/index.ts b/packages/backend/server/src/fundamentals/storage/index.ts index 52ce159f9aa11..7804f008694de 100644 --- a/packages/backend/server/src/fundamentals/storage/index.ts +++ b/packages/backend/server/src/fundamentals/storage/index.ts @@ -1,14 +1,16 @@ +import './config'; + import { Global, Module } from '@nestjs/common'; import { registerStorageProvider, StorageProviderFactory } from './providers'; import { FsStorageProvider } from './providers/fs'; registerStorageProvider('fs', (config, bucket) => { - if (!config.storage.providers.fs) { + if (!config.storageProviders.fs) { throw new Error('Missing fs storage provider configuration'); } - return new FsStorageProvider(config.storage.providers.fs, bucket); + return new FsStorageProvider(config.storageProviders.fs, bucket); }); @Global() @@ -19,6 +21,7 @@ registerStorageProvider('fs', (config, bucket) => { export class StorageProviderModule {} export * from '../../native'; +export type { StorageProviderType } from './config'; export type { BlobInputType, BlobOutputType, diff --git a/packages/backend/server/src/fundamentals/storage/providers/fs.ts b/packages/backend/server/src/fundamentals/storage/providers/fs.ts index 6358ee6b93ec5..9b2dafea68aa3 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/fs.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/fs.ts @@ -15,7 +15,7 @@ import { Readable } from 'node:stream'; import { Logger } from '@nestjs/common'; -import { FsStorageConfig } from '../../config/storage'; +import { FsStorageConfig } from '../config'; import { BlobInputType, GetObjectMetadata, diff --git a/packages/backend/server/src/fundamentals/storage/providers/index.ts b/packages/backend/server/src/fundamentals/storage/providers/index.ts index 59d07f34861a6..5ee118c7edc05 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/index.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/index.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { Config } from '../../config'; -import type { StorageProviderType, Storages } from '../../config/storage'; +import { StorageConfig, StorageProviderType } from '../config'; import type { StorageProvider } from './provider'; const availableProviders = new Map< @@ -20,17 +20,14 @@ export function registerStorageProvider( export class StorageProviderFactory { constructor(private readonly config: Config) {} - create(storage: Storages): StorageProvider { - const storageConfig = this.config.storage.storages[storage]; - const providerFactory = availableProviders.get(storageConfig.provider); + create(storage: StorageConfig): StorageProvider { + const providerFactory = availableProviders.get(storage.provider); if (!providerFactory) { - throw new Error( - `Unknown storage provider type: ${storageConfig.provider}` - ); + throw new Error(`Unknown storage provider type: ${storage.provider}`); } - return providerFactory(this.config, storageConfig.bucket); + return providerFactory(this.config, storage.bucket); } } diff --git a/packages/backend/server/src/fundamentals/storage/providers/provider.ts b/packages/backend/server/src/fundamentals/storage/providers/provider.ts index 0c9114b65767f..46f8ef688db35 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/provider.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/provider.ts @@ -1,6 +1,6 @@ import type { Readable } from 'node:stream'; -import { StorageProviderType } from '../../config'; +import { StorageProviderType } from '../config'; export interface GetObjectMetadata { /** diff --git a/packages/backend/server/src/fundamentals/throttler/config.ts b/packages/backend/server/src/fundamentals/throttler/config.ts new file mode 100644 index 0000000000000..61c32ef4d7b87 --- /dev/null +++ b/packages/backend/server/src/fundamentals/throttler/config.ts @@ -0,0 +1,27 @@ +import { defineStartupConfig, ModuleConfig } from '../config'; + +export type ThrottlerType = 'default' | 'strict'; + +type ThrottlerStartupConfigurations = { + [key in ThrottlerType]: { + ttl: number; + limit: number; + }; +}; + +declare module '../config' { + interface AppConfig { + throttler: ModuleConfig; + } +} + +defineStartupConfig('throttler', { + default: { + ttl: 60, + limit: 120, + }, + strict: { + ttl: 60, + limit: 20, + }, +}); diff --git a/packages/backend/server/src/fundamentals/throttler/decorators.ts b/packages/backend/server/src/fundamentals/throttler/decorators.ts index 742a32d729b7c..fe75db861455d 100644 --- a/packages/backend/server/src/fundamentals/throttler/decorators.ts +++ b/packages/backend/server/src/fundamentals/throttler/decorators.ts @@ -1,6 +1,8 @@ import { applyDecorators, SetMetadata } from '@nestjs/common'; import { SkipThrottle, Throttle as RawThrottle } from '@nestjs/throttler'; +import { ThrottlerType } from './config'; + export type Throttlers = 'default' | 'strict' | 'authenticated'; export const THROTTLER_PROTECTED = 'affine_throttler:protected'; @@ -25,7 +27,7 @@ export const THROTTLER_PROTECTED = 'affine_throttler:protected'; * */ export function Throttle( - type: Throttlers = 'default', + type: ThrottlerType | 'authenticated' = 'default', override: { limit?: number; ttl?: number } = {} ): MethodDecorator & ClassDecorator { return applyDecorators( diff --git a/packages/backend/server/src/fundamentals/throttler/index.ts b/packages/backend/server/src/fundamentals/throttler/index.ts index f15f408c12c82..fd68967a3f1b6 100644 --- a/packages/backend/server/src/fundamentals/throttler/index.ts +++ b/packages/backend/server/src/fundamentals/throttler/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { ExecutionContext, Global, Injectable, Module } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; import { @@ -14,6 +16,7 @@ import type { Request } from 'express'; import { Config } from '../config'; import { getRequestResponseFromContext } from '../utils/request'; +import type { ThrottlerType } from './config'; import { THROTTLER_PROTECTED, Throttlers } from './decorators'; @Injectable() @@ -21,25 +24,14 @@ export class ThrottlerStorage extends ThrottlerStorageService {} @Injectable() class CustomOptionsFactory implements ThrottlerOptionsFactory { - constructor( - private readonly config: Config, - private readonly storage: ThrottlerStorage - ) {} + constructor(private readonly storage: ThrottlerStorage) {} createThrottlerOptions() { const options: ThrottlerModuleOptions = { - throttlers: [ - { - name: 'default', - ttl: this.config.rateLimiter.ttl * 1000, - limit: this.config.rateLimiter.limit, - }, - { - name: 'strict', - ttl: this.config.rateLimiter.ttl * 1000, - limit: 20, - }, - ], + throttlers: Object.entries(AFFiNE.throttler).map(([name, config]) => ({ + name, + ...config, + })), storage: this.storage, }; @@ -165,7 +157,7 @@ export class CloudThrottlerGuard extends ThrottlerGuard { return super.canActivate(context); } - getSpecifiedThrottler(context: ExecutionContext) { + getSpecifiedThrottler(context: ExecutionContext): ThrottlerType | undefined { const throttler = this.reflector.getAllAndOverride( THROTTLER_PROTECTED, [context.getHandler(), context.getClass()] diff --git a/packages/backend/server/src/fundamentals/utils/promise.ts b/packages/backend/server/src/fundamentals/utils/promise.ts index 81881a12057ae..f59f1e6488c26 100644 --- a/packages/backend/server/src/fundamentals/utils/promise.ts +++ b/packages/backend/server/src/fundamentals/utils/promise.ts @@ -1,4 +1,4 @@ -import { defer, retry } from 'rxjs'; +import { defer as rxjsDefer, retry } from 'rxjs'; export class RetryablePromise extends Promise { constructor( @@ -10,7 +10,7 @@ export class RetryablePromise extends Promise { retryIntervalInMs: number = 300 ) { super((resolve, reject) => { - defer(() => new Promise(executor)) + rxjsDefer(() => new Promise(executor)) .pipe( retry({ count: retryTimes, @@ -42,3 +42,9 @@ export function retryable( retryIntervalInMs ); } + +export function defer(dispose: () => Promise) { + return { + [Symbol.asyncDispose]: dispose, + }; +} diff --git a/packages/backend/server/src/fundamentals/utils/types.ts b/packages/backend/server/src/fundamentals/utils/types.ts index d2435e513ba6b..57b95fa631da0 100644 --- a/packages/backend/server/src/fundamentals/utils/types.ts +++ b/packages/backend/server/src/fundamentals/utils/types.ts @@ -7,7 +7,20 @@ export function ApplyType(): ConstructorOf { }; } -type Join = Prefix extends string | number +export type PathType = + T extends Record + ? string extends Path + ? unknown + : Path extends keyof T + ? T[Path] + : Path extends `${infer K}.${infer R}` + ? K extends keyof T + ? PathType + : unknown + : unknown + : unknown; + +export type Join = Prefix extends string | number ? Suffixes extends string | number ? Prefix extends '' ? Suffixes @@ -18,7 +31,7 @@ type Join = Prefix extends string | number export type LeafPaths< T, Path extends string = '', - MaxDepth extends string = '...', + MaxDepth extends string = '.....', Depth extends string = '', > = Depth extends MaxDepth ? never diff --git a/packages/backend/server/src/global.d.ts b/packages/backend/server/src/global.d.ts index ce59a7a2d4507..fecd5d8551ce0 100644 --- a/packages/backend/server/src/global.d.ts +++ b/packages/backend/server/src/global.d.ts @@ -13,6 +13,12 @@ declare type PrimitiveType = | null | undefined; +declare type UnionToIntersection = ( + T extends any ? (x: T) => any : never +) extends (x: infer R) => any + ? R + : never; + declare type ConstructorOf = { new (): T; }; @@ -22,7 +28,7 @@ declare type DeepPartial = ? DeepPartial[] : T extends ReadonlyArray ? ReadonlyArray> - : T extends object + : T extends Record ? { [K in keyof T]?: DeepPartial; } diff --git a/packages/backend/server/src/index.ts b/packages/backend/server/src/index.ts index 1b13da97a0e53..d0d65a7db2de3 100644 --- a/packages/backend/server/src/index.ts +++ b/packages/backend/server/src/index.ts @@ -2,15 +2,22 @@ import './prelude'; import { Logger } from '@nestjs/common'; +import { omit } from 'lodash-es'; import { createApp } from './app'; +import { URLHelper } from './fundamentals'; const app = await createApp(); const listeningHost = AFFiNE.deploy ? '0.0.0.0' : 'localhost'; -await app.listen(AFFiNE.port, listeningHost); +await app.listen(AFFiNE.server.port, listeningHost); +const url = app.get(URLHelper); const logger = new Logger('App'); logger.log(`AFFiNE Server is running in [${AFFiNE.type}] mode`); -logger.log(`Listening on http://${listeningHost}:${AFFiNE.port}`); -logger.log(`And the public server should be recognized as ${AFFiNE.baseUrl}`); +if (AFFiNE.node.dev) { + logger.log('Startup Configration:'); + logger.log(omit(globalThis.AFFiNE, 'ENV_MAP')); +} +logger.log(`Listening on http://${listeningHost}:${AFFiNE.server.port}`); +logger.log(`And the public server should be recognized as ${url.home}`); diff --git a/packages/backend/server/src/plugins/config.ts b/packages/backend/server/src/plugins/config.ts index ba512a65d96d3..0c67f0753f7ad 100644 --- a/packages/backend/server/src/plugins/config.ts +++ b/packages/backend/server/src/plugins/config.ts @@ -1,30 +1,20 @@ -import { CopilotConfig } from './copilot'; -import { GCloudConfig } from './gcloud/config'; -import { OAuthConfig } from './oauth'; -import { PaymentConfig } from './payment'; -import { RedisOptions } from './redis'; -import { R2StorageConfig, S3StorageConfig } from './storage'; +import { ModuleStartupConfigDescriptions } from '../fundamentals/config/types'; +export interface PluginsConfig {} +export type AvailablePlugins = keyof PluginsConfig; + +declare module '../fundamentals/config' {} declare module '../fundamentals/config' { - interface PluginsConfig { - readonly copilot: CopilotConfig; - readonly payment: PaymentConfig; - readonly redis: RedisOptions; - readonly gcloud: GCloudConfig; - readonly 'cloudflare-r2': R2StorageConfig; - readonly 'aws-s3': S3StorageConfig; - readonly oauth: OAuthConfig; + interface AppConfig { + plugins: PluginsConfig; } - export type AvailablePlugins = keyof PluginsConfig; - - interface AFFiNEConfig { - readonly plugins: { - enabled: Set; - use( - plugin: Plugin, - config?: DeepPartial - ): void; - } & Partial; + interface AppPluginsConfig { + use( + plugin: Plugin, + config?: DeepPartial< + ModuleStartupConfigDescriptions + > + ): void; } } diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts new file mode 100644 index 0000000000000..720d7dd0330c0 --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/config.ts @@ -0,0 +1,26 @@ +import type { ClientOptions as OpenAIClientOptions } from 'openai'; + +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; +import { StorageConfig } from '../../fundamentals/storage/config'; +import type { FalConfig } from './providers/fal'; + +export interface CopilotStartupConfigurations { + openai?: OpenAIClientOptions; + fal?: FalConfig; + test?: never; + unsplashKey?: string; + storage: StorageConfig; +} + +declare module '../config' { + interface PluginsConfig { + copilot: ModuleConfig; + } +} + +defineStartupConfig('plugins.copilot', { + storage: { + provider: 'fs', + bucket: 'copilot', + }, +}); diff --git a/packages/backend/server/src/plugins/copilot/index.ts b/packages/backend/server/src/plugins/copilot/index.ts index f3ecca2f94224..f58ba8754cf3b 100644 --- a/packages/backend/server/src/plugins/copilot/index.ts +++ b/packages/backend/server/src/plugins/copilot/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { ServerFeature } from '../../core/config'; import { FeatureModule } from '../../core/features'; import { QuotaModule } from '../../core/quota'; @@ -43,5 +45,3 @@ registerCopilotProvider(OpenAIProvider); }, }) export class CopilotModule {} - -export type { CopilotConfig } from './types'; diff --git a/packages/backend/server/src/plugins/copilot/providers/index.ts b/packages/backend/server/src/plugins/copilot/providers/index.ts index d48a911ad2ae0..a2cce5fae92fa 100644 --- a/packages/backend/server/src/plugins/copilot/providers/index.ts +++ b/packages/backend/server/src/plugins/copilot/providers/index.ts @@ -2,16 +2,17 @@ import assert from 'node:assert'; import { Injectable, Logger } from '@nestjs/common'; -import { Config } from '../../../fundamentals'; +import { AFFiNEConfig, Config } from '../../../fundamentals'; +import { CopilotStartupConfigurations } from '../config'; import { CapabilityToCopilotProvider, CopilotCapability, - CopilotConfig, CopilotProvider, CopilotProviderType, } from '../types'; -type CopilotProviderConfig = CopilotConfig[keyof CopilotConfig]; +type CopilotProviderConfig = + CopilotStartupConfigurations[keyof CopilotStartupConfigurations]; interface CopilotProviderDefinition { // constructor signature @@ -37,7 +38,10 @@ const PROVIDER_CAPABILITY_MAP = new Map< >(); // config assertions for providers -const ASSERT_CONFIG = new Map void>(); +const ASSERT_CONFIG = new Map< + CopilotProviderType, + (config: AFFiNEConfig) => void +>(); export function registerCopilotProvider< C extends CopilotProviderConfig = CopilotProviderConfig, @@ -69,7 +73,7 @@ export function registerCopilotProvider< PROVIDER_CAPABILITY_MAP.set(capability, providers); } // register the provider config assertion - ASSERT_CONFIG.set(type, (config: Config) => { + ASSERT_CONFIG.set(type, (config: AFFiNEConfig) => { assert(config.plugins.copilot); const providerConfig = config.plugins.copilot[type]; if (!providerConfig) return false; @@ -89,7 +93,7 @@ export function unregisterCopilotProvider(type: CopilotProviderType) { } /// Asserts that the config is valid for any registered providers -export function assertProvidersConfigs(config: Config) { +export function assertProvidersConfigs(config: AFFiNEConfig) { return ( Array.from(ASSERT_CONFIG.values()).findIndex(assertConfig => assertConfig(config) diff --git a/packages/backend/server/src/plugins/copilot/storage.ts b/packages/backend/server/src/plugins/copilot/storage.ts index ecb47dd2bc467..44be26cd4cc86 100644 --- a/packages/backend/server/src/plugins/copilot/storage.ts +++ b/packages/backend/server/src/plugins/copilot/storage.ts @@ -9,6 +9,7 @@ import { type FileUpload, type StorageProvider, StorageProviderFactory, + URLHelper, } from '../../fundamentals'; @Injectable() @@ -17,10 +18,13 @@ export class CopilotStorage { constructor( private readonly config: Config, + private readonly url: URLHelper, private readonly storageFactory: StorageProviderFactory, private readonly quota: QuotaManagementService ) { - this.provider = this.storageFactory.create('copilot'); + this.provider = this.storageFactory.create( + this.config.plugins.copilot.storage + ); } async put( @@ -35,7 +39,7 @@ export class CopilotStorage { // return image base64url for dev environment return `data:image/png;base64,${blob.toString('base64')}`; } - return `${this.config.baseUrl}/api/copilot/blob/${name}`; + return this.url.link(`/api/copilot/blob/${name}`); } async get(userId: string, workspaceId: string, key: string) { diff --git a/packages/backend/server/src/plugins/copilot/types.ts b/packages/backend/server/src/plugins/copilot/types.ts index 9002d457a4add..3b652498deda7 100644 --- a/packages/backend/server/src/plugins/copilot/types.ts +++ b/packages/backend/server/src/plugins/copilot/types.ts @@ -1,18 +1,9 @@ import { type Tokenizer } from '@affine/server-native'; import { AiPromptRole } from '@prisma/client'; -import type { ClientOptions as OpenAIClientOptions } from 'openai'; import { z } from 'zod'; import { fromModelName } from '../../native'; import type { ChatPrompt } from './prompt'; -import type { FalConfig } from './providers/fal'; - -export interface CopilotConfig { - openai: OpenAIClientOptions; - fal: FalConfig; - unsplashKey: string; - test: never; -} export enum AvailableModels { // text to text diff --git a/packages/backend/server/src/plugins/gcloud/config.ts b/packages/backend/server/src/plugins/gcloud/config.ts index 9bca1ceeb1a9e..b536236393882 100644 --- a/packages/backend/server/src/plugins/gcloud/config.ts +++ b/packages/backend/server/src/plugins/gcloud/config.ts @@ -1 +1,14 @@ -export interface GCloudConfig {} +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; + +export interface GCloudConfig { + enabled: boolean; +} +declare module '../config' { + interface PluginsConfig { + gcloud: ModuleConfig; + } +} + +defineStartupConfig('plugins.gcloud', { + enabled: false, +}); diff --git a/packages/backend/server/src/plugins/gcloud/index.ts b/packages/backend/server/src/plugins/gcloud/index.ts index 16a5a4494ebdf..d82ac08ff5b9f 100644 --- a/packages/backend/server/src/plugins/gcloud/index.ts +++ b/packages/backend/server/src/plugins/gcloud/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { Global } from '@nestjs/common'; import { Plugin } from '../registry'; diff --git a/packages/backend/server/src/plugins/index.ts b/packages/backend/server/src/plugins/index.ts index 9d82b90c10128..1eeb08f2b1fc3 100644 --- a/packages/backend/server/src/plugins/index.ts +++ b/packages/backend/server/src/plugins/index.ts @@ -5,4 +5,8 @@ import './payment'; import './redis'; import './storage'; -export { REGISTERED_PLUGINS } from './registry'; +export { + enablePlugin, + REGISTERED_PLUGINS, + ENABLED_PLUGINS as USED_PLUGINS, +} from './registry'; diff --git a/packages/backend/server/src/plugins/oauth/types.ts b/packages/backend/server/src/plugins/oauth/config.ts similarity index 74% rename from packages/backend/server/src/plugins/oauth/types.ts rename to packages/backend/server/src/plugins/oauth/config.ts index ea7535f5e1bf0..ee060dd4789d1 100644 --- a/packages/backend/server/src/plugins/oauth/types.ts +++ b/packages/backend/server/src/plugins/oauth/config.ts @@ -1,3 +1,5 @@ +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; + export interface OAuthProviderConfig { clientId: string; clientSecret: string; @@ -29,6 +31,15 @@ type OAuthProviderConfigMapping = { }; export interface OAuthConfig { - enabled: boolean; providers: Partial; } + +declare module '../config' { + interface PluginsConfig { + oauth: ModuleConfig; + } +} + +defineStartupConfig('plugins.oauth', { + providers: {}, +}); diff --git a/packages/backend/server/src/plugins/oauth/controller.ts b/packages/backend/server/src/plugins/oauth/controller.ts index 3d4ac90f65c25..7007f9c770afd 100644 --- a/packages/backend/server/src/plugins/oauth/controller.ts +++ b/packages/backend/server/src/plugins/oauth/controller.ts @@ -12,10 +12,10 @@ import type { Request, Response } from 'express'; import { AuthService, Public } from '../../core/auth'; import { UserService } from '../../core/user'; import { URLHelper } from '../../fundamentals'; +import { OAuthProviderName } from './config'; import { OAuthAccount, Tokens } from './providers/def'; import { OAuthProviderFactory } from './register'; import { OAuthService } from './service'; -import { OAuthProviderName } from './types'; @Controller('/oauth') export class OAuthController { diff --git a/packages/backend/server/src/plugins/oauth/index.ts b/packages/backend/server/src/plugins/oauth/index.ts index 0b14d1d984719..a426d20cbdd2f 100644 --- a/packages/backend/server/src/plugins/oauth/index.ts +++ b/packages/backend/server/src/plugins/oauth/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { AuthModule } from '../../core/auth'; import { ServerFeature } from '../../core/config'; import { UserModule } from '../../core/user'; @@ -22,4 +24,3 @@ import { OAuthService } from './service'; if: config => !!config.plugins.oauth, }) export class OAuthModule {} -export type { OAuthConfig } from './types'; diff --git a/packages/backend/server/src/plugins/oauth/providers/def.ts b/packages/backend/server/src/plugins/oauth/providers/def.ts index 7e7913cdaf7bb..6102bf2e2844c 100644 --- a/packages/backend/server/src/plugins/oauth/providers/def.ts +++ b/packages/backend/server/src/plugins/oauth/providers/def.ts @@ -1,4 +1,4 @@ -import { OAuthProviderName } from '../types'; +import { OAuthProviderName } from '../config'; export interface OAuthAccount { id: string; diff --git a/packages/backend/server/src/plugins/oauth/providers/github.ts b/packages/backend/server/src/plugins/oauth/providers/github.ts index 50227539a7e8e..9b03c094b1bce 100644 --- a/packages/backend/server/src/plugins/oauth/providers/github.ts +++ b/packages/backend/server/src/plugins/oauth/providers/github.ts @@ -1,8 +1,8 @@ import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { Config, URLHelper } from '../../../fundamentals'; +import { OAuthProviderName } from '../config'; import { AutoRegisteredOAuthProvider } from '../register'; -import { OAuthProviderName } from '../types'; interface AuthTokenResponse { access_token: string; diff --git a/packages/backend/server/src/plugins/oauth/providers/google.ts b/packages/backend/server/src/plugins/oauth/providers/google.ts index 7c2f3e3600b9f..8db41bf97b9e9 100644 --- a/packages/backend/server/src/plugins/oauth/providers/google.ts +++ b/packages/backend/server/src/plugins/oauth/providers/google.ts @@ -1,8 +1,8 @@ import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { Config, URLHelper } from '../../../fundamentals'; +import { OAuthProviderName } from '../config'; import { AutoRegisteredOAuthProvider } from '../register'; -import { OAuthProviderName } from '../types'; interface GoogleOAuthTokenResponse { access_token: string; diff --git a/packages/backend/server/src/plugins/oauth/providers/oidc.ts b/packages/backend/server/src/plugins/oauth/providers/oidc.ts index 4f49fbf20d0b4..0854b874268e6 100644 --- a/packages/backend/server/src/plugins/oauth/providers/oidc.ts +++ b/packages/backend/server/src/plugins/oauth/providers/oidc.ts @@ -7,8 +7,12 @@ import { import { z } from 'zod'; import { Config, URLHelper } from '../../../fundamentals'; +import { + OAuthOIDCProviderConfig, + OAuthProviderName, + OIDCArgs, +} from '../config'; import { AutoRegisteredOAuthProvider } from '../register'; -import { OAuthOIDCProviderConfig, OAuthProviderName, OIDCArgs } from '../types'; import { OAuthAccount, Tokens } from './def'; const OIDCTokenSchema = z.object({ diff --git a/packages/backend/server/src/plugins/oauth/register.ts b/packages/backend/server/src/plugins/oauth/register.ts index 3eeccae7c6b4b..a4f225b5a0737 100644 --- a/packages/backend/server/src/plugins/oauth/register.ts +++ b/packages/backend/server/src/plugins/oauth/register.ts @@ -1,8 +1,8 @@ import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { Config } from '../../fundamentals'; +import { OAuthProviderName } from './config'; import { OAuthProvider } from './providers/def'; -import { OAuthProviderName } from './types'; const PROVIDERS: Map = new Map(); diff --git a/packages/backend/server/src/plugins/oauth/resolver.ts b/packages/backend/server/src/plugins/oauth/resolver.ts index 467cc90360170..8d0da0c682f1d 100644 --- a/packages/backend/server/src/plugins/oauth/resolver.ts +++ b/packages/backend/server/src/plugins/oauth/resolver.ts @@ -1,8 +1,8 @@ import { registerEnumType, ResolveField, Resolver } from '@nestjs/graphql'; import { ServerConfigType } from '../../core/config'; +import { OAuthProviderName } from './config'; import { OAuthProviderFactory } from './register'; -import { OAuthProviderName } from './types'; registerEnumType(OAuthProviderName, { name: 'OAuthProviderType' }); diff --git a/packages/backend/server/src/plugins/oauth/service.ts b/packages/backend/server/src/plugins/oauth/service.ts index d05dc623df460..faf95a683b946 100644 --- a/packages/backend/server/src/plugins/oauth/service.ts +++ b/packages/backend/server/src/plugins/oauth/service.ts @@ -3,8 +3,8 @@ import { randomUUID } from 'node:crypto'; import { Injectable } from '@nestjs/common'; import { SessionCache } from '../../fundamentals'; +import { OAuthProviderName } from './config'; import { OAuthProviderFactory } from './register'; -import { OAuthProviderName } from './types'; const OAUTH_STATE_KEY = 'OAUTH_STATE'; diff --git a/packages/backend/server/src/plugins/payment/config.ts b/packages/backend/server/src/plugins/payment/config.ts new file mode 100644 index 0000000000000..3f444b18689ef --- /dev/null +++ b/packages/backend/server/src/plugins/payment/config.ts @@ -0,0 +1,20 @@ +import type { Stripe } from 'stripe'; + +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; + +export interface PaymentStartupConfig { + stripe?: { + keys: { + APIKey: string; + webhookKey: string; + }; + } & Stripe.StripeConfig; +} + +declare module '../config' { + interface PluginsConfig { + payment: ModuleConfig; + } +} + +defineStartupConfig('plugins.payment', {}); diff --git a/packages/backend/server/src/plugins/payment/index.ts b/packages/backend/server/src/plugins/payment/index.ts index 975582a8792ec..1cc0f5477ae84 100644 --- a/packages/backend/server/src/plugins/payment/index.ts +++ b/packages/backend/server/src/plugins/payment/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { ServerFeature } from '../../core/config'; import { FeatureModule } from '../../core/features'; import { Plugin } from '../registry'; @@ -26,5 +28,3 @@ import { StripeWebhook } from './webhook'; if: config => config.flavor.graphql, }) export class PaymentModule {} - -export type { PaymentConfig } from './types'; diff --git a/packages/backend/server/src/plugins/payment/resolver.ts b/packages/backend/server/src/plugins/payment/resolver.ts index 968b66438e5d7..f342074d2a9f5 100644 --- a/packages/backend/server/src/plugins/payment/resolver.ts +++ b/packages/backend/server/src/plugins/payment/resolver.ts @@ -19,7 +19,7 @@ import { groupBy } from 'lodash-es'; import { CurrentUser, Public } from '../../core/auth'; import { UserType } from '../../core/user'; -import { Config } from '../../fundamentals'; +import { Config, URLHelper } from '../../fundamentals'; import { decodeLookupKey, SubscriptionService } from './service'; import { InvoiceStatus, @@ -146,8 +146,8 @@ class CreateCheckoutSessionInput { @Field(() => String, { nullable: true }) coupon!: string | null; - @Field(() => String, { nullable: true }) - successCallbackLink!: string | null; + @Field(() => String) + successCallbackLink!: string; // @FIXME(forehalo): we should put this field in the header instead of as a explicity args @Field(() => String) @@ -158,7 +158,7 @@ class CreateCheckoutSessionInput { export class SubscriptionResolver { constructor( private readonly service: SubscriptionService, - private readonly config: Config + private readonly url: URLHelper ) {} @Public() @@ -222,8 +222,7 @@ export class SubscriptionResolver { plan: input.plan, recurring: input.recurring, promotionCode: input.coupon, - redirectUrl: - input.successCallbackLink ?? `${this.config.baseUrl}/upgrade-success`, + redirectUrl: this.url.link(input.successCallbackLink), idempotencyKey: input.idempotencyKey, }); diff --git a/packages/backend/server/src/plugins/payment/stripe.ts b/packages/backend/server/src/plugins/payment/stripe.ts index 6d2aed7cea36e..1b984244bbf1e 100644 --- a/packages/backend/server/src/plugins/payment/stripe.ts +++ b/packages/backend/server/src/plugins/payment/stripe.ts @@ -9,8 +9,8 @@ import { Config } from '../../fundamentals'; export const StripeProvider: FactoryProvider = { provide: Stripe, useFactory: (config: Config) => { - assert(config.plugins.payment); const stripeConfig = config.plugins.payment.stripe; + assert(stripeConfig, 'Stripe configuration is missing'); return new Stripe(stripeConfig.keys.APIKey, omit(stripeConfig, 'keys')); }, diff --git a/packages/backend/server/src/plugins/payment/types.ts b/packages/backend/server/src/plugins/payment/types.ts index 7cf1b4f5d8ac2..16844088e7b40 100644 --- a/packages/backend/server/src/plugins/payment/types.ts +++ b/packages/backend/server/src/plugins/payment/types.ts @@ -1,17 +1,7 @@ import type { User } from '@prisma/client'; -import type { Stripe } from 'stripe'; import type { Payload } from '../../fundamentals/event/def'; -export interface PaymentConfig { - stripe: { - keys: { - APIKey: string; - webhookKey: string; - }; - } & Stripe.StripeConfig; -} - export enum SubscriptionRecurring { Monthly = 'monthly', Yearly = 'yearly', diff --git a/packages/backend/server/src/plugins/payment/webhook.ts b/packages/backend/server/src/plugins/payment/webhook.ts index 3ce709d396c35..0916e000c296d 100644 --- a/packages/backend/server/src/plugins/payment/webhook.ts +++ b/packages/backend/server/src/plugins/payment/webhook.ts @@ -25,7 +25,7 @@ export class StripeWebhook { private readonly stripe: Stripe, private readonly event: EventEmitter2 ) { - assert(config.plugins.payment); + assert(config.plugins.payment.stripe); this.webhookKey = config.plugins.payment.stripe.keys.webhookKey; } diff --git a/packages/backend/server/src/plugins/redis/config.ts b/packages/backend/server/src/plugins/redis/config.ts new file mode 100644 index 0000000000000..4916d31e73156 --- /dev/null +++ b/packages/backend/server/src/plugins/redis/config.ts @@ -0,0 +1,11 @@ +import { RedisOptions } from 'ioredis'; + +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; + +declare module '../config' { + interface PluginsConfig { + redis: ModuleConfig; + } +} + +defineStartupConfig('plugins.redis', {}); diff --git a/packages/backend/server/src/plugins/redis/index.ts b/packages/backend/server/src/plugins/redis/index.ts index ef32b426845a9..145ef4ccd4507 100644 --- a/packages/backend/server/src/plugins/redis/index.ts +++ b/packages/backend/server/src/plugins/redis/index.ts @@ -1,5 +1,6 @@ +import './config'; + import { Global, Provider, Type } from '@nestjs/common'; -import type { RedisOptions } from 'ioredis'; import { Redis } from 'ioredis'; import { ThrottlerStorageRedisService } from 'nestjs-throttler-storage-redis'; @@ -64,5 +65,3 @@ const mutexRedisAdapterProvider: Provider = { requires: ['plugins.redis.host'], }) export class RedisModule {} - -export { RedisOptions }; diff --git a/packages/backend/server/src/plugins/redis/instances.ts b/packages/backend/server/src/plugins/redis/instances.ts index 3c093e15f3221..a1e369918bbb6 100644 --- a/packages/backend/server/src/plugins/redis/instances.ts +++ b/packages/backend/server/src/plugins/redis/instances.ts @@ -30,20 +30,20 @@ class Redis extends IORedis implements OnModuleDestroy, OnModuleInit { @Injectable() export class CacheRedis extends Redis { constructor(config: Config) { - super(config.plugins.redis ?? {}); + super(config.plugins.redis); } } @Injectable() export class SessionRedis extends Redis { constructor(config: Config) { - super({ ...config.plugins.redis, db: (config.plugins.redis?.db ?? 0) + 2 }); + super({ ...config.plugins.redis, db: (config.plugins.redis.db ?? 0) + 2 }); } } @Injectable() export class SocketIoRedis extends Redis { constructor(config: Config) { - super({ ...config.plugins.redis, db: (config.plugins.redis?.db ?? 0) + 3 }); + super({ ...config.plugins.redis, db: (config.plugins.redis.db ?? 0) + 3 }); } } diff --git a/packages/backend/server/src/plugins/redis/types.ts b/packages/backend/server/src/plugins/redis/types.ts deleted file mode 100644 index e7340fab87568..0000000000000 --- a/packages/backend/server/src/plugins/redis/types.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { RedisOptions } from 'ioredis'; - -export type { RedisOptions }; diff --git a/packages/backend/server/src/plugins/registry.ts b/packages/backend/server/src/plugins/registry.ts index 2eea3728af1a8..4d838f851f894 100644 --- a/packages/backend/server/src/plugins/registry.ts +++ b/packages/backend/server/src/plugins/registry.ts @@ -1,11 +1,12 @@ -import { omit } from 'lodash-es'; +import { get, merge, omit, set } from 'lodash-es'; -import { AvailablePlugins } from '../fundamentals/config'; import { OptionalModule, OptionalModuleMetadata } from '../fundamentals/nestjs'; +import { AvailablePlugins } from './config'; export const REGISTERED_PLUGINS = new Map(); +export const ENABLED_PLUGINS = new Set(); -function register(plugin: AvailablePlugins, module: AFFiNEModule) { +function registerPlugin(plugin: AvailablePlugins, module: AFFiNEModule) { REGISTERED_PLUGINS.set(plugin, module); } @@ -15,8 +16,15 @@ interface PluginModuleMetadata extends OptionalModuleMetadata { export const Plugin = (options: PluginModuleMetadata) => { return (target: any) => { - register(options.name, target); + registerPlugin(options.name, target); return OptionalModule(omit(options, 'name'))(target); }; }; + +export function enablePlugin(plugin: AvailablePlugins, config: any = {}) { + config = merge(get(AFFiNE.plugins, plugin), config); + set(AFFiNE.plugins, plugin, config); + + ENABLED_PLUGINS.add(plugin); +} diff --git a/packages/backend/server/src/plugins/storage/config.ts b/packages/backend/server/src/plugins/storage/config.ts new file mode 100644 index 0000000000000..f70561ea83ccd --- /dev/null +++ b/packages/backend/server/src/plugins/storage/config.ts @@ -0,0 +1,27 @@ +import { S3ClientConfig, S3ClientConfigType } from '@aws-sdk/client-s3'; + +import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config'; + +type WARNING = '__YOU_SHOULD_NOT_MANUALLY_CONFIGURATE_THIS_TYPE__'; +declare module '../../fundamentals/storage/config' { + interface StorageProvidersConfig { + // the type here is only existing for extends [StorageProviderType] with better type inference and checking. + 'cloudflare-r2'?: WARNING; + 'aws-s3'?: WARNING; + } +} + +export type S3StorageConfig = S3ClientConfigType; +export type R2StorageConfig = S3ClientConfigType & { + accountId?: string; +}; + +declare module '../config' { + interface PluginsConfig { + 'aws-s3': ModuleConfig; + 'cloudflare-r2': ModuleConfig; + } +} + +defineStartupConfig('plugins.aws-s3', {}); +defineStartupConfig('plugins.cloudflare-r2', {}); diff --git a/packages/backend/server/src/plugins/storage/index.ts b/packages/backend/server/src/plugins/storage/index.ts index 7128f79126804..41bc6ac9bde20 100644 --- a/packages/backend/server/src/plugins/storage/index.ts +++ b/packages/backend/server/src/plugins/storage/index.ts @@ -1,3 +1,5 @@ +import './config'; + import { registerStorageProvider } from '../../fundamentals/storage'; import { Plugin } from '../registry'; import { R2StorageProvider } from './providers/r2'; @@ -38,5 +40,3 @@ export class CloudflareR2Module {} if: config => config.flavor.graphql, }) export class AwsS3Module {} - -export type { R2StorageConfig, S3StorageConfig } from './types'; diff --git a/packages/backend/server/src/plugins/storage/providers/r2.ts b/packages/backend/server/src/plugins/storage/providers/r2.ts index b2d9cbb5d8440..efdae3400cd2f 100644 --- a/packages/backend/server/src/plugins/storage/providers/r2.ts +++ b/packages/backend/server/src/plugins/storage/providers/r2.ts @@ -1,12 +1,15 @@ +import assert from 'node:assert'; + import { Logger } from '@nestjs/common'; -import type { R2StorageConfig } from '../types'; +import type { R2StorageConfig } from '../config'; import { S3StorageProvider } from './s3'; export class R2StorageProvider extends S3StorageProvider { override readonly type = 'cloudflare-r2' as any /* cast 'r2' to 's3' */; constructor(config: R2StorageConfig, bucket: string) { + assert(config.accountId, 'accountId is required for R2 storage provider'); super( { ...config, diff --git a/packages/backend/server/src/plugins/storage/providers/s3.ts b/packages/backend/server/src/plugins/storage/providers/s3.ts index cb0bc2a2b005e..73cbccbcd375f 100644 --- a/packages/backend/server/src/plugins/storage/providers/s3.ts +++ b/packages/backend/server/src/plugins/storage/providers/s3.ts @@ -20,7 +20,7 @@ import { StorageProvider, toBuffer, } from '../../../fundamentals/storage'; -import type { S3StorageConfig } from '../types'; +import type { S3StorageConfig } from '../config'; export class S3StorageProvider implements StorageProvider { protected logger: Logger; diff --git a/packages/backend/server/src/plugins/storage/types.ts b/packages/backend/server/src/plugins/storage/types.ts deleted file mode 100644 index 4f2a756f5c440..0000000000000 --- a/packages/backend/server/src/plugins/storage/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { S3ClientConfigType } from '@aws-sdk/client-s3'; - -type WARNING = '__YOU_SHOULD_NOT_MANUALLY_CONFIGURATE_THIS_TYPE__'; -export type R2StorageConfig = S3ClientConfigType & { - accountId: string; -}; - -export type S3StorageConfig = S3ClientConfigType; - -declare module '../../fundamentals/config/storage' { - interface StorageProvidersConfig { - // the type here is only existing for extends [StorageProviderType] with better type inference and checking. - 'cloudflare-r2'?: WARNING; - 'aws-s3'?: WARNING; - } -} diff --git a/packages/backend/server/src/prelude.ts b/packages/backend/server/src/prelude.ts index 39312a36e3bb8..857e8135aa9d8 100644 --- a/packages/backend/server/src/prelude.ts +++ b/packages/backend/server/src/prelude.ts @@ -5,12 +5,12 @@ import { join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { config } from 'dotenv'; -import { omit } from 'lodash-es'; import { applyEnvToConfig, - getDefaultAFFiNEConfig, + getAFFiNEConfigModifier, } from './fundamentals/config'; +import { enablePlugin } from './plugins'; const configDir = join(fileURLToPath(import.meta.url), '../config'); async function loadRemote(remoteDir: string, file: string) { @@ -37,7 +37,8 @@ async function load() { }); // 2. generate AFFiNE default config and assign to `globalThis.AFFiNE` - globalThis.AFFiNE = getDefaultAFFiNEConfig(); + globalThis.AFFiNE = getAFFiNEConfigModifier(); + globalThis.AFFiNE.use = enablePlugin; // TODO(@forehalo): // Modules may contribute to ENV_MAP, figure out a good way to involve them instead of hardcoding in `./config/affine.env` @@ -55,13 +56,6 @@ async function load() { // 6. apply `process.env` map overriding to `globalThis.AFFiNE` applyEnvToConfig(globalThis.AFFiNE); - - if (AFFiNE.node.dev) { - console.log( - 'AFFiNE Config:', - JSON.stringify(omit(globalThis.AFFiNE, 'ENV_MAP'), null, 2) - ); - } } await load(); diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index ed5dc4f94c41c..34105fd3b0c2b 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -60,7 +60,7 @@ input CreateCheckoutSessionInput { idempotencyKey: String! plan: SubscriptionPlan = Pro recurring: SubscriptionRecurring = Yearly - successCallbackLink: String + successCallbackLink: String! } type CredentialsRequirementType { @@ -175,6 +175,11 @@ The `JSON` scalar type represents JSON values as specified by [ECMA-404](http:// """ scalar JSON @specifiedBy(url: "http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf") +""" +The `JSONObject` scalar type represents JSON objects as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). +""" +scalar JSONObject @specifiedBy(url: "http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf") + type LimitedUserType { """User email""" email: String! @@ -234,6 +239,12 @@ type Mutation { setWorkspaceExperimentalFeature(enable: Boolean!, feature: FeatureType!, workspaceId: String!): Boolean! sharePage(pageId: String!, workspaceId: String!): Boolean! @deprecated(reason: "renamed to publishPage") updateProfile(input: UpdateUserInput!): UserType! + + """update server runtime configurable setting""" + updateRuntimeConfig(id: String!, value: JSON!): ServerRuntimeConfigType! + + """update multiple server runtime configurable settings""" + updateRuntimeConfigs(updates: JSONObject!): [ServerRuntimeConfigType!]! updateSubscriptionRecurring(idempotencyKey: String!, plan: SubscriptionPlan = Pro, recurring: SubscriptionRecurring!): UserSubscription! """Update workspace""" @@ -291,6 +302,9 @@ type Query { """server config""" serverConfig: ServerConfigType! + """get all server runtime configurable settings""" + serverRuntimeConfig: [ServerRuntimeConfigType!]! + """Get user by email""" user(email: String!): UserOrLimitedUser @@ -323,6 +337,14 @@ type RemoveAvatar { success: Boolean! } +enum RuntimeConfigType { + Array + Boolean + Number + Object + String +} + """ The `SafeInt` scalar type represents non-fractional signed whole numeric values that are considered safe as defined by the ECMAScript specification. """ @@ -341,6 +363,9 @@ type ServerConfigType { """enabled server features""" features: [ServerFeature!]! + """server flags""" + flags: ServerFlagsType! + """server flavor""" flavor: String! @deprecated(reason: "use `features`") @@ -366,6 +391,21 @@ enum ServerFeature { Payment } +type ServerFlagsType { + earlyAccessControl: Boolean! + syncClientVersionCheck: Boolean! +} + +type ServerRuntimeConfigType { + description: String! + id: String! + key: String! + module: String! + type: RuntimeConfigType! + updatedAt: DateTime! + value: JSON! +} + enum SubscriptionPlan { AI Enterprise diff --git a/packages/backend/server/tests/auth/controller.spec.ts b/packages/backend/server/tests/auth/controller.spec.ts index 2c08c6729d6bc..6c498c15bdeef 100644 --- a/packages/backend/server/tests/auth/controller.spec.ts +++ b/packages/backend/server/tests/auth/controller.spec.ts @@ -130,7 +130,7 @@ test('should not be able to sign in if forbidden', async t => { await request(app.getHttpServer()) .post('/api/auth/sign-in') .send({ email: u1.email }) - .expect(HttpStatus.PAYMENT_REQUIRED); + .expect(HttpStatus.BAD_REQUEST); t.true(mailer.sendSignInMail.notCalled); diff --git a/packages/backend/server/tests/cache.spec.ts b/packages/backend/server/tests/cache.spec.ts index ac18126ad9a13..71f0f491847ac 100644 --- a/packages/backend/server/tests/cache.spec.ts +++ b/packages/backend/server/tests/cache.spec.ts @@ -1,15 +1,13 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { TestingModule } from '@nestjs/testing'; import test from 'ava'; -import { Cache, CacheModule } from '../src/fundamentals/cache'; -import { ConfigModule } from '../src/fundamentals/config'; +import { Cache } from '../src/fundamentals/cache'; +import { createTestingModule } from './utils'; let cache: Cache; let module: TestingModule; test.beforeEach(async () => { - module = await Test.createTestingModule({ - imports: [ConfigModule.forRoot(), CacheModule], - }).compile(); + module = await createTestingModule(); const prefix = Math.random().toString(36).slice(2, 7); cache = new Proxy(module.get(Cache), { get(target, prop) { diff --git a/packages/backend/server/tests/config.spec.ts b/packages/backend/server/tests/config.spec.ts index ee5ab7dbb7d3e..45a3ad9e968fb 100644 --- a/packages/backend/server/tests/config.spec.ts +++ b/packages/backend/server/tests/config.spec.ts @@ -1,14 +1,13 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { TestingModule } from '@nestjs/testing'; import test from 'ava'; import { Config, ConfigModule } from '../src/fundamentals/config'; +import { createTestingModule } from './utils'; let config: Config; let module: TestingModule; test.beforeEach(async () => { - module = await Test.createTestingModule({ - imports: [ConfigModule.forRoot()], - }).compile(); + module = await createTestingModule(); config = module.get(Config); }); @@ -17,19 +16,21 @@ test.afterEach.always(async () => { }); test('should be able to get config', t => { - t.true(typeof config.host === 'string'); + t.true(typeof config.server.host === 'string'); t.is(config.NODE_ENV, 'test'); }); test('should be able to override config', async t => { - const module = await Test.createTestingModule({ + const module = await createTestingModule({ imports: [ ConfigModule.forRoot({ - host: 'testing', + server: { + host: 'testing', + }, }), ], - }).compile(); + }); const config = module.get(Config); - t.is(config.host, 'testing'); + t.is(config.server.host, 'testing'); }); diff --git a/packages/backend/server/tests/doc.spec.ts b/packages/backend/server/tests/doc.spec.ts index e23c2f17f05db..f0c1fcd056954 100644 --- a/packages/backend/server/tests/doc.spec.ts +++ b/packages/backend/server/tests/doc.spec.ts @@ -10,7 +10,7 @@ import { DocManager, DocModule } from '../src/core/doc'; import { QuotaModule } from '../src/core/quota'; import { StorageModule } from '../src/core/storage'; import { Config } from '../src/fundamentals/config'; -import { createTestingModule, initTestingDB } from './utils'; +import { createTestingModule } from './utils'; const createModule = () => { return createTestingModule({ @@ -28,7 +28,6 @@ test.beforeEach(async () => { }); m = await createModule(); await m.init(); - await initTestingDB(m.get(PrismaClient)); }); test.afterEach.always(async () => { diff --git a/packages/backend/server/tests/feature.spec.ts b/packages/backend/server/tests/feature.spec.ts index d8e19bc32b25b..ef0d8a2855c33 100644 --- a/packages/backend/server/tests/feature.spec.ts +++ b/packages/backend/server/tests/feature.spec.ts @@ -15,7 +15,7 @@ import { import { UserType } from '../src/core/user/types'; import { WorkspaceResolver } from '../src/core/workspaces/resolvers'; import { Permission } from '../src/core/workspaces/types'; -import { ConfigModule } from '../src/fundamentals/config'; +import { Config, ConfigModule } from '../src/fundamentals/config'; import { createTestingApp } from './utils'; @Injectable() @@ -51,10 +51,9 @@ test.beforeEach(async t => { const { app } = await createTestingApp({ imports: [ ConfigModule.forRoot({ - host: 'example.org', - https: true, - featureFlags: { - earlyAccessPreview: true, + server: { + host: 'example.org', + https: true, }, }), FeatureModule, @@ -67,6 +66,8 @@ test.beforeEach(async t => { }, }); + const config = app.get(Config); + await config.runtime.set('flags/earlyAccessControl', true); t.context.app = app; t.context.auth = app.get(AuthService); t.context.feature = app.get(FeatureService); diff --git a/packages/backend/server/tests/graphql.spec.ts b/packages/backend/server/tests/graphql.spec.ts index 815dc9c5cc870..815da59b95ded 100644 --- a/packages/backend/server/tests/graphql.spec.ts +++ b/packages/backend/server/tests/graphql.spec.ts @@ -4,13 +4,13 @@ import { INestApplication, } from '@nestjs/common'; import { Args, Mutation, Query, Resolver } from '@nestjs/graphql'; -import { Test } from '@nestjs/testing'; import testFn, { TestFn } from 'ava'; import request from 'supertest'; -import { ConfigModule } from '../src/fundamentals/config'; -import { GqlModule } from '../src/fundamentals/graphql'; +import { Public } from '../src/core/auth'; +import { createTestingApp } from './utils'; +@Public() @Resolver(() => String) class TestResolver { greating = 'hello world'; @@ -47,16 +47,15 @@ function gql(app: INestApplication, query: string) { } test.beforeEach(async ctx => { - const module = await Test.createTestingModule({ - imports: [ConfigModule.forRoot(), GqlModule], + const { app } = await createTestingApp({ providers: [TestResolver], - }).compile(); + }); - ctx.context.app = await module - .createNestApplication({ - logger: false, - }) - .init(); + ctx.context.app = app; +}); + +test.afterEach.always(async ctx => { + await ctx.context.app.close(); }); test('should be able to execute query', async t => { diff --git a/packages/backend/server/tests/nestjs/throttler.spec.ts b/packages/backend/server/tests/nestjs/throttler.spec.ts index 6b92898f33322..729ca7852f0c3 100644 --- a/packages/backend/server/tests/nestjs/throttler.spec.ts +++ b/packages/backend/server/tests/nestjs/throttler.spec.ts @@ -97,9 +97,11 @@ test.beforeEach(async t => { const { app } = await createTestingApp({ imports: [ ConfigModule.forRoot({ - rateLimiter: { - ttl: 60, - limit: 120, + throttler: { + default: { + ttl: 60, + limit: 120, + }, }, }), AppModule, diff --git a/packages/backend/server/tests/oauth/controller.spec.ts b/packages/backend/server/tests/oauth/controller.spec.ts index d6d4a257ddf33..7eaa52eed8b41 100644 --- a/packages/backend/server/tests/oauth/controller.spec.ts +++ b/packages/backend/server/tests/oauth/controller.spec.ts @@ -10,10 +10,11 @@ import { AppModule } from '../../src/app.module'; import { CurrentUser } from '../../src/core/auth'; import { AuthService } from '../../src/core/auth/service'; import { UserService } from '../../src/core/user'; -import { Config, ConfigModule } from '../../src/fundamentals/config'; +import { URLHelper } from '../../src/fundamentals'; +import { ConfigModule } from '../../src/fundamentals/config'; +import { OAuthProviderName } from '../../src/plugins/oauth/config'; import { GoogleOAuthProvider } from '../../src/plugins/oauth/providers/google'; import { OAuthService } from '../../src/plugins/oauth/service'; -import { OAuthProviderName } from '../../src/plugins/oauth/types'; import { createTestingApp, getSession } from '../utils'; const test = ava as TestFn<{ @@ -71,7 +72,7 @@ test("should be able to redirect to oauth provider's login page", async t => { t.is(redirect.searchParams.get('client_id'), 'google-client-id'); t.is( redirect.searchParams.get('redirect_uri'), - app.get(Config).baseUrl + '/oauth/callback' + app.get(URLHelper).link('/oauth/callback') ); t.is(redirect.searchParams.get('response_type'), 'code'); t.is(redirect.searchParams.get('prompt'), 'select_account'); diff --git a/packages/backend/server/tests/payment/service.spec.ts b/packages/backend/server/tests/payment/service.spec.ts index bca454ceb8c4a..cd64ccf0b82d8 100644 --- a/packages/backend/server/tests/payment/service.spec.ts +++ b/packages/backend/server/tests/payment/service.spec.ts @@ -1,3 +1,5 @@ +import '../../src/plugins/payment'; + import { INestApplication } from '@nestjs/common'; import { PrismaClient } from '@prisma/client'; import ava, { TestFn } from 'ava'; diff --git a/packages/backend/server/tests/utils/utils.ts b/packages/backend/server/tests/utils/utils.ts index 49c588b4ec044..567555bffd18f 100644 --- a/packages/backend/server/tests/utils/utils.ts +++ b/packages/backend/server/tests/utils/utils.ts @@ -100,8 +100,7 @@ export async function createTestingModule( const prisma = m.get(PrismaClient); if (prisma instanceof PrismaClient) { - await flushDB(prisma); - await initFeatureConfigs(prisma); + await initTestingDB(prisma); } return m; diff --git a/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/ai/actions/subscribe.tsx b/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/ai/actions/subscribe.tsx index 0d9a98a9fb5b2..123ad8b17cc54 100644 --- a/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/ai/actions/subscribe.tsx +++ b/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/ai/actions/subscribe.tsx @@ -52,7 +52,7 @@ export const AISubscribe = ({ ...btnProps }: AISubscribeProps) => { idempotencyKey, plan: SubscriptionPlan.AI, coupon: null, - successCallbackLink: null, + successCallbackLink: '/ai-upgrade-success', }); popupWindow(session); setOpenedExternalWindow(true); diff --git a/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/plan-card.tsx b/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/plan-card.tsx index a0d65b2787b85..021998fe2c18c 100644 --- a/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/plan-card.tsx +++ b/packages/frontend/core/src/components/affine/setting-modal/general-setting/plans/plan-card.tsx @@ -263,7 +263,7 @@ const Upgrade = ({ recurring }: { recurring: SubscriptionRecurring }) => { idempotencyKey, plan: SubscriptionPlan.Pro, // Only support prod plan now. coupon: null, - successCallbackLink: null, + successCallbackLink: '/upgrade-success', }); setMutating(false); setIdempotencyKey(nanoid()); diff --git a/packages/frontend/core/src/pages/subscribe.tsx b/packages/frontend/core/src/pages/subscribe.tsx index 5d182459b7e6e..4688abb071b36 100644 --- a/packages/frontend/core/src/pages/subscribe.tsx +++ b/packages/frontend/core/src/pages/subscribe.tsx @@ -65,7 +65,10 @@ export const Component = () => { recurring?.toLowerCase() === 'monthly' ? SubscriptionRecurring.Monthly : SubscriptionRecurring.Yearly, - successCallbackLink: null, + successCallbackLink: + plan?.toLowerCase() === 'ai' + ? '/ai-upgrade-success' + : '/upgrade-success', }); setMessage('Redirecting...'); location.href = checkout; diff --git a/packages/frontend/graphql/src/schema.ts b/packages/frontend/graphql/src/schema.ts index 8ca451e55868d..3e800c93260ad 100644 --- a/packages/frontend/graphql/src/schema.ts +++ b/packages/frontend/graphql/src/schema.ts @@ -56,7 +56,7 @@ export interface CreateCheckoutSessionInput { idempotencyKey: Scalars['String']['input']; plan: InputMaybe; recurring: InputMaybe; - successCallbackLink: InputMaybe; + successCallbackLink: Scalars['String']['input']; } export interface DeleteSessionInput { @@ -114,6 +114,14 @@ export interface QueryChatHistoriesInput { skip: InputMaybe; } +export enum RuntimeConfigType { + Array = 'Array', + Boolean = 'Boolean', + Number = 'Number', + Object = 'Object', + String = 'String', +} + export enum ServerDeploymentType { Affine = 'Affine', Selfhosted = 'Selfhosted',