feat(server): runtime setting support

toeverything · May 22, 2024 · bd2ee8a · bd2ee8a
1 parent 2783361
commit bd2ee8a
Show file tree

Hide file tree

Showing 115 changed files with 1,862 additions and 1,086 deletions.
diff --git a/packages/backend/server/migrations/20240520055805_runtime_setting/migration.sql b/packages/backend/server/migrations/20240520055805_runtime_setting/migration.sql
@@ -0,0 +1,23 @@
+-- CreateEnum
+CREATE TYPE "RuntimeConfigType" AS ENUM ('String', 'Number', 'Boolean', 'Object', 'Array');
+
+-- CreateTable
+CREATE TABLE "app_runtime_settings" (
+    "id" VARCHAR NOT NULL,
+    "type" "RuntimeConfigType" NOT NULL,
+    "module" VARCHAR NOT NULL,
+    "key" VARCHAR NOT NULL,
+    "value" JSON NOT NULL,
+    "description" TEXT NOT NULL,
+    "updated_at" TIMESTAMPTZ(6) NOT NULL,
+    "deleted_at" TIMESTAMPTZ(6),
+    "last_updated_by" VARCHAR(36),
+
+    CONSTRAINT "app_runtime_settings_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "app_runtime_settings_module_key_key" ON "app_runtime_settings"("module", "key");
+
+-- AddForeignKey
+ALTER TABLE "app_runtime_settings" ADD CONSTRAINT "app_runtime_settings_last_updated_by_fkey" FOREIGN KEY ("last_updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prisma
@@ -22,15 +22,16 @@ model User {
   /// for example, the value will be false if user never registered and invited into a workspace by others.
   registered      Boolean   @default(true)
 
-  features             UserFeatures[]
-  customer             UserStripeCustomer?
-  subscriptions        UserSubscription[]
-  invoices             UserInvoice[]
-  workspacePermissions WorkspaceUserPermission[]
-  pagePermissions      WorkspacePageUserPermission[]
-  connectedAccounts    ConnectedAccount[]
-  sessions             UserSession[]
-  aiSessions           AiSession[]
+  features              UserFeatures[]
+  customer              UserStripeCustomer?
+  subscriptions         UserSubscription[]
+  invoices              UserInvoice[]
+  workspacePermissions  WorkspaceUserPermission[]
+  pagePermissions       WorkspacePageUserPermission[]
+  connectedAccounts     ConnectedAccount[]
+  sessions              UserSession[]
+  aiSessions            AiSession[]
+  updatedRuntimeConfigs RuntimeConfig[]
 
   @@index([email])
   @@map("users")
@@ -502,3 +503,28 @@ model DataMigration {
 
   @@map("_data_migrations")
 }
+
+enum RuntimeConfigType {
+  String
+  Number
+  Boolean
+  Object
+  Array
+}
+
+model RuntimeConfig {
+  id            String            @id @db.VarChar
+  type          RuntimeConfigType
+  module        String            @db.VarChar
+  key           String            @db.VarChar
+  value         Json              @db.Json
+  description   String            @db.Text
+  updatedAt     DateTime          @updatedAt @map("updated_at") @db.Timestamptz(6)
+  deletedAt     DateTime?         @map("deleted_at") @db.Timestamptz(6)
+  lastUpdatedBy String?           @map("last_updated_by") @db.VarChar(36)
+
+  lastUpdatedByUser User? @relation(fields: [lastUpdatedBy], references: [id])
+
+  @@unique([module, key])
+  @@map("app_runtime_settings")
+}
diff --git a/packages/backend/server/src/app.module.ts b/packages/backend/server/src/app.module.ts
@@ -17,8 +17,11 @@ import { UserModule } from './core/user';
 import { WorkspaceModule } from './core/workspaces';
 import { getOptionalModuleMetadata } from './fundamentals';
 import { CacheModule } from './fundamentals/cache';
-import type { AvailablePlugins } from './fundamentals/config';
-import { Config, ConfigModule } from './fundamentals/config';
+import {
+  Config,
+  ConfigModule,
+  mergeConfigOverride,
+} from './fundamentals/config';
 import { EventModule } from './fundamentals/event';
 import { GqlModule } from './fundamentals/graphql';
 import { HelpersModule } from './fundamentals/helpers';
@@ -30,8 +33,10 @@ import { StorageProviderModule } from './fundamentals/storage';
 import { RateLimiterModule } from './fundamentals/throttler';
 import { WebSocketModule } from './fundamentals/websocket';
 import { REGISTERED_PLUGINS } from './plugins';
+import { ENABLED_PLUGINS } from './plugins/registry';
 
 export const FunctionalityModules = [
+  ConfigModule.forRoot(),
   ConfigModule.forRoot(),
   ScheduleModule.forRoot(),
   EventModule,
@@ -112,6 +117,8 @@ export class AppModuleBuilder {
 }
 
 function buildAppModule() {
+  AFFiNE = mergeConfigOverride(AFFiNE);
+  // @ts-expect-error runtime instance will be injected after config instance created
   const factor = new AppModuleBuilder(AFFiNE);
 
   factor
@@ -147,8 +154,8 @@ function buildAppModule() {
     );
 
   // plugin modules
-  AFFiNE.plugins.enabled.forEach(name => {
-    const plugin = REGISTERED_PLUGINS.get(name as AvailablePlugins);
+  ENABLED_PLUGINS.forEach(name => {
+    const plugin = REGISTERED_PLUGINS.get(name);
     if (!plugin) {
       throw new Error(`Unknown plugin ${name}`);
     }

diff --git a/packages/backend/server/src/app.ts b/packages/backend/server/src/app.ts
@@ -50,11 +50,13 @@ export async function createApp() {
     app.useWebSocketAdapter(adapter);
   }
 
-  if (AFFiNE.isSelfhosted && AFFiNE.telemetry.enabled) {
+  if (AFFiNE.isSelfhosted && AFFiNE.metrics.telemetry.enabled) {
     const mixpanel = await import('mixpanel');
-    mixpanel.init(AFFiNE.telemetry.token).track('selfhost-server-started', {
-      version: AFFiNE.version,
-    });
+    mixpanel
+      .init(AFFiNE.metrics.telemetry.token)
+      .track('selfhost-server-started', {
+        version: AFFiNE.version,
+      });
   }
 
   return app;

diff --git a/packages/backend/server/src/config/affine.env.ts b/packages/backend/server/src/config/affine.env.ts
@@ -1,24 +1,20 @@
 // Convenient way to map environment variables to config values.
 AFFiNE.ENV_MAP = {
-  AFFINE_SERVER_PORT: ['port', 'int'],
-  AFFINE_SERVER_HOST: 'host',
-  AFFINE_SERVER_SUB_PATH: 'path',
-  AFFINE_SERVER_HTTPS: ['https', 'boolean'],
-  DATABASE_URL: 'db.url',
-  ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'],
-  CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'],
-  OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId',
-  OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret',
-  OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId',
-  OAUTH_GITHUB_CLIENT_SECRET: 'plugins.oauth.providers.github.clientSecret',
+  AFFINE_SERVER_PORT: ['server.port', 'int'],
+  AFFINE_SERVER_HOST: 'server.host',
+  AFFINE_SERVER_SUB_PATH: 'server.path',
+  AFFINE_SERVER_HTTPS: ['server.https', 'boolean'],
+  TELEMETRY_ENABLE: ['metrics.telemetry.enabled', 'boolean'],
   MAILER_HOST: 'mailer.host',
   MAILER_PORT: ['mailer.port', 'int'],
   MAILER_USER: 'mailer.auth.user',
   MAILER_PASSWORD: 'mailer.auth.pass',
   MAILER_SENDER: 'mailer.from.address',
   MAILER_SECURE: ['mailer.secure', 'boolean'],
-  THROTTLE_TTL: ['rateLimiter.ttl', 'int'],
-  THROTTLE_LIMIT: ['rateLimiter.limit', 'int'],
+  OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId',
+  OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret',
+  OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId',
+  OAUTH_GITHUB_CLIENT_SECRET: 'plugins.oauth.providers.github.clientSecret',
   COPILOT_OPENAI_API_KEY: 'plugins.copilot.openai.apiKey',
   COPILOT_FAL_API_KEY: 'plugins.copilot.fal.apiKey',
   COPILOT_UNSPLASH_API_KEY: 'plugins.copilot.unsplashKey',
@@ -28,16 +24,6 @@ AFFiNE.ENV_MAP = {
   REDIS_SERVER_PASSWORD: 'plugins.redis.password',
   REDIS_SERVER_DATABASE: ['plugins.redis.db', 'int'],
   DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'],
-  DOC_MERGE_USE_JWST_CODEC: [
-    'doc.manager.experimentalMergeWithYOcto',
-    'boolean',
-  ],
   STRIPE_API_KEY: 'plugins.payment.stripe.keys.APIKey',
   STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey',
-  FEATURES_EARLY_ACCESS_PREVIEW: ['featureFlags.earlyAccessPreview', 'boolean'],
-  FEATURES_SYNC_CLIENT_VERSION_CHECK: [
-    'featureFlags.syncClientVersionCheck',
-    'boolean',
-  ],
-  TELEMETRY_ENABLE: ['telemetry.enabled', 'boolean'],
 };
diff --git a/packages/backend/server/src/config/affine.self.ts b/packages/backend/server/src/config/affine.self.ts
@@ -20,35 +20,47 @@ const env = process.env;
 AFFiNE.metrics.enabled = !AFFiNE.node.test;
 
 if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
-  AFFiNE.plugins.use('cloudflare-r2', {
+  AFFiNE.use('cloudflare-r2', {
     accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
     credentials: {
       accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
       secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
     },
   });
-  AFFiNE.storage.storages.avatar.provider = 'cloudflare-r2';
-  AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
-  AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
+  AFFiNE.storages.avatar.provider = 'cloudflare-r2';
+  AFFiNE.storages.avatar.bucket = 'account-avatar';
+  AFFiNE.storages.avatar.publicLinkFactory = key =>
     `https://avatar.affineassets.com/${key}`;
 
-  AFFiNE.storage.storages.blob.provider = 'cloudflare-r2';
-  AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
+  AFFiNE.storages.blob.provider = 'cloudflare-r2';
+  AFFiNE.storages.blob.bucket = `workspace-blobs-${
     AFFiNE.affine.canary ? 'canary' : 'prod'
   }`;
 
-  AFFiNE.storage.storages.copilot.provider = 'cloudflare-r2';
-  AFFiNE.storage.storages.copilot.bucket = `workspace-copilot-${
-    AFFiNE.affine.canary ? 'canary' : 'prod'
-  }`;
+  AFFiNE.use('copilot', {
+    storage: {
+      provider: 'cloudflare-r2',
+      bucket: `workspace-copilot-${AFFiNE.affine.canary ? 'canary' : 'prod'}`,
+    },
+  });
 }
 
-AFFiNE.plugins.use('copilot', {
-  openai: {},
-  fal: {},
+AFFiNE.use('copilot', {
+  openai: {
+    apiKey: '',
+  },
+  fal: {
+    apiKey: '',
+  },
+});
+AFFiNE.use('redis', {
+  host: env.REDIS_SERVER_HOST,
+  db: 0,
+  port: 6379,
+  username: env.REDIS_SERVER_USER,
+  password: env.REDIS_SERVER_PASSWORD,
 });
-AFFiNE.plugins.use('redis');
-AFFiNE.plugins.use('payment', {
+AFFiNE.use('payment', {
   stripe: {
     keys: {
       // fake the key to ensure the server generate full GraphQL Schema even env vars are not set
@@ -57,7 +69,7 @@ AFFiNE.plugins.use('payment', {
     },
   },
 });
-AFFiNE.plugins.use('oauth');
+AFFiNE.use('oauth');
 
 if (AFFiNE.deploy) {
   AFFiNE.mailer = {
@@ -68,5 +80,5 @@ if (AFFiNE.deploy) {
     },
   };
 
-  AFFiNE.plugins.use('gcloud');
+  AFFiNE.use('gcloud');
 }
diff --git a/packages/backend/server/src/config/affine.ts b/packages/backend/server/src/config/affine.ts
@@ -26,22 +26,14 @@
 // AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
 //
 // /* Whether the server is deployed behind a HTTPS proxied environment */
-AFFiNE.https = false;
+AFFiNE.server.https = false;
 // /* Domain of your server that your server will be available at */
-AFFiNE.host = 'localhost';
+AFFiNE.server.host = 'localhost';
 // /* The local port of your server that will listen on */
-AFFiNE.port = 3010;
+AFFiNE.server.port = 3010;
 // /* The sub path of your server */
-// /* For example, if you set `AFFiNE.path = '/affine'`, then the server will be available at `${domain}/affine` */
-// AFFiNE.path = '/affine';
-//
-//
-// ###############################################################
-// ##                       Database settings                   ##
-// ###############################################################
-//
-// /* The URL of the database where most of AFFiNE server data will be stored in */
-// AFFiNE.db.url = 'postgres://user:passsword@localhost:5432/affine';
+// /* For example, if you set `AFFiNE.server.path = '/affine'`, then the server will be available at `${domain}/affine` */
+// AFFiNE.server.path = '/affine';
 //
 //
 // ###############################################################
@@ -52,19 +44,12 @@ AFFiNE.port = 3010;
 // /* The metrics will be available at `http://localhost:9464/metrics` with [Prometheus] format exported */
 // AFFiNE.metrics.enabled = true;
 //
-// /* Authentication Settings */
-// /* Whether allow anyone signup */
-// AFFiNE.auth.allowSignup = true;
-//
-// /* User Signup password limitation */
-// AFFiNE.auth.password = {
-//   minLength: 8,
-//   maxLength: 32,
-// };
 //
-// /* How long the login session would last by default */
 // AFFiNE.auth.session = {
+//   /* How long the login session would last by default */
 //   ttl: 15 * 24 * 60 * 60, // 15 days
+//   /* How long we should refresh the token before it getting expired */
+//   ttr: 7 * 24 * 60 * 60, // 7 days
 // };
 //
 // /* GraphQL configurations that control the behavior of the Apollo Server behind */
@@ -85,9 +70,6 @@ AFFiNE.port = 3010;
 // /* How long the buffer time of creating a new history snapshot when doc get updated */
 // AFFiNE.doc.history.interval = 1000 * 60 * 10; // 10 minutes
 //
-// /* Use `y-octo` to merge updates at the same time when merging using Yjs */
-// AFFiNE.doc.manager.experimentalMergeWithYOcto = true;
-//
 // /* How often the manager will start a new turn of merging pending updates into doc snapshot */
 // AFFiNE.doc.manager.updatePollInterval = 1000 * 3;
 //
@@ -99,20 +81,20 @@ AFFiNE.port = 3010;
 // /* Redis Plugin */
 // /* Provide caching and session storing backed by Redis. */
 // /* Useful when you deploy AFFiNE server in a cluster. */
-// AFFiNE.plugins.use('redis', {
+// AFFiNE.use('redis', {
 //   /* override options */
 // });
 //
 //
 // /* Payment Plugin */
-// AFFiNE.plugins.use('payment', {
+// AFFiNE.use('payment', {
 //   stripe: { keys: {}, apiVersion: '2023-10-16' },
 // });
 //
 //
 // /* Cloudflare R2 Plugin */
 // /* Enable if you choose to store workspace blobs or user avatars in Cloudflare R2 Storage Service */
-// AFFiNE.plugins.use('cloudflare-r2', {
+// AFFiNE.use('cloudflare-r2', {
 //   accountId: '',
 //   credentials: {
 //     accessKeyId: '',
@@ -122,17 +104,17 @@ AFFiNE.port = 3010;
 //
 // /* AWS S3 Plugin */
 // /* Enable if you choose to store workspace blobs or user avatars in AWS S3 Storage Service */
-// AFFiNE.plugins.use('aws-s3', {
+// AFFiNE.use('aws-s3', {
 //  credentials: {
 //    accessKeyId: '',
 //    secretAccessKey: '',
 // })
 // /* Update the provider of storages */
-// AFFiNE.storage.storages.blob.provider = 'r2';
-// AFFiNE.storage.storages.avatar.provider = 'r2';
+// AFFiNE.storages.blob.provider = 'cloudflare-r2';
+// AFFiNE.storages.avatar.provider = 'cloudflare-r2';
 //
 // /* OAuth Plugin */
-// AFFiNE.plugins.use('oauth', {
+// AFFiNE.use('oauth', {
 //   providers: {
 //     github: {
 //       clientId: '',
@@ -154,3 +136,18 @@ AFFiNE.port = 3010;
 //     },
 //   },
 // });
+//
+// /* Copilot Plugin */
+// AFFiNE.use('copilot', {
+//   openai: {
+//     apiKey: 'your-key',
+//   },
+//   fal: {
+//     apiKey: 'your-key',
+//   },
+//   unsplashKey: 'your-key',
+//   storage: {
+//     provider: 'cloudflare-r2',
+//     bucket: 'copilot',
+//   }
+// })