feat: add support s3 custom domain

apps/nestjs-backend/src/configs/storage.ts

@@ -4,13 +4,18 @@ import type { ConfigType } from '@nestjs/config';
 import { registerAs } from '@nestjs/config';
 
 export const storageConfig = registerAs('storage', () => ({
-  provider: (process.env.BACKEND_STORAGE_PROVIDER ?? 'local') as 'local' | 'minio' | 's3',
+  provider: (process.env.BACKEND_STORAGE_PROVIDER ?? 'local') as
+    | 'local'
+    | 'minio'
+    | 's3'
+    | 'aliyun',
   local: {
     path: process.env.BACKEND_STORAGE_LOCAL_PATH ?? '.assets/uploads',
   },
   publicUrl: process.env.BACKEND_STORAGE_PUBLIC_URL,
   publicBucket: process.env.BACKEND_STORAGE_PUBLIC_BUCKET || 'public',
   privateBucket: process.env.BACKEND_STORAGE_PRIVATE_BUCKET || 'private',
+  privateBucketEndpoint: process.env.BACKEND_STORAGE_PRIVATE_BUCKET_ENDPOINT,
   minio: {
     endPoint: process.env.BACKEND_STORAGE_MINIO_ENDPOINT,
     internalEndPoint: process.env.BACKEND_STORAGE_MINIO_INTERNAL_ENDPOINT,

apps/nestjs-backend/src/features/attachments/plugins/aliyun.ts

@@ -0,0 +1,68 @@
+/* eslint-disable sonarjs/no-duplicate-string */
+/* eslint-disable @typescript-eslint/naming-convention */
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { Injectable } from '@nestjs/common';
+import { NodeHttpHandler } from '@smithy/node-http-handler';
+import { IStorageConfig, StorageConfig } from '../../../configs/storage';
+import { second } from '../../../utils/second';
+import type StorageAdapter from './adapter';
+import { S3Storage } from './s3';
+import type { IRespHeaders } from './types';
+
+@Injectable()
+export class AliyunStorage extends S3Storage implements StorageAdapter {
+  private aliyunClient: S3Client;
+
+  constructor(@StorageConfig() readonly config: IStorageConfig) {
+    super(config);
+    const { endpoint, region, accessKey, secretKey, maxSockets } = this.config.s3;
+    const requestHandler = maxSockets
+      ? new NodeHttpHandler({
+          httpsAgent: {
+            maxSockets: maxSockets,
+          },
+        })
+      : undefined;
+    this.aliyunClient = new S3Client({
+      region,
+      endpoint,
+      requestHandler,
+      credentials: {
+        accessKeyId: accessKey,
+        secretAccessKey: secretKey,
+      },
+    });
+  }
+
+  private replacePrivateBucketEndpoint(url: string, bucket: string) {
+    const { privateBucketEndpoint, privateBucket } = this.config;
+    if (privateBucketEndpoint && bucket === privateBucket) {
+      const resUrl = new URL(url);
+      const newUrl = new URL(privateBucketEndpoint);
+      resUrl.protocol = newUrl.protocol;
+      resUrl.hostname = newUrl.hostname;
+      resUrl.port = newUrl.port;
+      return resUrl.toString();
+    }
+    return url;
+  }
+
+  async getPreviewUrl(
+    bucket: string,
+    path: string,
+    expiresIn: number = second(this.config.urlExpireIn),
+    respHeaders?: IRespHeaders
+  ): Promise<string> {
+    const command = new GetObjectCommand({
+      Bucket: bucket,
+      Key: path,
+      ResponseContentDisposition: respHeaders?.['Content-Disposition'],
+    });
+
+    const res = await getSignedUrl(this.aliyunClient, command, {
+      expiresIn: expiresIn ?? second(this.config.tokenExpireIn),
+    });
+    return this.replacePrivateBucketEndpoint(res, bucket);
+  }
+}

apps/nestjs-backend/src/features/attachments/plugins/s3.ts

@@ -1,5 +1,6 @@
 /* eslint-disable sonarjs/no-duplicate-string */
 /* eslint-disable @typescript-eslint/naming-convention */
+import https from 'https';
 import { join, resolve } from 'path';
 import type { Readable } from 'stream';
 import {
@@ -12,7 +13,7 @@ import {
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
-import { BadRequestException, Injectable } from '@nestjs/common';
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import { NodeHttpHandler } from '@smithy/node-http-handler';
 import { getRandomString } from '@teable/core';
 import * as fse from 'fs-extra';
@@ -27,15 +28,20 @@ import type { IPresignParams, IPresignRes, IObjectMeta, IRespHeaders } from './t
 export class S3Storage implements StorageAdapter {
   private s3Client: S3Client;
   private s3ClientPrivateNetwork: S3Client;
+  private httpsAgent: https.Agent;
+  private s3ClientPreSigner: S3Client;
+  private logger = new Logger(S3Storage.name);
 
   constructor(@StorageConfig() readonly config: IStorageConfig) {
     const { endpoint, region, accessKey, secretKey, maxSockets } = this.config.s3;
     this.checkConfig();
+    this.httpsAgent = new https.Agent({
+      maxSockets,
+      keepAlive: true,
+    });
     const requestHandler = maxSockets
       ? new NodeHttpHandler({
-          httpsAgent: {
-            maxSockets: maxSockets,
-          },
+          httpsAgent: this.httpsAgent,
         })
       : undefined;
     this.s3Client = new S3Client({
@@ -49,6 +55,60 @@ export class S3Storage implements StorageAdapter {
     });
     this.s3ClientPrivateNetwork = this.s3Client;
     fse.ensureDirSync(StorageAdapter.TEMPORARY_DIR);
+
+    this.s3ClientPreSigner = this.config.privateBucketEndpoint
+      ? new S3Client({
+          region,
+          endpoint,
+          bucketEndpoint: true,
+          requestHandler,
+          credentials: {
+            accessKeyId: accessKey,
+            secretAccessKey: secretKey,
+          },
+        })
+      : this.s3Client;
+
+    const logS3ConnectionsRate = Number(process.env.LOG_S3_CONNECTIONS_RATE);
+    if (Number.isNaN(logS3ConnectionsRate)) {
+      this.logger.log('LOG_S3_CONNECTIONS_RATE not set, skipping log');
+      return;
+    }
+    this.logger.log(`Logging S3 connections rate every ${logS3ConnectionsRate} milliseconds`);
+    setInterval(() => {
+      const countRecords: Record<
+        string,
+        { socketsCount: number; freeSocketsCount: number; requestsCount: number }
+      > = {};
+      Object.entries(this.httpsAgent.sockets).forEach(([key, sockets]) => {
+        if (sockets) {
+          const currentCountRecord = countRecords[key] ?? {};
+          countRecords[key] = {
+            ...countRecords[key],
+            socketsCount: (currentCountRecord?.socketsCount ?? 0) + sockets.length,
+          };
+        }
+      });
+      Object.entries(this.httpsAgent.freeSockets).forEach(([key, sockets]) => {
+        if (sockets) {
+          const currentCountRecord = countRecords[key] ?? {};
+          countRecords[key] = {
+            ...countRecords[key],
+            freeSocketsCount: (currentCountRecord?.freeSocketsCount ?? 0) + sockets.length,
+          };
+        }
+      });
+      Object.entries(this.httpsAgent.requests).forEach(([key, requests]) => {
+        if (requests) {
+          const currentCountRecord = countRecords[key] ?? {};
+          countRecords[key] = {
+            ...countRecords[key],
+            requestsCount: (currentCountRecord?.requestsCount ?? 0) + requests.length,
+          };
+        }
+      });
+      this.logger.log(`httpsAgent connections: ${JSON.stringify(countRecords, null, 2)}`);
+    }, logS3ConnectionsRate);
   }
 
   private checkConfig() {
@@ -73,6 +133,14 @@ export class S3Storage implements StorageAdapter {
     }
   }
 
+  private replaceBucketEndpoint(bucket: string, internal?: boolean) {
+    const { privateBucketEndpoint, privateBucket } = this.config;
+    if (privateBucketEndpoint && bucket === privateBucket && !internal) {
+      return privateBucketEndpoint;
+    }
+    return bucket;
+  }
+
   async presigned(bucket: string, dir: string, params: IPresignParams): Promise<IPresignRes> {
     try {
       const { tokenExpireIn, uploadMethod } = this.config;
@@ -124,7 +192,7 @@ export class S3Storage implements StorageAdapter {
       ContentLength: size,
       ContentType: s3Mimetype = 'application/octet-stream',
       ETag: hash,
-    } = await this.s3Client.send(command);
+    } = await this.s3ClientPrivateNetwork.send(command);
     const mimetype = s3Mimetype || 'application/octet-stream';
     if (!size || !mimetype || !hash) {
       throw new BadRequestException('Invalid object meta');
@@ -142,7 +210,7 @@ export class S3Storage implements StorageAdapter {
       Bucket: bucket,
       Key: path,
     });
-    const { Body } = await this.s3Client.send(getObjectCommand);
+    const { Body } = await this.s3ClientPrivateNetwork.send(getObjectCommand);
     const stream = Body as Readable;
     if (!stream) {
       throw new BadRequestException('Invalid image stream');
@@ -171,12 +239,12 @@ export class S3Storage implements StorageAdapter {
     respHeaders?: IRespHeaders
   ): Promise<string> {
     const command = new GetObjectCommand({
-      Bucket: bucket,
+      Bucket: this.replaceBucketEndpoint(bucket),
       Key: path,
       ResponseContentDisposition: respHeaders?.['Content-Disposition'],
     });
 
-    return getSignedUrl(this.s3Client, command, {
+    return getSignedUrl(this.s3ClientPreSigner, command, {
       expiresIn: expiresIn ?? second(this.config.tokenExpireIn),
     });
   }
@@ -198,7 +266,7 @@ export class S3Storage implements StorageAdapter {
       ContentLanguage: metadata['Content-Language'] as string,
       ContentMD5: metadata['Content-MD5'] as string,
     });
-    return this.s3Client
+    return this.s3ClientPrivateNetwork
       .send(command)
       .then((res) => ({
         hash: res.ETag!,
@@ -268,7 +336,7 @@ export class S3Storage implements StorageAdapter {
         Bucket: bucket,
         Key: path,
       });
-      await this.s3Client.send(command);
+      await this.s3ClientPrivateNetwork.send(command);
       return true;
     } catch (error) {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -298,8 +366,9 @@ export class S3Storage implements StorageAdapter {
       Bucket: bucket,
       Key: path,
     });
-    const { Body: stream, ContentType: mimetype } = await this.s3Client.send(command);
+    const { Body: stream, ContentType: mimetype } = await this.s3ClientPrivateNetwork.send(command);
     if (!mimetype?.startsWith('image/')) {
+      (stream as Readable)?.destroy?.();
       throw new BadRequestException('Invalid image');
     }
     if (!stream) {
@@ -332,14 +401,14 @@ export class S3Storage implements StorageAdapter {
       Bucket: bucket,
       Key: path,
     });
-    const { Body: stream } = await this.s3Client.send(command);
+    const { Body: stream } = await this.s3ClientPrivateNetwork.send(command);
     return stream as Readable;
   }
 
   async deleteDir(bucket: string, path: string, throwError: boolean = true) {
     const prefix = path.endsWith('/') ? path : `${path}/`;
 
-    const { Contents } = await this.s3Client.send(
+    const { Contents } = await this.s3ClientPrivateNetwork.send(
       new ListObjectsV2Command({
         Bucket: bucket,
         Prefix: prefix,
@@ -349,7 +418,7 @@ export class S3Storage implements StorageAdapter {
     if (!Contents || Contents.length === 0) return;
 
     try {
-      await this.s3Client.send(
+      await this.s3ClientPrivateNetwork.send(
         new DeleteObjectsCommand({
           Bucket: bucket,
           Delete: {

apps/nestjs-backend/src/features/attachments/plugins/storage.ts

@@ -1,12 +1,13 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import type { Provider } from '@nestjs/common';
-import { Inject } from '@nestjs/common';
+import { Inject, Logger } from '@nestjs/common';
 import { ClsService } from 'nestjs-cls';
 import { CacheService } from '../../../cache/cache.service';
 import { baseConfig, type IBaseConfig } from '../../../configs/base.config';
 import type { IStorageConfig } from '../../../configs/storage';
 import { storageConfig } from '../../../configs/storage';
 import type { IClsStore } from '../../../types/cls';
+import { AliyunStorage } from './aliyun';
 import { LocalStorage } from './local';
 import { MinioStorage } from './minio';
 import { S3Storage } from './s3';
@@ -23,13 +24,16 @@ export const storageAdapterProvider: Provider = {
     cacheService: CacheService,
     cls: ClsService<IClsStore>
   ) => {
+    Logger.log(`[Storage provider]: ${config.provider}`);
     switch (config.provider) {
       case 'local':
         return new LocalStorage(config, baseConfig, cacheService, cls);
       case 'minio':
         return new MinioStorage(config);
       case 's3':
         return new S3Storage(config);
+      case 'aliyun':
+        return new AliyunStorage(config);
       default:
         throw new Error('Invalid storage provider');
     }

apps/nextjs-app/.env.example

@@ -12,6 +12,8 @@ BACKEND_STORAGE_PUBLIC_URL=http://localhost:3000/api/attachments/read/public
 # s3 cloud storage
 BACKEND_STORAGE_S3_REGION=us-east-2	
 BACKEND_STORAGE_S3_ENDPOINT=https://s3.us-east-2.amazonaws.com
+# Used to configure a custom domain for accessing private buckets, can replace the default S3 endpoint
+BACKEND_STORAGE_PRIVATE_BUCKET_ENDPOINT=https://custom.domain
 # s3 internal endpoint, optional
 BACKEND_STORAGE_S3_INTERNAL_ENDPOINT=https://s3.us-east-2.amazonaws-internal.com
 BACKEND_STORAGE_S3_ACCESS_KEY=your_access_key

packages/openapi/src/attachment/utils.ts

@@ -8,7 +8,7 @@ export const READ_PATH = '/api/attachments/read';
 
 export const getPublicFullStorageUrl = (
   storage: {
-    provider?: 'local' | 's3' | 'minio';
+    provider?: 'local' | 's3' | 'minio' | 'aliyun';
     prefix?: string;
     publicBucket?: string;
     publicUrl?: string;
@@ -24,7 +24,7 @@ export const getPublicFullStorageUrl = (
   if (provider === 'minio') {
     return prefix + pathJoin('/', bucket, path);
   }
-  if (provider === 's3') {
+  if (provider === 's3' || provider === 'aliyun') {
     return prefix + pathJoin('/', path);
   }
   return prefix + pathJoin(READ_PATH, bucket, path);