diff --git a/apps/nestjs-backend/src/configs/storage.ts b/apps/nestjs-backend/src/configs/storage.ts index cc03356581..4106347735 100644 --- a/apps/nestjs-backend/src/configs/storage.ts +++ b/apps/nestjs-backend/src/configs/storage.ts @@ -4,13 +4,18 @@ import type { ConfigType } from '@nestjs/config'; import { registerAs } from '@nestjs/config'; export const storageConfig = registerAs('storage', () => ({ - provider: (process.env.BACKEND_STORAGE_PROVIDER ?? 'local') as 'local' | 'minio' | 's3', + provider: (process.env.BACKEND_STORAGE_PROVIDER ?? 'local') as + | 'local' + | 'minio' + | 's3' + | 'aliyun', local: { path: process.env.BACKEND_STORAGE_LOCAL_PATH ?? '.assets/uploads', }, publicUrl: process.env.BACKEND_STORAGE_PUBLIC_URL, publicBucket: process.env.BACKEND_STORAGE_PUBLIC_BUCKET || 'public', privateBucket: process.env.BACKEND_STORAGE_PRIVATE_BUCKET || 'private', + privateBucketEndpoint: process.env.BACKEND_STORAGE_PRIVATE_BUCKET_ENDPOINT, minio: { endPoint: process.env.BACKEND_STORAGE_MINIO_ENDPOINT, internalEndPoint: process.env.BACKEND_STORAGE_MINIO_INTERNAL_ENDPOINT, diff --git a/apps/nestjs-backend/src/features/attachments/plugins/aliyun.ts b/apps/nestjs-backend/src/features/attachments/plugins/aliyun.ts new file mode 100644 index 0000000000..285b42ab56 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/plugins/aliyun.ts @@ -0,0 +1,68 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { Injectable } from '@nestjs/common'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { IStorageConfig, StorageConfig } from '../../../configs/storage'; +import { second } from '../../../utils/second'; +import type StorageAdapter from './adapter'; +import { S3Storage } from './s3'; +import type { IRespHeaders } from './types'; + +@Injectable() +export class AliyunStorage extends S3Storage implements StorageAdapter { + private aliyunClient: S3Client; + + constructor(@StorageConfig() readonly config: IStorageConfig) { + super(config); + const { endpoint, region, accessKey, secretKey, maxSockets } = this.config.s3; + const requestHandler = maxSockets + ? new NodeHttpHandler({ + httpsAgent: { + maxSockets: maxSockets, + }, + }) + : undefined; + this.aliyunClient = new S3Client({ + region, + endpoint, + requestHandler, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + }, + }); + } + + private replacePrivateBucketEndpoint(url: string, bucket: string) { + const { privateBucketEndpoint, privateBucket } = this.config; + if (privateBucketEndpoint && bucket === privateBucket) { + const resUrl = new URL(url); + const newUrl = new URL(privateBucketEndpoint); + resUrl.protocol = newUrl.protocol; + resUrl.hostname = newUrl.hostname; + resUrl.port = newUrl.port; + return resUrl.toString(); + } + return url; + } + + async getPreviewUrl( + bucket: string, + path: string, + expiresIn: number = second(this.config.urlExpireIn), + respHeaders?: IRespHeaders + ): Promise { + const command = new GetObjectCommand({ + Bucket: bucket, + Key: path, + ResponseContentDisposition: respHeaders?.['Content-Disposition'], + }); + + const res = await getSignedUrl(this.aliyunClient, command, { + expiresIn: expiresIn ?? second(this.config.tokenExpireIn), + }); + return this.replacePrivateBucketEndpoint(res, bucket); + } +} diff --git a/apps/nestjs-backend/src/features/attachments/plugins/s3.ts b/apps/nestjs-backend/src/features/attachments/plugins/s3.ts index 1d54c4ce40..3380737543 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/s3.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/s3.ts @@ -1,5 +1,6 @@ /* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/naming-convention */ +import https from 'https'; import { join, resolve } from 'path'; import type { Readable } from 'stream'; import { @@ -12,7 +13,7 @@ import { } from '@aws-sdk/client-s3'; import { Upload } from '@aws-sdk/lib-storage'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { BadRequestException, Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; import { NodeHttpHandler } from '@smithy/node-http-handler'; import { getRandomString } from '@teable/core'; import * as fse from 'fs-extra'; @@ -27,15 +28,20 @@ import type { IPresignParams, IPresignRes, IObjectMeta, IRespHeaders } from './t export class S3Storage implements StorageAdapter { private s3Client: S3Client; private s3ClientPrivateNetwork: S3Client; + private httpsAgent: https.Agent; + private s3ClientPreSigner: S3Client; + private logger = new Logger(S3Storage.name); constructor(@StorageConfig() readonly config: IStorageConfig) { const { endpoint, region, accessKey, secretKey, maxSockets } = this.config.s3; this.checkConfig(); + this.httpsAgent = new https.Agent({ + maxSockets, + keepAlive: true, + }); const requestHandler = maxSockets ? new NodeHttpHandler({ - httpsAgent: { - maxSockets: maxSockets, - }, + httpsAgent: this.httpsAgent, }) : undefined; this.s3Client = new S3Client({ @@ -49,6 +55,60 @@ export class S3Storage implements StorageAdapter { }); this.s3ClientPrivateNetwork = this.s3Client; fse.ensureDirSync(StorageAdapter.TEMPORARY_DIR); + + this.s3ClientPreSigner = this.config.privateBucketEndpoint + ? new S3Client({ + region, + endpoint, + bucketEndpoint: true, + requestHandler, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + }, + }) + : this.s3Client; + + const logS3ConnectionsRate = Number(process.env.LOG_S3_CONNECTIONS_RATE); + if (Number.isNaN(logS3ConnectionsRate)) { + this.logger.log('LOG_S3_CONNECTIONS_RATE not set, skipping log'); + return; + } + this.logger.log(`Logging S3 connections rate every ${logS3ConnectionsRate} milliseconds`); + setInterval(() => { + const countRecords: Record< + string, + { socketsCount: number; freeSocketsCount: number; requestsCount: number } + > = {}; + Object.entries(this.httpsAgent.sockets).forEach(([key, sockets]) => { + if (sockets) { + const currentCountRecord = countRecords[key] ?? {}; + countRecords[key] = { + ...countRecords[key], + socketsCount: (currentCountRecord?.socketsCount ?? 0) + sockets.length, + }; + } + }); + Object.entries(this.httpsAgent.freeSockets).forEach(([key, sockets]) => { + if (sockets) { + const currentCountRecord = countRecords[key] ?? {}; + countRecords[key] = { + ...countRecords[key], + freeSocketsCount: (currentCountRecord?.freeSocketsCount ?? 0) + sockets.length, + }; + } + }); + Object.entries(this.httpsAgent.requests).forEach(([key, requests]) => { + if (requests) { + const currentCountRecord = countRecords[key] ?? {}; + countRecords[key] = { + ...countRecords[key], + requestsCount: (currentCountRecord?.requestsCount ?? 0) + requests.length, + }; + } + }); + this.logger.log(`httpsAgent connections: ${JSON.stringify(countRecords, null, 2)}`); + }, logS3ConnectionsRate); } private checkConfig() { @@ -73,6 +133,14 @@ export class S3Storage implements StorageAdapter { } } + private replaceBucketEndpoint(bucket: string, internal?: boolean) { + const { privateBucketEndpoint, privateBucket } = this.config; + if (privateBucketEndpoint && bucket === privateBucket && !internal) { + return privateBucketEndpoint; + } + return bucket; + } + async presigned(bucket: string, dir: string, params: IPresignParams): Promise { try { const { tokenExpireIn, uploadMethod } = this.config; @@ -124,7 +192,7 @@ export class S3Storage implements StorageAdapter { ContentLength: size, ContentType: s3Mimetype = 'application/octet-stream', ETag: hash, - } = await this.s3Client.send(command); + } = await this.s3ClientPrivateNetwork.send(command); const mimetype = s3Mimetype || 'application/octet-stream'; if (!size || !mimetype || !hash) { throw new BadRequestException('Invalid object meta'); @@ -142,7 +210,7 @@ export class S3Storage implements StorageAdapter { Bucket: bucket, Key: path, }); - const { Body } = await this.s3Client.send(getObjectCommand); + const { Body } = await this.s3ClientPrivateNetwork.send(getObjectCommand); const stream = Body as Readable; if (!stream) { throw new BadRequestException('Invalid image stream'); @@ -171,12 +239,12 @@ export class S3Storage implements StorageAdapter { respHeaders?: IRespHeaders ): Promise { const command = new GetObjectCommand({ - Bucket: bucket, + Bucket: this.replaceBucketEndpoint(bucket), Key: path, ResponseContentDisposition: respHeaders?.['Content-Disposition'], }); - return getSignedUrl(this.s3Client, command, { + return getSignedUrl(this.s3ClientPreSigner, command, { expiresIn: expiresIn ?? second(this.config.tokenExpireIn), }); } @@ -198,7 +266,7 @@ export class S3Storage implements StorageAdapter { ContentLanguage: metadata['Content-Language'] as string, ContentMD5: metadata['Content-MD5'] as string, }); - return this.s3Client + return this.s3ClientPrivateNetwork .send(command) .then((res) => ({ hash: res.ETag!, @@ -268,7 +336,7 @@ export class S3Storage implements StorageAdapter { Bucket: bucket, Key: path, }); - await this.s3Client.send(command); + await this.s3ClientPrivateNetwork.send(command); return true; } catch (error) { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -298,8 +366,9 @@ export class S3Storage implements StorageAdapter { Bucket: bucket, Key: path, }); - const { Body: stream, ContentType: mimetype } = await this.s3Client.send(command); + const { Body: stream, ContentType: mimetype } = await this.s3ClientPrivateNetwork.send(command); if (!mimetype?.startsWith('image/')) { + (stream as Readable)?.destroy?.(); throw new BadRequestException('Invalid image'); } if (!stream) { @@ -332,14 +401,14 @@ export class S3Storage implements StorageAdapter { Bucket: bucket, Key: path, }); - const { Body: stream } = await this.s3Client.send(command); + const { Body: stream } = await this.s3ClientPrivateNetwork.send(command); return stream as Readable; } async deleteDir(bucket: string, path: string, throwError: boolean = true) { const prefix = path.endsWith('/') ? path : `${path}/`; - const { Contents } = await this.s3Client.send( + const { Contents } = await this.s3ClientPrivateNetwork.send( new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, @@ -349,7 +418,7 @@ export class S3Storage implements StorageAdapter { if (!Contents || Contents.length === 0) return; try { - await this.s3Client.send( + await this.s3ClientPrivateNetwork.send( new DeleteObjectsCommand({ Bucket: bucket, Delete: { diff --git a/apps/nestjs-backend/src/features/attachments/plugins/storage.ts b/apps/nestjs-backend/src/features/attachments/plugins/storage.ts index 6d4cfaa004..54560c1385 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/storage.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/storage.ts @@ -1,12 +1,13 @@ /* eslint-disable @typescript-eslint/naming-convention */ import type { Provider } from '@nestjs/common'; -import { Inject } from '@nestjs/common'; +import { Inject, Logger } from '@nestjs/common'; import { ClsService } from 'nestjs-cls'; import { CacheService } from '../../../cache/cache.service'; import { baseConfig, type IBaseConfig } from '../../../configs/base.config'; import type { IStorageConfig } from '../../../configs/storage'; import { storageConfig } from '../../../configs/storage'; import type { IClsStore } from '../../../types/cls'; +import { AliyunStorage } from './aliyun'; import { LocalStorage } from './local'; import { MinioStorage } from './minio'; import { S3Storage } from './s3'; @@ -23,6 +24,7 @@ export const storageAdapterProvider: Provider = { cacheService: CacheService, cls: ClsService ) => { + Logger.log(`[Storage provider]: ${config.provider}`); switch (config.provider) { case 'local': return new LocalStorage(config, baseConfig, cacheService, cls); @@ -30,6 +32,8 @@ export const storageAdapterProvider: Provider = { return new MinioStorage(config); case 's3': return new S3Storage(config); + case 'aliyun': + return new AliyunStorage(config); default: throw new Error('Invalid storage provider'); } diff --git a/apps/nextjs-app/.env.example b/apps/nextjs-app/.env.example index 7667ef1b70..cdaca61805 100644 --- a/apps/nextjs-app/.env.example +++ b/apps/nextjs-app/.env.example @@ -12,6 +12,8 @@ BACKEND_STORAGE_PUBLIC_URL=http://localhost:3000/api/attachments/read/public # s3 cloud storage BACKEND_STORAGE_S3_REGION=us-east-2 BACKEND_STORAGE_S3_ENDPOINT=https://s3.us-east-2.amazonaws.com +# Used to configure a custom domain for accessing private buckets, can replace the default S3 endpoint +BACKEND_STORAGE_PRIVATE_BUCKET_ENDPOINT=https://custom.domain # s3 internal endpoint, optional BACKEND_STORAGE_S3_INTERNAL_ENDPOINT=https://s3.us-east-2.amazonaws-internal.com BACKEND_STORAGE_S3_ACCESS_KEY=your_access_key diff --git a/packages/openapi/src/attachment/utils.ts b/packages/openapi/src/attachment/utils.ts index 03110322a7..f63a6a6966 100644 --- a/packages/openapi/src/attachment/utils.ts +++ b/packages/openapi/src/attachment/utils.ts @@ -8,7 +8,7 @@ export const READ_PATH = '/api/attachments/read'; export const getPublicFullStorageUrl = ( storage: { - provider?: 'local' | 's3' | 'minio'; + provider?: 'local' | 's3' | 'minio' | 'aliyun'; prefix?: string; publicBucket?: string; publicUrl?: string; @@ -24,7 +24,7 @@ export const getPublicFullStorageUrl = ( if (provider === 'minio') { return prefix + pathJoin('/', bucket, path); } - if (provider === 's3') { + if (provider === 's3' || provider === 'aliyun') { return prefix + pathJoin('/', path); } return prefix + pathJoin(READ_PATH, bucket, path);