feat: support import/export application db

.vscode/settings.json

@@ -122,7 +122,8 @@
     "withs",
     "xmlparser",
     "zcube",
-    "zustand"
+    "zustand",
+    "Streamable",
   ],
   "i18n-ally.localesPaths": "web/public/locales",
   "i18n-ally.enabledParsers": ["json"],

server/Dockerfile

@@ -8,6 +8,11 @@ ARG TARGETARCH
 ENV MINIO_CLIENT_PATH /usr/local/bin/mc
 RUN wget https://dl.min.io/client/mc/release/linux-$TARGETARCH/mc -O $MINIO_CLIENT_PATH && chmod +x $MINIO_CLIENT_PATH
 
+RUN export MONGO_TOOLS=mongodb-database-tools-ubuntu1804-$( [ "$TARGETARCH" = "amd64" ] && echo "x86_64" || echo "arm64" )-100.8.0 && \
+    wget https://fastdl.mongodb.org/tools/db/${MONGO_TOOLS}.tgz && \
+    tar -zxvf ${MONGO_TOOLS}.tgz -C /usr/local/bin/ --strip-components=1 ${MONGO_TOOLS}/bin/ && \
+    rm -rf ${MONGO_TOOLS}.tgz $MONGO_TOOLS
+
 WORKDIR /app
 
 EXPOSE 3000

server/src/database/database.controller.ts

@@ -1,10 +1,37 @@
-import { Controller, Logger, Param, Post, Req, UseGuards } from '@nestjs/common'
-import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger'
+import {
+  Controller,
+  Get,
+  Logger,
+  Param,
+  Post,
+  Req,
+  Res,
+  UseGuards,
+  Put,
+  StreamableFile,
+  UploadedFile,
+  UseInterceptors,
+  Body,
+} from '@nestjs/common'
+import {
+  ApiBearerAuth,
+  ApiBody,
+  ApiConsumes,
+  ApiOperation,
+  ApiTags,
+} from '@nestjs/swagger'
 import { Policy, Proxy } from 'database-proxy/dist'
 import { ApplicationAuthGuard } from 'src/authentication/application.auth.guard'
 import { JwtAuthGuard } from 'src/authentication/jwt.auth.guard'
-import { IRequest } from 'src/utils/interface'
+import { IRequest, IResponse } from 'src/utils/interface'
 import { DatabaseService } from './database.service'
+import * as path from 'path'
+import { createReadStream, existsSync, mkdirSync } from 'fs'
+import { FileInterceptor } from '@nestjs/platform-express'
+import { unlink, writeFile } from 'node:fs/promises'
+import * as os from 'os'
+import { ResponseUtil } from 'src/utils/response'
+import { ImportDatabaseDto } from './dto/import-database.dto'
 
 @ApiTags('Database')
 @ApiBearerAuth('Authorization')
@@ -57,4 +84,82 @@ export class DatabaseController {
       }
     }
   }
+
+  @ApiOperation({ summary: 'Export database of an application' })
+  @UseGuards(JwtAuthGuard, ApplicationAuthGuard)
+  @Get('export')
+  async exportDatabase(
+    @Param('appid') appid: string,
+    @Res({ passthrough: true }) res: IResponse,
+  ) {
+    const tempFilePath = path.join(
+      os.tmpdir(),
+      'mongodb-data',
+      'export',
+      `${appid}-db.gz`,
+    )
+
+    // check if dir exists
+    if (!existsSync(path.dirname(tempFilePath))) {
+      mkdirSync(path.dirname(tempFilePath), { recursive: true })
+    }
+
+    await this.dbService.exportDatabase(appid, tempFilePath)
+    const filename = path.basename(tempFilePath)
+
+    res.set({
+      'Content-Disposition': `attachment; filename="${filename}"`,
+    })
+    const file = createReadStream(tempFilePath)
+    return new StreamableFile(file)
+  }
+
+  @ApiOperation({ summary: 'Import database of an application' })
+  @ApiConsumes('multipart/form-data')
+  @ApiBody({
+    type: ImportDatabaseDto,
+  })
+  @UseGuards(JwtAuthGuard, ApplicationAuthGuard)
+  @Put('import')
+  @UseInterceptors(
+    FileInterceptor('file', {
+      limits: {
+        fileSize: 256 * 1024 * 1024 * 1024, // 256 GB
+      },
+    }),
+  )
+  async importDatabase(
+    @UploadedFile() file: Express.Multer.File,
+    @Body('sourceAppid') sourceAppid: string,
+    @Param('appid') appid: string,
+  ) {
+    // check if db is valid
+    if (!/^[a-z0-9]{6}$/.test(sourceAppid)) {
+      return ResponseUtil.error('Invalid source appid')
+    }
+    // check if file is .gz
+    if (file.mimetype !== 'application/gzip') {
+      return ResponseUtil.error('Invalid db file')
+    }
+
+    const tempFilePath = path.join(
+      os.tmpdir(),
+      'mongodb-data',
+      'import',
+      `${appid}-${sourceAppid}.gz`,
+    )
+
+    // check if dir exists
+    if (!existsSync(path.dirname(tempFilePath))) {
+      mkdirSync(path.dirname(tempFilePath), { recursive: true })
+    }
+
+    try {
+      await writeFile(tempFilePath, file.buffer)
+      await this.dbService.importDatabase(appid, sourceAppid, tempFilePath)
+      return ResponseUtil.ok({})
+    } finally {
+      if (existsSync(tempFilePath)) await unlink(tempFilePath)
+    }
+  }
 }

server/src/database/database.service.ts

@@ -14,6 +14,10 @@ import {
   DatabasePhase,
   DatabaseState,
 } from './entities/database'
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+
+const p_exec = promisify(exec)
 
 @Injectable()
 export class DatabaseService {
@@ -206,4 +210,44 @@ export class DatabaseService {
       throw error
     }
   }
+
+  async exportDatabase(appid: string, filePath: string) {
+    const region = await this.regionService.findByAppId(appid)
+    const database = await this.findOne(appid)
+    assert(database, 'Database not found')
+
+    const connectionUri = this.getControlConnectionUri(region, database)
+    assert(connectionUri, 'Database connection uri not found')
+
+    try {
+      await p_exec(
+        `mongodump --uri='${connectionUri}' --gzip --archive=${filePath}`,
+      )
+    } catch (error) {
+      this.logger.error(`failed to export db ${appid}`, error)
+      throw error
+    }
+  }
+
+  async importDatabase(
+    appid: string,
+    dbName: string,
+    filePath: string,
+  ): Promise<void> {
+    const region = await this.regionService.findByAppId(appid)
+    const database = await this.findOne(appid)
+    assert(database, 'Database not found')
+
+    const connectionUri = this.getControlConnectionUri(region, database)
+    assert(connectionUri, 'Database connection uri not found')
+
+    try {
+      await p_exec(
+        `mongorestore --uri='${connectionUri}' --gzip --archive='${filePath}' --nsFrom="${dbName}.*" --nsTo="${appid}.*" -v --nsInclude="${dbName}.*"`,
+      )
+    } catch (error) {
+      console.error(`failed to import db to ${appid}:`, error)
+      throw error
+    }
+  }
 }

server/src/database/dto/import-database.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from '@nestjs/swagger'
+
+export class ImportDatabaseDto {
+  @ApiProperty({ type: 'binary', format: 'binary' })
+  file: any
+
+  @ApiProperty({ type: 'string', description: 'source appid' })
+  sourceAppid: string
+}