diff --git a/.vscode/settings.json b/.vscode/settings.json index 7a03c82b31..1dc801f1d6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -122,7 +122,8 @@ "withs", "xmlparser", "zcube", - "zustand" + "zustand", + "Streamable", ], "i18n-ally.localesPaths": "web/public/locales", "i18n-ally.enabledParsers": ["json"], diff --git a/server/Dockerfile b/server/Dockerfile index b40b136a8b..7416bbc081 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -8,6 +8,11 @@ ARG TARGETARCH ENV MINIO_CLIENT_PATH /usr/local/bin/mc RUN wget https://dl.min.io/client/mc/release/linux-$TARGETARCH/mc -O $MINIO_CLIENT_PATH && chmod +x $MINIO_CLIENT_PATH +RUN export MONGO_TOOLS=mongodb-database-tools-ubuntu1804-$( [ "$TARGETARCH" = "amd64" ] && echo "x86_64" || echo "arm64" )-100.8.0 && \ + wget https://fastdl.mongodb.org/tools/db/${MONGO_TOOLS}.tgz && \ + tar -zxvf ${MONGO_TOOLS}.tgz -C /usr/local/bin/ --strip-components=1 ${MONGO_TOOLS}/bin/ && \ + rm -rf ${MONGO_TOOLS}.tgz $MONGO_TOOLS + WORKDIR /app EXPOSE 3000 diff --git a/server/src/database/database.controller.ts b/server/src/database/database.controller.ts index fa54b737e6..b7ff49279f 100644 --- a/server/src/database/database.controller.ts +++ b/server/src/database/database.controller.ts @@ -1,10 +1,37 @@ -import { Controller, Logger, Param, Post, Req, UseGuards } from '@nestjs/common' -import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger' +import { + Controller, + Get, + Logger, + Param, + Post, + Req, + Res, + UseGuards, + Put, + StreamableFile, + UploadedFile, + UseInterceptors, + Body, +} from '@nestjs/common' +import { + ApiBearerAuth, + ApiBody, + ApiConsumes, + ApiOperation, + ApiTags, +} from '@nestjs/swagger' import { Policy, Proxy } from 'database-proxy/dist' import { ApplicationAuthGuard } from 'src/authentication/application.auth.guard' import { JwtAuthGuard } from 'src/authentication/jwt.auth.guard' -import { IRequest } from 'src/utils/interface' +import { IRequest, IResponse } from 'src/utils/interface' import { DatabaseService } from './database.service' +import * as path from 'path' +import { createReadStream, existsSync, mkdirSync } from 'fs' +import { FileInterceptor } from '@nestjs/platform-express' +import { unlink, writeFile } from 'node:fs/promises' +import * as os from 'os' +import { ResponseUtil } from 'src/utils/response' +import { ImportDatabaseDto } from './dto/import-database.dto' @ApiTags('Database') @ApiBearerAuth('Authorization') @@ -57,4 +84,82 @@ export class DatabaseController { } } } + + @ApiOperation({ summary: 'Export database of an application' }) + @UseGuards(JwtAuthGuard, ApplicationAuthGuard) + @Get('export') + async exportDatabase( + @Param('appid') appid: string, + @Res({ passthrough: true }) res: IResponse, + ) { + const tempFilePath = path.join( + os.tmpdir(), + 'mongodb-data', + 'export', + `${appid}-db.gz`, + ) + + // check if dir exists + if (!existsSync(path.dirname(tempFilePath))) { + mkdirSync(path.dirname(tempFilePath), { recursive: true }) + } + + await this.dbService.exportDatabase(appid, tempFilePath) + const filename = path.basename(tempFilePath) + + res.set({ + 'Content-Disposition': `attachment; filename="${filename}"`, + }) + const file = createReadStream(tempFilePath) + return new StreamableFile(file) + } + + @ApiOperation({ summary: 'Import database of an application' }) + @ApiConsumes('multipart/form-data') + @ApiBody({ + type: ImportDatabaseDto, + }) + @UseGuards(JwtAuthGuard, ApplicationAuthGuard) + @Put('import') + @UseInterceptors( + FileInterceptor('file', { + limits: { + fileSize: 256 * 1024 * 1024 * 1024, // 256 GB + }, + }), + ) + async importDatabase( + @UploadedFile() file: Express.Multer.File, + @Body('sourceAppid') sourceAppid: string, + @Param('appid') appid: string, + ) { + // check if db is valid + if (!/^[a-z0-9]{6}$/.test(sourceAppid)) { + return ResponseUtil.error('Invalid source appid') + } + // check if file is .gz + if (file.mimetype !== 'application/gzip') { + return ResponseUtil.error('Invalid db file') + } + + const tempFilePath = path.join( + os.tmpdir(), + 'mongodb-data', + 'import', + `${appid}-${sourceAppid}.gz`, + ) + + // check if dir exists + if (!existsSync(path.dirname(tempFilePath))) { + mkdirSync(path.dirname(tempFilePath), { recursive: true }) + } + + try { + await writeFile(tempFilePath, file.buffer) + await this.dbService.importDatabase(appid, sourceAppid, tempFilePath) + return ResponseUtil.ok({}) + } finally { + if (existsSync(tempFilePath)) await unlink(tempFilePath) + } + } } diff --git a/server/src/database/database.service.ts b/server/src/database/database.service.ts index c032c10b0a..707955551d 100644 --- a/server/src/database/database.service.ts +++ b/server/src/database/database.service.ts @@ -14,6 +14,10 @@ import { DatabasePhase, DatabaseState, } from './entities/database' +import { exec } from 'node:child_process' +import { promisify } from 'node:util' + +const p_exec = promisify(exec) @Injectable() export class DatabaseService { @@ -206,4 +210,44 @@ export class DatabaseService { throw error } } + + async exportDatabase(appid: string, filePath: string) { + const region = await this.regionService.findByAppId(appid) + const database = await this.findOne(appid) + assert(database, 'Database not found') + + const connectionUri = this.getControlConnectionUri(region, database) + assert(connectionUri, 'Database connection uri not found') + + try { + await p_exec( + `mongodump --uri='${connectionUri}' --gzip --archive=${filePath}`, + ) + } catch (error) { + this.logger.error(`failed to export db ${appid}`, error) + throw error + } + } + + async importDatabase( + appid: string, + dbName: string, + filePath: string, + ): Promise { + const region = await this.regionService.findByAppId(appid) + const database = await this.findOne(appid) + assert(database, 'Database not found') + + const connectionUri = this.getControlConnectionUri(region, database) + assert(connectionUri, 'Database connection uri not found') + + try { + await p_exec( + `mongorestore --uri='${connectionUri}' --gzip --archive='${filePath}' --nsFrom="${dbName}.*" --nsTo="${appid}.*" -v --nsInclude="${dbName}.*"`, + ) + } catch (error) { + console.error(`failed to import db to ${appid}:`, error) + throw error + } + } } diff --git a/server/src/database/dto/import-database.dto.ts b/server/src/database/dto/import-database.dto.ts new file mode 100644 index 0000000000..601b985a46 --- /dev/null +++ b/server/src/database/dto/import-database.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger' + +export class ImportDatabaseDto { + @ApiProperty({ type: 'binary', format: 'binary' }) + file: any + + @ApiProperty({ type: 'string', description: 'source appid' }) + sourceAppid: string +}