diff --git a/redisinsight/api/config/default.ts b/redisinsight/api/config/default.ts index c163451dfe..013f9adb08 100644 --- a/redisinsight/api/config/default.ts +++ b/redisinsight/api/config/default.ts @@ -50,6 +50,7 @@ export default { contentUri: '/static/content', defaultPluginsUri: '/static/plugins', pluginsAssetsUri: '/static/resources/plugins', + base: process.env.RI_BASE || '/', secretStoragePassword: process.env.SECRET_STORAGE_PASSWORD, tls: process.env.SERVER_TLS ? process.env.SERVER_TLS === 'true' : true, tlsCert: process.env.SERVER_TLS_CERT, diff --git a/redisinsight/api/src/modules/bulk-actions/bulk-import.controller.ts b/redisinsight/api/src/modules/bulk-actions/bulk-import.controller.ts index 00110f1792..98bb34715b 100644 --- a/redisinsight/api/src/modules/bulk-actions/bulk-import.controller.ts +++ b/redisinsight/api/src/modules/bulk-actions/bulk-import.controller.ts @@ -14,6 +14,7 @@ import { UploadImportFileDto } from 'src/modules/bulk-actions/dto/upload-import- import { ClientMetadataParam } from 'src/common/decorators'; import { ClientMetadata } from 'src/common/models'; import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface'; +import { UploadImportFileByPathDto } from 'src/modules/bulk-actions/dto/upload-import-file-by-path.dto'; @UsePipes(new ValidationPipe({ transform: true })) @UseInterceptors(ClassSerializerInterceptor) @@ -40,4 +41,21 @@ export class BulkImportController { ): Promise { return this.service.import(clientMetadata, dto); } + + @Post('import/tutorial-data') + @HttpCode(200) + @ApiEndpoint({ + description: 'Import data from tutorial by path', + responses: [ + { + type: Object, + }, + ], + }) + async uploadFromTutorial( + @Body() dto: UploadImportFileByPathDto, + @ClientMetadataParam() clientMetadata: ClientMetadata, + ): Promise { + return this.service.uploadFromTutorial(clientMetadata, dto); + } } diff --git a/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts b/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts index 4fb410217c..5da086fedf 100644 --- a/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts +++ b/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts @@ -5,14 +5,19 @@ import { mockClientMetadata, mockDatabaseConnectionService, mockIORedisClient, - mockIORedisCluster, MockType + mockIORedisCluster, MockType, } from 'src/__mocks__'; import { MemoryStoredFile } from 'nestjs-form-data'; import { BulkActionSummary } from 'src/modules/bulk-actions/models/bulk-action-summary'; import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface'; import { BulkActionStatus, BulkActionType } from 'src/modules/bulk-actions/constants'; -import { NotFoundException } from '@nestjs/common'; +import { BadRequestException, NotFoundException } from '@nestjs/common'; import { BulkActionsAnalyticsService } from 'src/modules/bulk-actions/bulk-actions-analytics.service'; +import * as fs from 'fs-extra'; +import config from 'src/utils/config'; +import { join } from 'path'; + +const PATH_CONFIG = config.get('dir_path'); const generateNCommandsBuffer = (n: number) => Buffer.from( (new Array(n)).fill(1).map(() => ['set', ['foo', 'bar']]).join('\n'), @@ -55,12 +60,20 @@ const mockUploadImportFileDto = { } as unknown as MemoryStoredFile, }; +const mockUploadImportFileByPathDto = { + path: '/some/path', +}; + +jest.mock('fs-extra'); +const mockedFs = fs as jest.Mocked; + describe('BulkImportService', () => { let service: BulkImportService; let databaseConnectionService: MockType; let analytics: MockType; beforeEach(async () => { + jest.mock('fs-extra', () => mockedFs); jest.clearAllMocks(); const module: TestingModule = await Test.createTestingModule({ @@ -186,4 +199,79 @@ describe('BulkImportService', () => { } }); }); + + describe('uploadFromTutorial', () => { + let spy; + + beforeEach(() => { + spy = jest.spyOn(service as any, 'import'); + spy.mockResolvedValue(mockSummary); + mockedFs.readFile.mockResolvedValue(Buffer.from('set foo bar')); + }); + + it('should import file by path', async () => { + mockedFs.pathExists.mockImplementationOnce(async () => true); + + await service.uploadFromTutorial(mockClientMetadata, mockUploadImportFileByPathDto); + + expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, mockUploadImportFileByPathDto.path)); + }); + + it('should import file by path with static', async () => { + mockedFs.pathExists.mockImplementationOnce(async () => true); + + await service.uploadFromTutorial(mockClientMetadata, { path: '/static/guides/_data.file' }); + + expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, '/guides/_data.file')); + }); + + it('should normalize path before importing and not search for file outside home folder', async () => { + mockedFs.pathExists.mockImplementationOnce(async () => true); + + await service.uploadFromTutorial(mockClientMetadata, { + path: '/../../../danger', + }); + + expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, 'danger')); + }); + + it('should normalize path before importing and not search for file outside home folder (relative)', async () => { + mockedFs.pathExists.mockImplementationOnce(async () => true); + + await service.uploadFromTutorial(mockClientMetadata, { + path: '../../../danger', + }); + + expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, 'danger')); + }); + + it('should throw BadRequest when no file found', async () => { + mockedFs.pathExists.mockImplementationOnce(async () => false); + + try { + await service.uploadFromTutorial(mockClientMetadata, { + path: '../../../danger', + }); + fail(); + } catch (e) { + expect(e).toBeInstanceOf(BadRequestException); + expect(e.message).toEqual('Data file was not found'); + } + }); + + it('should throw BadRequest when file size is greater then 100MB', async () => { + mockedFs.pathExists.mockImplementationOnce(async () => true); + mockedFs.stat.mockImplementationOnce(async () => ({ size: 100 * 1024 * 1024 + 1 } as fs.Stats)); + + try { + await service.uploadFromTutorial(mockClientMetadata, { + path: '../../../danger', + }); + fail(); + } catch (e) { + expect(e).toBeInstanceOf(BadRequestException); + expect(e.message).toEqual('Maximum file size is 100MB'); + } + }); + }); }); diff --git a/redisinsight/api/src/modules/bulk-actions/bulk-import.service.ts b/redisinsight/api/src/modules/bulk-actions/bulk-import.service.ts index 022cfc71f9..98255228e0 100644 --- a/redisinsight/api/src/modules/bulk-actions/bulk-import.service.ts +++ b/redisinsight/api/src/modules/bulk-actions/bulk-import.service.ts @@ -1,4 +1,6 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { join, resolve } from 'path'; +import * as fs from 'fs-extra'; +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; import { Readable } from 'stream'; import * as readline from 'readline'; import { wrapHttpError } from 'src/common/utils'; @@ -10,8 +12,13 @@ import { BulkActionSummary } from 'src/modules/bulk-actions/models/bulk-action-s import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface'; import { BulkActionStatus, BulkActionType } from 'src/modules/bulk-actions/constants'; import { BulkActionsAnalyticsService } from 'src/modules/bulk-actions/bulk-actions-analytics.service'; +import { UploadImportFileByPathDto } from 'src/modules/bulk-actions/dto/upload-import-file-by-path.dto'; +import config from 'src/utils/config'; +import { MemoryStoredFile } from 'nestjs-form-data'; const BATCH_LIMIT = 10_000; +const PATH_CONFIG = config.get('dir_path'); +const SERVER_CONFIG = config.get('server'); @Injectable() export class BulkImportService { @@ -132,4 +139,47 @@ export class BulkImportService { throw wrapHttpError(e); } } + + /** + * Upload file from tutorial by path + * @param clientMetadata + * @param dto + */ + public async uploadFromTutorial( + clientMetadata: ClientMetadata, + dto: UploadImportFileByPathDto, + ): Promise { + try { + const staticPath = join(SERVER_CONFIG.base, SERVER_CONFIG.staticUri); + + let trimmedPath = dto.path; + if (dto.path.indexOf(staticPath) === 0) { + trimmedPath = dto.path.slice(staticPath.length); + } + + const resolvedPath = resolve( + '/', + trimmedPath, + ); + + const path = join(PATH_CONFIG.homedir, resolvedPath); + + if (!await fs.pathExists(path)) { + throw new BadRequestException('Data file was not found'); + } + + if ((await fs.stat(path))?.size > 100 * 1024 * 1024) { + throw new BadRequestException('Maximum file size is 100MB'); + } + + const buffer = await fs.readFile(path); + + return this.import(clientMetadata, { + file: { buffer } as MemoryStoredFile, + }); + } catch (e) { + this.logger.error('Unable to process an import file path from tutorial', e); + throw wrapHttpError(e); + } + } } diff --git a/redisinsight/api/src/modules/bulk-actions/dto/upload-import-file-by-path.dto.ts b/redisinsight/api/src/modules/bulk-actions/dto/upload-import-file-by-path.dto.ts new file mode 100644 index 0000000000..d9a73964fd --- /dev/null +++ b/redisinsight/api/src/modules/bulk-actions/dto/upload-import-file-by-path.dto.ts @@ -0,0 +1,12 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNotEmpty, IsString } from 'class-validator'; + +export class UploadImportFileByPathDto { + @ApiProperty({ + type: 'string', + description: 'Internal path to data file', + }) + @IsString() + @IsNotEmpty() + path: string; +} diff --git a/redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-import-tutorial_data.test.ts b/redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-import-tutorial_data.test.ts new file mode 100644 index 0000000000..0c4534dce3 --- /dev/null +++ b/redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-import-tutorial_data.test.ts @@ -0,0 +1,122 @@ +import { + expect, + describe, + it, + deps, + requirements, + validateApiCall, +} from '../deps'; +import { AdmZip, path } from '../../helpers/test'; +const { rte, request, server, constants } = deps; + +const endpoint = ( + id = constants.TEST_INSTANCE_ID, +) => request(server).post(`/${constants.API.DATABASES}/${id}/bulk-actions/import/tutorial-data`); + +const creatCustomTutorialsEndpoint = () => request(server).post(`/custom-tutorials`); + +const getZipArchive = () => { + const zipArchive = new AdmZip(); + + zipArchive.addFile('info.md', Buffer.from('# info.md', 'utf8')); + zipArchive.addFile('_data/data.txt', Buffer.from( + `set ${constants.TEST_STRING_KEY_1} bulkimport`, + 'utf8', + )); + + return zipArchive; +} + +describe('POST /databases/:id/bulk-actions/import/tutorial-data', () => { + requirements('!rte.sharedData', '!rte.bigData', 'rte.serverType=local') + + beforeEach(async () => await rte.data.truncate()); + + describe('Common', function () { + let tutorialId; + it('should import data', async () => { + expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.not.eq('bulkimport'); + + // create tutorial + const zip = getZipArchive(); + await validateApiCall({ + endpoint: creatCustomTutorialsEndpoint, + attach: ['file', zip.toBuffer(), 'a.zip'], + statusCode: 201, + checkFn: ({ body }) => { + tutorialId = body.id; + }, + }); + + await validateApiCall({ + endpoint, + data: { + path: path.join('/custom-tutorials', tutorialId, '_data/data.txt'), + }, + responseBody: { + id: 'empty', + databaseId: constants.TEST_INSTANCE_ID, + type: 'import', + summary: { processed: 1, succeed: 1, failed: 0, errors: [] }, + progress: null, + filter: null, + status: 'completed', + }, + checkFn: async ({ body }) => { + expect(body.duration).to.gt(0); + + expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.eq('bulkimport'); + }, + }); + }); + it('should import data with static path', async () => { + expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.not.eq('bulkimport'); + + // create tutorial + const zip = getZipArchive(); + await validateApiCall({ + endpoint: creatCustomTutorialsEndpoint, + attach: ['file', zip.toBuffer(), 'a.zip'], + statusCode: 201, + checkFn: ({ body }) => { + tutorialId = body.id; + }, + }); + + await validateApiCall({ + endpoint, + data: { + path: path.join('/static/custom-tutorials', tutorialId, '_data/data.txt'), + }, + responseBody: { + id: 'empty', + databaseId: constants.TEST_INSTANCE_ID, + type: 'import', + summary: { processed: 1, succeed: 1, failed: 0, errors: [] }, + progress: null, + filter: null, + status: 'completed', + }, + checkFn: async ({ body }) => { + expect(body.duration).to.gt(0); + + expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.eq('bulkimport'); + }, + }); + }); + it('should return BadRequest when path does not exists', async () => { + await validateApiCall({ + endpoint, + data: { + path: path.join('/custom-tutorials', tutorialId, '../../../../../_data/data.txt'), + }, + statusCode: 400, + responseBody: { + statusCode: 400, + message: 'Data file was not found', + error: 'Bad Request', + }, + }); + }); + }); +}); diff --git a/redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-upload.test.ts b/redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-import.test.ts similarity index 100% rename from redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-upload.test.ts rename to redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-import.test.ts diff --git a/redisinsight/api/test/api/custom-tutorials/POST-custom-tutorials.test.ts b/redisinsight/api/test/api/custom-tutorials/POST-custom-tutorials.test.ts index 983bd5499a..6fec8089e7 100644 --- a/redisinsight/api/test/api/custom-tutorials/POST-custom-tutorials.test.ts +++ b/redisinsight/api/test/api/custom-tutorials/POST-custom-tutorials.test.ts @@ -12,7 +12,7 @@ import { _, } from '../deps'; import { getBaseURL } from '../../helpers/server'; -const { server, request } = deps; +const { server, request, localDb } = deps; // create endpoint const creatEndpoint = () => request(server).post(`/custom-tutorials`); @@ -110,11 +110,12 @@ const globalManifest = { describe('POST /custom-tutorials', () => { requirements('rte.serverType=local'); - describe('Common', () => { - before(async () => { - await fsExtra.remove(customTutorialsFolder); - }); + before(async () => { + await fsExtra.remove(customTutorialsFolder); + await (await localDb.getRepository(localDb.repositories.CUSTOM_TUTORIAL)).clear(); + }); + describe('Common', () => { it('should import tutorial from file and generate _manifest.json', async () => { const zip = getZipArchive(); zip.writeZip(path.join(staticsFolder, 'test_no_manifest.zip'));