Feature/ri 4186 upload custom tutorials

redisinsight/api/config/default.ts

@@ -24,6 +24,7 @@ export default {
     logs: join(homedir, 'logs'),
     defaultPlugins: join(staticDir, 'plugins'),
     customPlugins: join(homedir, 'plugins'),
+    customTutorials: join(homedir, 'custom-tutorials'),
     pluginsAssets: join(staticDir, 'resources', 'plugins'),
     commands: join(homedir, 'commands'),
     defaultCommandsDir: join(defaultsDir, 'commands'),
@@ -45,6 +46,7 @@ export default {
     staticUri: '/static',
     guidesUri: '/static/guides',
     tutorialsUri: '/static/tutorials',
+    customTutorialsUri: '/static/custom-tutorials',
     contentUri: '/static/content',
     defaultPluginsUri: '/static/plugins',
     pluginsAssetsUri: '/static/resources/plugins',
@@ -95,6 +97,7 @@ export default {
   },
   analytics: {
     writeKey: process.env.SEGMENT_WRITE_KEY || 'SOURCE_WRITE_KEY',
+    flushInterval: parseInt(process.env.ANALYTICS_FLUSH_INTERVAL, 10) || 3000,
   },
   logger: {
     logLevel: process.env.LOG_LEVEL || 'info', // log level
@@ -108,14 +111,14 @@ export default {
   },
   guides: {
     updateUrl: process.env.GUIDES_UPDATE_URL
-      || 'https://github.com/RedisInsight/Guides/releases/download/release',
+      || 'https://github.com/RedisInsight/Guides/releases/download/2.x.x',
     zip: process.env.GUIDES_ZIP || dataZipFileName,
     buildInfo: process.env.GUIDES_CHECKSUM || buildInfoFileName,
     devMode: !!process.env.GUIDES_DEV_PATH,
   },
   tutorials: {
     updateUrl: process.env.TUTORIALS_UPDATE_URL
-      || 'https://github.com/RedisInsight/Tutorials/releases/download/release',
+      || 'https://github.com/RedisInsight/Tutorials/releases/download/2.x.x',
     zip: process.env.TUTORIALS_ZIP || dataZipFileName,
     buildInfo: process.env.TUTORIALS_CHECKSUM || buildInfoFileName,
     devMode: !!process.env.TUTORIALS_DEV_PATH,

redisinsight/api/config/ormconfig.ts

@@ -12,6 +12,7 @@ import { ClientCertificateEntity } from 'src/modules/certificate/entities/client
 import { DatabaseEntity } from 'src/modules/database/entities/database.entity';
 import { SshOptionsEntity } from 'src/modules/ssh/entities/ssh-options.entity';
 import { BrowserHistoryEntity } from 'src/modules/browser/entities/browser-history.entity';
+import { CustomTutorialEntity } from 'src/modules/custom-tutorial/entities/custom-tutorial.entity';
 import migrations from '../migration';
 import * as config from '../src/utils/config';
 
@@ -35,6 +36,7 @@ const ormConfig = {
     DatabaseAnalysisEntity,
     BrowserHistoryEntity,
     SshOptionsEntity,
+    CustomTutorialEntity,
   ],
   migrations,
 };

redisinsight/api/config/production.ts

@@ -12,6 +12,7 @@ export default {
     prevHomedir,
     logs: join(homedir, 'logs'),
     customPlugins: join(homedir, 'plugins'),
+    customTutorials: join(homedir, 'custom-tutorials'),
     commands: join(homedir, 'commands'),
     guides: process.env.GUIDES_DEV_PATH || join(homedir, 'guides'),
     tutorials: process.env.TUTORIALS_DEV_PATH || join(homedir, 'tutorials'),
@@ -24,6 +25,7 @@ export default {
   },
   analytics: {
     writeKey: process.env.SEGMENT_WRITE_KEY || 'lK5MNZgHbxj6vQwFgqZxygA0BiDQb32n',
+    flushInterval: parseInt(process.env.ANALYTICS_FLUSH_INTERVAL, 10) || 10000,
   },
   db: {
     database: join(homedir, 'redisinsight.db'),

redisinsight/api/config/staging.ts

@@ -12,6 +12,7 @@ export default {
     prevHomedir,
     logs: join(homedir, 'logs'),
     customPlugins: join(homedir, 'plugins'),
+    customTutorials: join(homedir, 'custom-tutorials'),
     commands: join(homedir, 'commands'),
     guides: process.env.GUIDES_DEV_PATH || join(homedir, 'guides'),
     tutorials: process.env.TUTORIALS_DEV_PATH || join(homedir, 'tutorials'),

redisinsight/api/migration/1677135091633-custom-tutorials.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class customTutorials1677135091633 implements MigrationInterface {
+    name = 'customTutorials1677135091633'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TABLE "custom_tutorials" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "link" varchar, "createdAt" datetime NOT NULL DEFAULT (datetime('now')))`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE "custom_tutorials"`);
+    }
+
+}

redisinsight/api/migration/index.ts

@@ -28,6 +28,7 @@ import { browserHistory1674539211397 } from './1674539211397-browser-history';
 import { databaseAnalysisRecommendations1674660306971 } from './1674660306971-database-analysis-recommendations';
 import { databaseTimeout1675398140189 } from './1675398140189-database-timeout';
 import { databaseCompressor1678182722874 } from './1678182722874-database-compressor';
+import { customTutorials1677135091633 } from './1677135091633-custom-tutorials';
 
 export default [
   initialMigration1614164490968,
@@ -60,4 +61,5 @@ export default [
   browserHistory1674539211397,
   databaseTimeout1675398140189,
   databaseCompressor1678182722874,
+  customTutorials1677135091633,
 ];

redisinsight/api/package.json

@@ -65,6 +65,7 @@
     "lodash": "^4.17.20",
     "nest-router": "^1.0.9",
     "nest-winston": "^1.4.0",
+    "nestjs-form-data": "^1.8.7",
     "node-version-compare": "^1.0.3",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.5.6",
@@ -83,6 +84,7 @@
     "@nestjs/cli": "^9.1.2",
     "@nestjs/schematics": "^9.0.3",
     "@nestjs/testing": "^9.0.11",
+    "@types/adm-zip": "^0.5.0",
     "@types/axios": "^0.14.0",
     "@types/express": "^4.17.3",
     "@types/jest": "^26.0.15",
@@ -108,6 +110,7 @@
     "mocha": "^8.4.0",
     "mocha-junit-reporter": "^2.0.0",
     "mocha-multi-reporters": "^1.5.1",
+    "nock": "^13.3.0",
     "nyc": "^15.1.0",
     "object-diff": "^0.0.4",
     "rimraf": "^3.0.2",

redisinsight/api/src/__mocks__/custom-tutorial.ts

@@ -0,0 +1,180 @@
+import { CustomTutorial, CustomTutorialActions } from 'src/modules/custom-tutorial/models/custom-tutorial';
+import { CustomTutorialEntity } from 'src/modules/custom-tutorial/entities/custom-tutorial.entity';
+import { CustomTutorialManifestType } from 'src/modules/custom-tutorial/models/custom-tutorial.manifest';
+import { MemoryStoredFile } from 'nestjs-form-data';
+import { UploadCustomTutorialDto } from 'src/modules/custom-tutorial/dto/upload.custom-tutorial.dto';
+import AdmZip from 'adm-zip';
+
+export const mockCustomTutorialId = 'a77b23c1-7816-4ea4-b61f-d37795a0f805-ct-id';
+
+export const mockCustomTutorialId2 = 'a77b23c1-7816-4ea4-b61f-d37795a0f805-ct-id-2';
+
+export const mockCustomTutorialTmpPath = '/tmp/path';
+
+export const mockCustomTutorialsHttpLink = 'https://somesime.com/archive.zip';
+
+export const mockCustomTutorial = Object.assign(new CustomTutorial(), {
+  id: mockCustomTutorialId,
+  name: 'custom tutorial',
+  createdAt: new Date(),
+});
+
+export const mockCustomTutorialEntity = Object.assign(new CustomTutorialEntity(), {
+  ...mockCustomTutorial,
+});
+
+export const mockCustomTutorial2 = Object.assign(new CustomTutorial(), {
+  id: mockCustomTutorialId2,
+  name: 'custom tutorial 2',
+  link: mockCustomTutorialsHttpLink,
+  createdAt: new Date(),
+});
+
+export const mockCustomTutorialZipFile = Object.assign(new MemoryStoredFile(), {
+  size: 100,
+  buffer: Buffer.from('zip-content', 'utf8'),
+});
+
+export const mockCustomTutorialZipFileAxiosResponse = {
+  data: mockCustomTutorialZipFile.buffer,
+};
+
+export const mockCustomTutorialAdmZipEntry = {
+  entryName: 'somefolder/info.md',
+} as AdmZip.IZipEntry;
+
+export const mockCustomTutorialMacosxAdmZipEntry = {
+  entryName: '__MACOSX/info.md',
+} as AdmZip.IZipEntry;
+
+export const mockUploadCustomTutorialDto = Object.assign(new UploadCustomTutorialDto(), {
+  file: mockCustomTutorialZipFile,
+});
+
+export const mockUploadCustomTutorialExternalLinkDto = Object.assign(new UploadCustomTutorialDto(), {
+  link: mockCustomTutorialsHttpLink,
+});
+
+export const mockCustomTutorialManifestJson = {
+  type: CustomTutorialManifestType.Group,
+  id: mockCustomTutorialId,
+  label: mockCustomTutorial.name,
+  children: [
+    {
+      type: 'group',
+      id: 'ct-folder-1',
+      label: 'ct-folder-1',
+      children: [
+        {
+          type: CustomTutorialManifestType.Group,
+          id: 'ct-sub-folder-1',
+          label: 'ct-sub-folder-1',
+          children: [
+            {
+              type: CustomTutorialManifestType.InternalLink,
+              id: 'introduction',
+              label: 'introduction',
+              args: {
+                path: '/ct-folder-1/ct-sub-folder-1/introduction.md',
+              },
+            },
+            {
+              type: CustomTutorialManifestType.InternalLink,
+              id: 'working-with-hashes',
+              label: 'working-with-hashes',
+              args: {
+                path: '/ct-folder-1/ct-sub-folder-1/working-with-hashes.md',
+              },
+            },
+          ],
+        },
+        {
+          type: CustomTutorialManifestType.Group,
+          id: 'ct-sub-folder-2',
+          label: 'ct-sub-folder-2',
+          children: [
+            {
+              type: CustomTutorialManifestType.InternalLink,
+              id: 'introduction',
+              label: 'introduction',
+              args: {
+                path: '/ct-folder-1/ct-sub-folder-2/introduction.md',
+              },
+            },
+            {
+              type: CustomTutorialManifestType.InternalLink,
+              id: 'working-with-graphs',
+              label: 'working-with-graphs',
+              args: {
+                path: '/ct-folder-1/ct-sub-folder-2/working-with-graphs.md',
+              },
+            },
+          ],
+        },
+      ],
+    },
+  ],
+};
+
+export const mockCustomTutorialManifest = {
+  ...mockCustomTutorialManifestJson,
+  type: CustomTutorialManifestType.Group,
+  id: mockCustomTutorialId,
+  label: mockCustomTutorial.name,
+  _actions: mockCustomTutorial.actions,
+  _path: mockCustomTutorial.path,
+};
+
+export const mockCustomTutorialManifest2 = {
+  type: CustomTutorialManifestType.Group,
+  id: mockCustomTutorialId2,
+  label: mockCustomTutorial2.name,
+  _actions: mockCustomTutorial2.actions,
+  _path: mockCustomTutorial2.path,
+  children: [mockCustomTutorialManifestJson],
+};
+
+export const globalCustomTutorialManifest = {
+  type: CustomTutorialManifestType.Group,
+  id: 'custom-tutorials',
+  label: 'MY TUTORIALS',
+  _actions: [CustomTutorialActions.CREATE],
+  args: {
+    withBorder: true,
+    initialIsOpen: true,
+  },
+  children: [
+    mockCustomTutorialManifest,
+    mockCustomTutorialManifest2,
+  ],
+};
+
+export const mockCustomTutorialFsProvider = jest.fn(() => ({
+  unzipFromMemoryStoredFile: jest.fn().mockResolvedValue(mockCustomTutorialTmpPath),
+  unzipFromExternalLink: jest.fn().mockResolvedValue(mockCustomTutorialTmpPath),
+  unzipToTmpFolder: jest.fn().mockResolvedValue(mockCustomTutorialTmpPath),
+  moveFolder: jest.fn(),
+  removeFolder: jest.fn(),
+}));
+
+export const mockCustomTutorialManifestProvider = jest.fn(() => ({
+  getOriginalManifestJson: jest.fn().mockResolvedValue(mockCustomTutorialManifestJson),
+  getManifestJson: jest.fn().mockResolvedValue(mockCustomTutorialManifestJson),
+  generateTutorialManifest: jest.fn().mockResolvedValue(mockCustomTutorialManifest),
+  isOriginalManifestExists: jest.fn().mockResolvedValue(true),
+}));
+
+export const mockCustomTutorialRepository = jest.fn(() => ({
+  get: jest.fn().mockResolvedValue(mockCustomTutorial),
+  create: jest.fn().mockResolvedValue(mockCustomTutorial),
+  delete: jest.fn(),
+  list: jest.fn().mockResolvedValue([
+    mockCustomTutorial,
+    mockCustomTutorial2,
+  ]),
+}));
+
+export const mockCustomTutorialAnalytics = jest.fn(() => ({
+  sendImportSucceeded: jest.fn(),
+  sendImportFailed: jest.fn(),
+}));

redisinsight/api/src/__mocks__/index.ts

@@ -11,6 +11,7 @@ export * from './analytics';
 export * from './profiler';
 export * from './user';
 export * from './databases';
+export * from './custom-tutorial';
 export * from './autodiscovery';
 export * from './redis';
 export * from './server';

redisinsight/api/src/app.module.ts

@@ -21,6 +21,7 @@ import { CoreModule } from 'src/core.module';
 import { AutodiscoveryModule } from 'src/modules/autodiscovery/autodiscovery.module';
 import { DatabaseImportModule } from 'src/modules/database-import/database-import.module';
 import { DummyAuthMiddleware } from 'src/common/middlewares/dummy-auth.middleware';
+import { CustomTutorialModule } from 'src/modules/custom-tutorial/custom-tutorial.module';
 import { BrowserModule } from './modules/browser/browser.module';
 import { RedisEnterpriseModule } from './modules/redis-enterprise/redis-enterprise.module';
 import { RedisSentinelModule } from './modules/redis-sentinel/redis-sentinel.module';
@@ -53,6 +54,7 @@ const PATH_CONFIG = config.get('dir_path');
     NotificationModule,
     BulkActionsModule,
     ClusterMonitorModule,
+    CustomTutorialModule.register(),
     DatabaseAnalysisModule,
     DatabaseImportModule,
     ...(SERVER_CONFIG.staticContent

redisinsight/api/src/common/utils/errors.util.ts

@@ -0,0 +1,9 @@
+import { HttpException, InternalServerErrorException } from '@nestjs/common';
+
+export const wrapHttpError = (error: Error, message?: string) => {
+  if (error instanceof HttpException) {
+    return error;
+  }
+
+  return new InternalServerErrorException(error.message || message);
+};

redisinsight/api/src/common/utils/index.ts

@@ -1 +1,2 @@
 export * from './certificate-import.util';
+export * from './errors.util';

redisinsight/api/src/constants/error-messages.ts

@@ -7,6 +7,8 @@ export default {
   PROFILER_LOG_FILE_NOT_FOUND: 'Profiler log file was not found.',
   CONSUMER_GROUP_NOT_FOUND: 'Consumer Group with such name was not found.',
   PLUGIN_STATE_NOT_FOUND: 'Plugin state was not found.',
+  CUSTOM_TUTORIAL_NOT_FOUND: 'Custom Tutorial was not found.',
+  CUSTOM_TUTORIAL_UNABLE_TO_FETCH_FROM_EXTERNAL: 'Unable fetch zip file from external source.',
   UNDEFINED_INSTANCE_ID: 'Undefined redis database instance id.',
   NO_CONNECTION_TO_REDIS_DB: 'No connection to the Redis Database.',
   WRONG_DATABASE_TYPE: 'Wrong database type.',

redisinsight/api/src/constants/telemetry-events.ts

@@ -44,6 +44,9 @@ export enum TelemetryEvents {
   WorkbenchCommandExecuted = 'WORKBENCH_COMMAND_EXECUTED',
   WorkbenchCommandErrorReceived = 'WORKBENCH_COMMAND_ERROR_RECEIVED',
   WorkbenchCommandDeleted = 'WORKBENCH_COMMAND_DELETE_COMMAND',
+  // Custom tutorials
+  WorkbenchEnablementAreaImportSucceeded = 'WORKBENCH_ENABLEMENT_AREA_IMPORT_SUCCEEDED',
+  WorkbenchEnablementAreaImportFailed = 'WORKBENCH_ENABLEMENT_AREA_IMPORT_FAILED',
 
   // Profiler
   ProfilerLogDownloaded = 'PROFILER_LOG_DOWNLOADED',

redisinsight/api/src/core.module.ts

@@ -7,6 +7,7 @@ import { EventEmitterModule } from '@nestjs/event-emitter';
 import { RedisModule } from 'src/modules/redis/redis.module';
 import { AnalyticsModule } from 'src/modules/analytics/analytics.module';
 import { SshModule } from 'src/modules/ssh/ssh.module';
+import { NestjsFormDataModule } from 'nestjs-form-data';
 
 @Global()
 @Module({
@@ -19,6 +20,7 @@ import { SshModule } from 'src/modules/ssh/ssh.module';
     DatabaseModule.register(),
     RedisModule,
     SshModule,
+    NestjsFormDataModule,
   ],
   exports: [
     EncryptionModule,
@@ -27,6 +29,7 @@ import { SshModule } from 'src/modules/ssh/ssh.module';
     DatabaseModule,
     RedisModule,
     SshModule,
+    NestjsFormDataModule,
   ],
 })
 export class CoreModule {}