#RI-4145 - decompress SNAPPY

redisinsight/api/migration/1678182722874-database-compressor.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class databaseCompressor1678182722874 implements MigrationInterface {
+    name = 'databaseCompressor1678182722874'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TABLE "temporary_database_instance" ("id" varchar PRIMARY KEY NOT NULL, "host" varchar NOT NULL, "port" integer NOT NULL, "name" varchar NOT NULL, "username" varchar, "password" varchar, "tls" boolean, "verifyServerCert" boolean, "lastConnection" datetime, "caCertId" varchar, "clientCertId" varchar, "connectionType" varchar NOT NULL DEFAULT ('STANDALONE'), "nodes" varchar DEFAULT ('[]'), "nameFromProvider" varchar, "sentinelMasterName" varchar, "sentinelMasterUsername" varchar, "sentinelMasterPassword" varchar, "provider" varchar DEFAULT ('UNKNOWN'), "modules" varchar NOT NULL DEFAULT ('[]'), "db" integer, "encryption" varchar, "tlsServername" varchar, "new" boolean, "ssh" boolean, "timeout" integer, "compressor" varchar NOT NULL DEFAULT ('NONE'), CONSTRAINT "FK_d1bc747b5938e22b4b708d8e9a5" FOREIGN KEY ("caCertId") REFERENCES "ca_certificate" ("id") ON DELETE SET NULL ON UPDATE NO ACTION, CONSTRAINT "FK_3b9b625266c00feb2d66a9f36e4" FOREIGN KEY ("clientCertId") REFERENCES "client_certificate" ("id") ON DELETE SET NULL ON UPDATE NO ACTION)`);
+        await queryRunner.query(`INSERT INTO "temporary_database_instance"("id", "host", "port", "name", "username", "password", "tls", "verifyServerCert", "lastConnection", "caCertId", "clientCertId", "connectionType", "nodes", "nameFromProvider", "sentinelMasterName", "sentinelMasterUsername", "sentinelMasterPassword", "provider", "modules", "db", "encryption", "tlsServername", "new", "ssh", "timeout") SELECT "id", "host", "port", "name", "username", "password", "tls", "verifyServerCert", "lastConnection", "caCertId", "clientCertId", "connectionType", "nodes", "nameFromProvider", "sentinelMasterName", "sentinelMasterUsername", "sentinelMasterPassword", "provider", "modules", "db", "encryption", "tlsServername", "new", "ssh", "timeout" FROM "database_instance"`);
+        await queryRunner.query(`DROP TABLE "database_instance"`);
+        await queryRunner.query(`ALTER TABLE "temporary_database_instance" RENAME TO "database_instance"`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "database_instance" RENAME TO "temporary_database_instance"`);
+        await queryRunner.query(`CREATE TABLE "database_instance" ("id" varchar PRIMARY KEY NOT NULL, "host" varchar NOT NULL, "port" integer NOT NULL, "name" varchar NOT NULL, "username" varchar, "password" varchar, "tls" boolean, "verifyServerCert" boolean, "lastConnection" datetime, "caCertId" varchar, "clientCertId" varchar, "connectionType" varchar NOT NULL DEFAULT ('STANDALONE'), "nodes" varchar DEFAULT ('[]'), "nameFromProvider" varchar, "sentinelMasterName" varchar, "sentinelMasterUsername" varchar, "sentinelMasterPassword" varchar, "provider" varchar DEFAULT ('UNKNOWN'), "modules" varchar NOT NULL DEFAULT ('[]'), "db" integer, "encryption" varchar, "tlsServername" varchar, "new" boolean, "ssh" boolean, "timeout" integer, CONSTRAINT "FK_d1bc747b5938e22b4b708d8e9a5" FOREIGN KEY ("caCertId") REFERENCES "ca_certificate" ("id") ON DELETE SET NULL ON UPDATE NO ACTION, CONSTRAINT "FK_3b9b625266c00feb2d66a9f36e4" FOREIGN KEY ("clientCertId") REFERENCES "client_certificate" ("id") ON DELETE SET NULL ON UPDATE NO ACTION)`);
+        await queryRunner.query(`INSERT INTO "database_instance"("id", "host", "port", "name", "username", "password", "tls", "verifyServerCert", "lastConnection", "caCertId", "clientCertId", "connectionType", "nodes", "nameFromProvider", "sentinelMasterName", "sentinelMasterUsername", "sentinelMasterPassword", "provider", "modules", "db", "encryption", "tlsServername", "new", "ssh", "timeout") SELECT "id", "host", "port", "name", "username", "password", "tls", "verifyServerCert", "lastConnection", "caCertId", "clientCertId", "connectionType", "nodes", "nameFromProvider", "sentinelMasterName", "sentinelMasterUsername", "sentinelMasterPassword", "provider", "modules", "db", "encryption", "tlsServername", "new", "ssh", "timeout" FROM "temporary_database_instance"`);
+        await queryRunner.query(`DROP TABLE "temporary_database_instance"`);
+    }
+
+}

redisinsight/api/migration/index.ts

@@ -27,6 +27,7 @@ import { workbenchAndAnalysisDbIndex1673934231410 } from './1673934231410-workbe
 import { browserHistory1674539211397 } from './1674539211397-browser-history';
 import { databaseAnalysisRecommendations1674660306971 } from './1674660306971-database-analysis-recommendations';
 import { databaseTimeout1675398140189 } from './1675398140189-database-timeout';
+import { databaseCompressor1678182722874 } from './1678182722874-database-compressor';
 
 export default [
   initialMigration1614164490968,
@@ -58,4 +59,5 @@ export default [
   databaseAnalysisRecommendations1674660306971,
   browserHistory1674539211397,
   databaseTimeout1675398140189,
+  databaseCompressor1678182722874,
 ];

redisinsight/api/src/__mocks__/databases.ts

@@ -1,7 +1,7 @@
 import { Database } from 'src/modules/database/models/database';
 import { mockCaCertificate, mockClientCertificate } from 'src/__mocks__/certificates';
 import { SentinelMaster } from 'src/modules/redis-sentinel/models/sentinel-master';
-import { ConnectionType, DatabaseEntity } from 'src/modules/database/entities/database.entity';
+import { Compressor, ConnectionType, DatabaseEntity } from 'src/modules/database/entities/database.entity';
 import { EncryptionStrategy } from 'src/modules/encryption/models';
 import { mockIORedisClient } from 'src/__mocks__/redis';
 import { mockSentinelMasterDto } from 'src/__mocks__/redis-sentinel';
@@ -34,6 +34,7 @@ export const mockDatabase = Object.assign(new Database(), {
   connectionType: ConnectionType.STANDALONE,
   timeout: 30_000,
   new: false,
+  compressor: Compressor.NONE,
 });
 
 export const mockDatabaseEntity = Object.assign(new DatabaseEntity(), {

redisinsight/api/src/modules/database/database.analytics.spec.ts

@@ -104,6 +104,7 @@ describe('DatabaseAnalytics', () => {
           numberOfModules: 0,
           timeout: mockDatabaseWithTlsAuth.timeout / 1_000, // milliseconds to seconds
           databaseIndex: 0,
+          useDecompression: mockDatabaseWithTlsAuth.compressor,
           ...DEFAULT_REDIS_MODULES_SUMMARY,
         },
       );
@@ -134,6 +135,7 @@ describe('DatabaseAnalytics', () => {
           numberOfModules: 0,
           timeout: mockDatabaseWithTlsAuth.timeout / 1_000, // milliseconds to seconds
           databaseIndex: 0,
+          useDecompression: mockDatabaseWithTlsAuth.compressor,
           ...DEFAULT_REDIS_MODULES_SUMMARY,
         },
       );
@@ -166,6 +168,7 @@ describe('DatabaseAnalytics', () => {
           numberOfModules: 2,
           timeout: mockDatabaseWithTlsAuth.timeout / 1_000, // milliseconds to seconds
           databaseIndex: 0,
+          useDecompression: mockDatabaseWithTlsAuth.compressor,
           ...DEFAULT_REDIS_MODULES_SUMMARY,
           RediSearch: {
             loaded: true,
@@ -204,6 +207,7 @@ describe('DatabaseAnalytics', () => {
           numberOfModules: 2,
           timeout: mockDatabaseWithTlsAuth.timeout / 1_000, // milliseconds to seconds
           databaseIndex: 2,
+          useDecompression: mockDatabaseWithTlsAuth.compressor,
           ...DEFAULT_REDIS_MODULES_SUMMARY,
           RediSearch: {
             loaded: true,
@@ -240,6 +244,7 @@ describe('DatabaseAnalytics', () => {
           useSNI: 'enabled',
           useSSH: 'disabled',
           timeout: mockDatabaseWithTlsAuth.timeout / 1_000, // milliseconds to seconds
+          useDecompression: mockDatabaseWithTlsAuth.compressor,
           previousValues: {
             connectionType: prev.connectionType,
             provider: prev.provider,
@@ -275,6 +280,7 @@ describe('DatabaseAnalytics', () => {
           useSNI: 'enabled',
           useSSH: 'disabled',
           timeout: mockDatabaseWithTlsAuth.timeout / 1_000, // milliseconds to seconds
+          useDecompression: mockDatabaseWithTlsAuth.compressor,
           previousValues: {
             connectionType: prev.connectionType,
             provider: prev.provider,

redisinsight/api/src/modules/database/database.analytics.ts

@@ -67,6 +67,7 @@ export class DatabaseAnalytics extends TelemetryBaseService {
           numberOfModules: instance.modules?.length || 0,
           timeout: instance.timeout / 1_000, // milliseconds to seconds
           databaseIndex: instance.db || 0,
+          useDecompression: instance.compressor || null,
           ...modulesSummary,
         },
       );
@@ -100,6 +101,7 @@ export class DatabaseAnalytics extends TelemetryBaseService {
             useSNI: cur?.tlsServername ? 'enabled' : 'disabled',
             useSSH: cur?.ssh ? 'enabled' : 'disabled',
             timeout: cur?.timeout / 1_000, // milliseconds to seconds
+            useDecompression: cur?.compressor || null,
             previousValues: {
               connectionType: prev.connectionType,
               provider: prev.provider,

redisinsight/api/src/modules/database/dto/create.database.dto.ts

@@ -23,7 +23,7 @@ import { sshOptionsTransformer } from 'src/modules/ssh/transformers/ssh-options.
 )
 export class CreateDatabaseDto extends PickType(Database, [
   'host', 'port', 'name', 'db', 'username', 'password', 'timeout', 'nameFromProvider', 'provider',
-  'tls', 'tlsServername', 'verifyServerCert', 'sentinelMaster', 'ssh',
+  'tls', 'tlsServername', 'verifyServerCert', 'sentinelMaster', 'ssh', 'compressor',
 ] as const) {
   @ApiPropertyOptional({
     description: 'CA Certificate',

redisinsight/api/src/modules/database/dto/update.database.dto.ts

@@ -1,6 +1,7 @@
 import { ApiPropertyOptional, getSchemaPath } from '@nestjs/swagger';
 import {
   IsBoolean,
+  IsEnum,
   IsInt, IsNotEmpty, IsNotEmptyObject, IsOptional, IsString, MaxLength, Min, ValidateIf, ValidateNested,
 } from 'class-validator';
 import { CreateCaCertificateDto } from 'src/modules/certificate/dto/create.ca-certificate.dto';
@@ -16,6 +17,7 @@ import { CreateDatabaseDto } from 'src/modules/database/dto/create.database.dto'
 import { CreateBasicSshOptionsDto } from 'src/modules/ssh/dto/create.basic-ssh-options.dto';
 import { CreateCertSshOptionsDto } from 'src/modules/ssh/dto/create.cert-ssh-options.dto';
 import { sshOptionsTransformer } from 'src/modules/ssh/transformers/ssh-options.transformer';
+import { Compressor } from '../entities/database.entity';
 
 export class UpdateDatabaseDto extends CreateDatabaseDto {
   @ValidateIf((object, value) => value !== undefined)
@@ -157,4 +159,14 @@ export class UpdateDatabaseDto extends CreateDatabaseDto {
   @ValidateNested()
   @Default(null)
   sshOptions?: CreateBasicSshOptionsDto | CreateCertSshOptionsDto;
+
+  @ApiPropertyOptional({
+    description: 'Database compressor',
+    default: Compressor.NONE,
+    enum: Compressor,
+  })
+  @Expose()
+  @IsEnum(Compressor)
+  @IsOptional()
+  compressor?: Compressor = Compressor.NONE;
 }

redisinsight/api/src/modules/database/entities/database.entity.ts

@@ -25,6 +25,14 @@ export enum ConnectionType {
   NOT_CONNECTED = 'NOT CONNECTED',
 }
 
+export enum Compressor {
+  NONE = 'NONE',
+  GZIP = 'GZIP',
+  ZSTD = 'ZSTD',
+  LZ4 = 'LZ4',
+  SNAPPY = 'SNAPPY',
+}
+
 @Entity('database_instance')
 export class DatabaseEntity {
   @Expose()
@@ -184,4 +192,11 @@ export class DatabaseEntity {
   )
   @Type(() => SshOptionsEntity)
   sshOptions: SshOptionsEntity;
+
+  @Expose()
+  @Column({
+    nullable: false,
+    default: Compressor.NONE,
+  })
+  compressor: Compressor;
 }

redisinsight/api/src/modules/database/models/database.ts

@@ -3,7 +3,7 @@ import { Expose, Type } from 'class-transformer';
 import config from 'src/utils/config';
 import { CaCertificate } from 'src/modules/certificate/models/ca-certificate';
 import { ClientCertificate } from 'src/modules/certificate/models/client-certificate';
-import { ConnectionType, HostingProvider } from 'src/modules/database/entities/database.entity';
+import { Compressor, ConnectionType, HostingProvider } from 'src/modules/database/entities/database.entity';
 import {
   IsBoolean, IsEnum,
   IsInt,
@@ -255,4 +255,14 @@ export class Database {
   @Type(() => SshOptions)
   @ValidateNested()
   sshOptions?: SshOptions;
+
+  @ApiPropertyOptional({
+    description: 'Database compressor',
+    default: Compressor.NONE,
+    enum: Compressor,
+  })
+  @Expose()
+  @IsEnum(Compressor)
+  @IsOptional()
+  compressor?: Compressor = Compressor.NONE;
 }

redisinsight/api/test/api/database/GET-databases.test.ts

@@ -14,6 +14,7 @@ const responseSchema = Joi.array().items(Joi.object().keys({
   provider: Joi.string().required(),
   new: Joi.boolean().allow(null).required(),
   timeout: Joi.number().integer().required(),
+  compressor: Joi.string().valid('NONE', 'LZ4', 'GZIP', 'ZSTD', 'SNAPPY').allow(null),
   connectionType: Joi.string().valid('STANDALONE', 'SENTINEL', 'CLUSTER', 'NOT CONNECTED').required(),
   lastConnection: Joi.string().isoDate().allow(null).required(),
   modules: Joi.array().items(Joi.object().keys({

redisinsight/api/test/api/database/PATCH-databases-id.test.ts

@@ -24,6 +24,7 @@ const dataSchema = Joi.object({
   username: Joi.string().allow(null),
   password: Joi.string().allow(null),
   timeout: Joi.number().integer().allow(null),
+  compressor: Joi.string().valid('NONE', 'LZ4', 'GZIP', 'ZSTD', 'SNAPPY').allow(null),
   tls: Joi.boolean().allow(null),
   tlsServername: Joi.string().allow(null),
   verifyServerCert: Joi.boolean().allow(null),
@@ -176,6 +177,7 @@ describe(`PATCH /databases/:id`, () => {
             host: constants.TEST_REDIS_HOST,
             port: constants.TEST_REDIS_PORT,
             timeout: constants.TEST_REDIS_TIMEOUT,
+            compressor: constants.TEST_REDIS_COMPRESSOR,
             username: null,
             password: null,
             connectionType: constants.STANDALONE,
@@ -186,7 +188,7 @@ describe(`PATCH /databases/:id`, () => {
           after: async () => {
             newDatabase = await localDb.getInstanceById(constants.TEST_INSTANCE_ID_3);
             expect(newDatabase).to.contain({
-              ..._.omit(oldDatabase, ['modules', 'provider', 'lastConnection', 'new', 'timeout']),
+              ..._.omit(oldDatabase, ['modules', 'provider', 'lastConnection', 'new', 'timeout', 'compressor']),
               host: constants.TEST_REDIS_HOST,
               port: constants.TEST_REDIS_PORT,
             });

redisinsight/api/test/api/database/POST-databases-test.test.ts

@@ -22,6 +22,7 @@ const dataSchema = Joi.object({
   username: Joi.string().allow(null),
   password: Joi.string().allow(null),
   timeout: Joi.number().integer().allow(null),
+  compressor: Joi.string().valid('NONE', 'LZ4', 'GZIP', 'ZSTD', 'SNAPPY').allow(null),
   tls: Joi.boolean().allow(null),
   tlsServername: Joi.string().allow(null),
   verifyServerCert: Joi.boolean().allow(null),

redisinsight/api/test/api/database/POST-databases.test.ts

@@ -23,6 +23,7 @@ const dataSchema = Joi.object({
   username: Joi.string().allow(null),
   password: Joi.string().allow(null),
   timeout: Joi.number().integer().allow(null),
+  compressor: Joi.string().valid('NONE', 'LZ4', 'GZIP', 'ZSTD', 'SNAPPY').allow(null),
   tls: Joi.boolean().allow(null),
   tlsServername: Joi.string().allow(null),
   verifyServerCert: Joi.boolean().allow(null),
@@ -55,6 +56,7 @@ const baseDatabaseData = {
   host: constants.TEST_REDIS_HOST,
   port: constants.TEST_REDIS_PORT,
   timeout: constants.TEST_REDIS_TIMEOUT,
+  compressor: constants.TEST_REDIS_COMPRESSOR,
   username: constants.TEST_REDIS_USER || undefined,
   password: constants.TEST_REDIS_PASSWORD || undefined,
 }

redisinsight/api/test/api/database/PUT-databases-id.test.ts

@@ -24,6 +24,7 @@ const dataSchema = Joi.object({
   username: Joi.string().allow(null),
   password: Joi.string().allow(null),
   timeout: Joi.number().integer().allow(null),
+  compressor: Joi.string().valid('NONE', 'LZ4', 'GZIP', 'ZSTD', 'SNAPPY').allow(null),
   tls: Joi.boolean().allow(null),
   tlsServername: Joi.string().allow(null),
   verifyServerCert: Joi.boolean().allow(null),
@@ -42,6 +43,7 @@ const baseDatabaseData = {
   host: constants.TEST_REDIS_HOST,
   port: constants.TEST_REDIS_PORT,
   timeout: constants.TEST_REDIS_TIMEOUT,
+  compressor: constants.TEST_REDIS_COMPRESSOR,
   username: constants.TEST_REDIS_USER || undefined,
   password: constants.TEST_REDIS_PASSWORD || undefined,
 }

redisinsight/api/test/api/database/constants.ts

@@ -11,6 +11,7 @@ export const databaseSchema = Joi.object().keys({
   username: Joi.string().allow(null),
   password: Joi.string().allow(null),
   timeout: Joi.number().integer().required(),
+  compressor: Joi.string().valid('NONE', 'LZ4', 'GZIP', 'ZSTD', 'SNAPPY').required(),
   nameFromProvider: Joi.string().allow(null),
   lastConnection: Joi.string().isoDate().allow(null),
   provider: Joi.string().valid('LOCALHOST', 'UNKNOWN', 'RE_CLOUD', 'RE_CLUSTER'),

redisinsight/api/test/helpers/constants.ts

@@ -3,6 +3,7 @@ import * as path from 'path';
 import { randomBytes } from 'crypto';
 import { getASCIISafeStringFromBuffer, getBufferFromSafeASCIIString } from "src/utils/cli-helper";
 import { RECOMMENDATION_NAMES } from 'src/constants';
+import { Compressor } from 'src/modules/database/entities/database.entity';
 
 const API = {
   DATABASES: 'databases',
@@ -66,6 +67,7 @@ export const constants = {
   TEST_REDIS_HOST: process.env.TEST_REDIS_HOST || 'localhost',
   TEST_REDIS_PORT: parseInt(process.env.TEST_REDIS_PORT) || 6379,
   TEST_REDIS_TIMEOUT: 30_000,
+  TEST_REDIS_COMPRESSOR: Compressor.NONE,
   TEST_REDIS_DB_INDEX: 7,
   TEST_REDIS_USER: process.env.TEST_REDIS_USER,
   TEST_REDIS_PASSWORD: process.env.TEST_REDIS_PASSWORD,

redisinsight/ui/src/pages/browser/components/hash-details/HashDetails.tsx

@@ -55,7 +55,7 @@ import {
   stringToSerializedBufferFormat
 } from 'uiSrc/utils'
 import { stringToBuffer } from 'uiSrc/utils/formatters/bufferFormatters'
-import { decompressingBuffer, getCompressor } from 'uiSrc/utils/decompressors'
+import { decompressingBuffer } from 'uiSrc/utils/decompressors'
 import { AddFieldsToHashDto, GetHashFieldsResponse, HashFieldDto, } from 'apiSrc/modules/browser/dto/hash.dto'
 
 import PopoverDelete from '../popover-delete/PopoverDelete'
@@ -88,7 +88,7 @@ const HashDetails = (props: Props) => {
   } = useSelector(hashDataSelector)
   const { loading } = useSelector(hashSelector)
   const { viewType } = useSelector(keysSelector)
-  const { id: instanceId } = useSelector(connectedInstanceSelector)
+  const { id: instanceId, compressor = null } = useSelector(connectedInstanceSelector)
   const { viewFormat: viewFormatProp } = useSelector(selectedKeySelector)
   const { name: key, length } = useSelector(selectedKeyDataSelector) ?? { name: '' }
   const { loading: updateLoading } = useSelector(updateHashValueStateSelector)
@@ -295,7 +295,7 @@ const HashDetails = (props: Props) => {
       className: 'value-table-separate-border',
       headerClassName: 'value-table-separate-border',
       render: (_name: string, { field: fieldItem }: HashFieldDto, expanded?: boolean) => {
-        const { value: decompressedItem } = decompressingBuffer(fieldItem)
+        const { value: decompressedItem } = decompressingBuffer(fieldItem, compressor)
         const field = bufferToString(fieldItem) || ''
         // Better to cut the long string, because it could affect virtual scroll performance
         const tooltipContent = formatLongName(field)
@@ -333,8 +333,8 @@ const HashDetails = (props: Props) => {
         expanded?: boolean,
         rowIndex = 0
       ) {
-        const { value: decompressedFieldItem } = decompressingBuffer(fieldItem)
-        const { value: decompressedValueItem } = decompressingBuffer(valueItem)
+        const { value: decompressedFieldItem } = decompressingBuffer(fieldItem, compressor)
+        const { value: decompressedValueItem } = decompressingBuffer(valueItem, compressor)
         const value = bufferToString(valueItem)
         const field = bufferToString(decompressedFieldItem)
         // Better to cut the long string, because it could affect virtual scroll performance
@@ -434,7 +434,6 @@ const HashDetails = (props: Props) => {
       minWidth: 95,
       maxWidth: 95,
       render: function Actions(_act: any, { field: fieldItem, value: valueItem }: HashFieldDto, _, rowIndex?: number) {
-        const compressor = getCompressor(valueItem)
         const field = bufferToString(fieldItem, viewFormat)
         const isEditable = !compressor && isFormatEditable(viewFormat)
         const tooltipContent = compressor ? TEXT_DISABLED_COMPRESSED_VALUE : TEXT_DISABLED_FORMATTER_EDITING