From 8a0a0598e74fdde1f292d088bf3f18f46029840b Mon Sep 17 00:00:00 2001 From: Roman Sergeenko Date: Fri, 31 Mar 2023 15:40:15 +0200 Subject: [PATCH 1/2] #RI-4146 - add PHP GZCompress decompression --- package.json | 3 +++ .../database/entities/database.entity.ts | 1 + redisinsight/ui/src/constants/keys.ts | 3 ++- .../form-components/DbCompressor.tsx | 6 ++++- .../src/utils/decompressors/decompressors.ts | 16 ++++++++++++- .../src/utils/formatters/bufferFormatters.ts | 6 +++-- .../utils/tests/decompressors/constants.ts | 3 +++ .../tests/decompressors/decompressors.spec.ts | 23 ++++++++++++++++++- .../tests/formatters/bufferFormatters.spec.ts | 9 +++++++- yarn.lock | 15 ++++++++++++ 10 files changed, 78 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 8313ad2277..d9fc3bb5ac 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "@types/jsonpath": "^0.2.0", "@types/lodash": "^4.14.171", "@types/node": "14.14.10", + "@types/pako": "^2.0.0", "@types/react": "^18.0.20", "@types/react-dom": "^18.0.5", "@types/react-redux": "^7.1.12", @@ -233,6 +234,7 @@ "electron-log": "^4.2.4", "electron-store": "^8.0.0", "electron-updater": "^5.0.5", + "fflate": "^0.7.4", "file-saver": "^2.0.5", "formik": "^2.2.9", "fzstd": "^0.1.0", @@ -244,6 +246,7 @@ "jsonpath": "^1.1.1", "lodash": "^4.17.21", "lz4js": "^0.2.0", + "pako": "^2.1.0", "php-serialize": "^4.0.2", "rawproto": "^0.7.6", "react": "^18.2.0", diff --git a/redisinsight/api/src/modules/database/entities/database.entity.ts b/redisinsight/api/src/modules/database/entities/database.entity.ts index 650f5530dd..a42cad5f23 100644 --- a/redisinsight/api/src/modules/database/entities/database.entity.ts +++ b/redisinsight/api/src/modules/database/entities/database.entity.ts @@ -32,6 +32,7 @@ export enum Compressor { LZ4 = 'LZ4', SNAPPY = 'SNAPPY', Brotli = 'Brotli', + PHPGZCompress = 'PHPGZCompress', } @Entity('database_instance') diff --git a/redisinsight/ui/src/constants/keys.ts b/redisinsight/ui/src/constants/keys.ts index 971d39bf28..2cce1b921d 100644 --- a/redisinsight/ui/src/constants/keys.ts +++ b/redisinsight/ui/src/constants/keys.ts @@ -187,7 +187,7 @@ export enum KeyValueCompressor { LZ4 = 'LZ4', SNAPPY = 'SNAPPY', Brotli = 'Brotli', - // PHPGZCompress = 'PHPGZCompress', + PHPGZCompress = 'PHPGZCompress', } export const COMPRESSOR_MAGIC_SYMBOLS: ICompressorMagicSymbols = Object.freeze({ @@ -196,6 +196,7 @@ export const COMPRESSOR_MAGIC_SYMBOLS: ICompressorMagicSymbols = Object.freeze({ [KeyValueCompressor.LZ4]: '4,34,77,24', // 04 22 4d 18 hex [KeyValueCompressor.SNAPPY]: '', // no magic symbols [KeyValueCompressor.Brotli]: '', // no magic symbols + [KeyValueCompressor.PHPGZCompress]: '', // no magic symbols }) export type ICompressorMagicSymbols = { diff --git a/redisinsight/ui/src/pages/home/components/AddInstanceForm/InstanceForm/form-components/DbCompressor.tsx b/redisinsight/ui/src/pages/home/components/AddInstanceForm/InstanceForm/form-components/DbCompressor.tsx index 7683a7af16..320811ea2c 100644 --- a/redisinsight/ui/src/pages/home/components/AddInstanceForm/InstanceForm/form-components/DbCompressor.tsx +++ b/redisinsight/ui/src/pages/home/components/AddInstanceForm/InstanceForm/form-components/DbCompressor.tsx @@ -49,7 +49,11 @@ const DbCompressor = (props: Props) => { { value: KeyValueCompressor.Brotli, inputDisplay: 'Brotli' - } + }, + { + value: KeyValueCompressor.PHPGZCompress, + inputDisplay: 'PHP GZCompress' + }, ] const handleChangeDbCompressorCheckbox = (e: ChangeEvent): void => { diff --git a/redisinsight/ui/src/utils/decompressors/decompressors.ts b/redisinsight/ui/src/utils/decompressors/decompressors.ts index 6db9b71192..2e10502ddf 100644 --- a/redisinsight/ui/src/utils/decompressors/decompressors.ts +++ b/redisinsight/ui/src/utils/decompressors/decompressors.ts @@ -1,12 +1,15 @@ import { forIn } from 'lodash' import { unzip } from 'gzip-js' import { decompress as decompressFzstd } from 'fzstd' +// @ts-ignore import { decompress as decompressLz4 } from 'lz4js' import { decompress as decompressSnappy } from '@stablelib/snappy' +// @ts-ignore import { decompress as decompressBrotli } from 'brotli-unicode/js' +import { inflate } from 'pako' import { COMPRESSOR_MAGIC_SYMBOLS, ICompressorMagicSymbols, KeyValueCompressor } from 'uiSrc/constants' import { RedisResponseBuffer, RedisString } from 'uiSrc/slices/interfaces' -import { anyToBuffer, bufferToString, isEqualBuffers, Nullable } from 'uiSrc/utils' +import { anyToBuffer, bufferToString, bufferToUint8Array, isEqualBuffers, Nullable } from 'uiSrc/utils' const decompressingBuffer = ( reply: RedisResponseBuffer, @@ -66,6 +69,17 @@ const decompressingBuffer = ( isCompressed: !isEqualBuffers(value, reply), } } + case KeyValueCompressor.PHPGZCompress: { + const decompressedValue = inflate(bufferToUint8Array(reply)) + if (!decompressedValue) return { value: reply, compressor: null, isCompressed: false } + + const value = anyToBuffer(decompressedValue) + return { + value, + compressor, + isCompressed: !isEqualBuffers(value, reply), + } + } default: { return { value: reply, compressor: null, isCompressed: false } } diff --git a/redisinsight/ui/src/utils/formatters/bufferFormatters.ts b/redisinsight/ui/src/utils/formatters/bufferFormatters.ts index d4284a4540..dcc1c79f8e 100644 --- a/redisinsight/ui/src/utils/formatters/bufferFormatters.ts +++ b/redisinsight/ui/src/utils/formatters/bufferFormatters.ts @@ -106,7 +106,8 @@ const ASCIIToBuffer = (strInit: string) => { return anyToBuffer(Array.from(Buffer.from(result, 'hex'))) } -const bufferToUTF8 = (reply: RedisResponseBuffer): string => decoder.decode(new Uint8Array(reply.data)) +const bufferToUint8Array = (reply: RedisResponseBuffer): Uint8Array => new Uint8Array(reply.data) +const bufferToUTF8 = (reply: RedisResponseBuffer): string => decoder.decode(bufferToUint8Array(reply)) const UintArrayToString = (reply: UintArray): string => decoder.decode(new Uint8Array(reply)) @@ -138,7 +139,7 @@ const hexToBuffer = (data: string): RedisResponseBuffer => { } const bufferToJava = (reply: RedisResponseBuffer) => { - const stream = new ObjectInputStream(new Uint8Array(reply.data)) + const stream = new ObjectInputStream(bufferToUint8Array(reply)) const decoded = stream.readObject() const { fields } = decoded const fieldsArray = Array.from(fields, ([key, value]) => ({ [key]: value })) @@ -172,6 +173,7 @@ export { ASCIIToBuffer, isEqualBuffers, stringToBuffer, + bufferToUint8Array, bufferToString, UintArrayToString, hexToBuffer, diff --git a/redisinsight/ui/src/utils/tests/decompressors/constants.ts b/redisinsight/ui/src/utils/tests/decompressors/constants.ts index 419789a4fc..5f90837f1b 100644 --- a/redisinsight/ui/src/utils/tests/decompressors/constants.ts +++ b/redisinsight/ui/src/utils/tests/decompressors/constants.ts @@ -21,3 +21,6 @@ export const SNAPPY_COMPRESSED_VALUE_2 = [1, 0, 50] export const BROTLI_COMPRESSED_VALUE_1 = [49, 65, 76, 231, 187, 141, 68] export const BROTLI_COMPRESSED_VALUE_2 = [49, 65, 76, 231, 191, 141, 68] + +export const PHPGZCOMPRESS_COMPRESSED_VALUE_1 = [120, 156, 51, 4, 0, 0, 50, 0, 50] +export const PHPGZCOMPRESS_COMPRESSED_VALUE_2 = [120, 156, 51, 2, 0, 0, 51, 0, 51] diff --git a/redisinsight/ui/src/utils/tests/decompressors/decompressors.spec.ts b/redisinsight/ui/src/utils/tests/decompressors/decompressors.spec.ts index 8954074195..9dab8e8716 100644 --- a/redisinsight/ui/src/utils/tests/decompressors/decompressors.spec.ts +++ b/redisinsight/ui/src/utils/tests/decompressors/decompressors.spec.ts @@ -18,6 +18,8 @@ import { SNAPPY_COMPRESSED_VALUE_1, BROTLI_COMPRESSED_VALUE_1, BROTLI_COMPRESSED_VALUE_2, + PHPGZCOMPRESS_COMPRESSED_VALUE_1, + PHPGZCOMPRESS_COMPRESSED_VALUE_2, } from './constants' const defaultValues = [ @@ -129,6 +131,22 @@ const defaultValues = [ outputStr: DECOMPRESSED_VALUE_STR_2, isCompressed: true, }, + { + input: PHPGZCOMPRESS_COMPRESSED_VALUE_1, + compressor: KeyValueCompressor.PHPGZCompress, + compressorInit: KeyValueCompressor.PHPGZCompress, + output: DECOMPRESSED_VALUE_1, + outputStr: DECOMPRESSED_VALUE_STR_1, + isCompressed: true, + }, + { + input: PHPGZCOMPRESS_COMPRESSED_VALUE_2, + compressor: KeyValueCompressor.PHPGZCompress, + compressorInit: KeyValueCompressor.PHPGZCompress, + output: DECOMPRESSED_VALUE_2, + outputStr: DECOMPRESSED_VALUE_STR_2, + isCompressed: true, + }, ].map((value) => ({ ...value, input: anyToBuffer(value.input) @@ -139,7 +157,10 @@ describe('getCompressor', () => { let expected = compressorByValue || compressor // SNAPPY doesn't have magic symbols - if (compressor === KeyValueCompressor.SNAPPY || compressor === KeyValueCompressor.Brotli) { + if (compressor === KeyValueCompressor.SNAPPY + || compressor === KeyValueCompressor.Brotli + || compressor === KeyValueCompressor.PHPGZCompress + ) { expected = null } diff --git a/redisinsight/ui/src/utils/tests/formatters/bufferFormatters.spec.ts b/redisinsight/ui/src/utils/tests/formatters/bufferFormatters.spec.ts index f4e2ff114d..79419e1751 100644 --- a/redisinsight/ui/src/utils/tests/formatters/bufferFormatters.spec.ts +++ b/redisinsight/ui/src/utils/tests/formatters/bufferFormatters.spec.ts @@ -11,7 +11,8 @@ import { bufferToHex, bufferToBinary, binaryToBuffer, - bufferToJava + bufferToJava, + bufferToUint8Array, } from 'uiSrc/utils' const defaultValues = [ @@ -141,3 +142,9 @@ describe('bufferToJava', () => { expect(bufferToJava(input)).toEqual(expected) }) }) + +describe('bufferToUint8Array', () => { + test.each(javaValues)('%o', ({ uint8Array }) => { + expect(bufferToUint8Array(anyToBuffer(uint8Array))).toEqual(new Uint8Array(uint8Array)) + }) +}) diff --git a/yarn.lock b/yarn.lock index ca85d67965..dd9d2e7f06 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2674,6 +2674,11 @@ resolved "https://registry.yarnpkg.com/@types/numeral/-/numeral-0.0.28.tgz#e43928f0bda10b169b6f7ecf99e3ddf836b8ebe4" integrity sha512-Sjsy10w6XFHDktJJdXzBJmoondAKW+LcGpRFH+9+zXEDj0cOH8BxJuZA9vUDSMAzU1YRJlsPKmZEEiTYDlICLw== +"@types/pako@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@types/pako/-/pako-2.0.0.tgz#12ab4c19107528452e73ac99132c875ccd43bdfb" + integrity sha512-10+iaz93qR5WYxTo+PMifD5TSxiOtdRaxBf7INGGXMQgTCu8Z/7GYWYFUOS3q/G0nE5boj1r4FEB+WSy7s5gbA== + "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" @@ -7637,6 +7642,11 @@ fengari@^0.1.4: sprintf-js "^1.1.1" tmp "^0.0.33" +fflate@^0.7.4: + version "0.7.4" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.7.4.tgz#61587e5d958fdabb5a9368a302c25363f4f69f50" + integrity sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw== + figures@^3.0.0, figures@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -12390,6 +12400,11 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== +pako@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86" + integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug== + pako@~1.0.2, pako@~1.0.5: version "1.0.11" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" From a37f8d7bf727124eef789a94aa6822955c496d56 Mon Sep 17 00:00:00 2001 From: Roman Sergeenko Date: Mon, 3 Apr 2023 09:04:56 +0200 Subject: [PATCH 2/2] #RI-4146 - remove fflate --- package.json | 1 - yarn.lock | 5 ----- 2 files changed, 6 deletions(-) diff --git a/package.json b/package.json index d9fc3bb5ac..ae670ab013 100644 --- a/package.json +++ b/package.json @@ -234,7 +234,6 @@ "electron-log": "^4.2.4", "electron-store": "^8.0.0", "electron-updater": "^5.0.5", - "fflate": "^0.7.4", "file-saver": "^2.0.5", "formik": "^2.2.9", "fzstd": "^0.1.0", diff --git a/yarn.lock b/yarn.lock index dd9d2e7f06..3380653960 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7642,11 +7642,6 @@ fengari@^0.1.4: sprintf-js "^1.1.1" tmp "^0.0.33" -fflate@^0.7.4: - version "0.7.4" - resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.7.4.tgz#61587e5d958fdabb5a9368a302c25363f4f69f50" - integrity sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw== - figures@^3.0.0, figures@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af"