Skip to content

add formatters #3885

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 7, 2024
Merged

add formatters #3885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
987fdd8
add brotli
mariasergeenko Oct 2, 2024
57f4249
add gzip and lz4
mariasergeenko Oct 3, 2024
525d050
add php-gzc
mariasergeenko Oct 3, 2024
8c9123c
update package.json
mariasergeenko Oct 3, 2024
5a060be
fix per comments
mariasergeenko Oct 4, 2024
fb77390
text fix#2
mariasergeenko Oct 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions tests/e2e/helpers/decompressors/base-decompressors-populator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { createClient } from 'redis';

export abstract class BaseDatabasePopulator {
private client: ReturnType<typeof createClient>;

protected abstract createCompressedKeys(): Promise<void>;

constructor(private host: string, private port: string) {
const dbConf = { port: Number.parseInt(port), host, username: 'default' };
this.client = createClient(dbConf);

this.client.on('error', (error: string) => {
throw new Error(`Redis connection error: ${error}`);
});
}

/**
* Populate db with compressed keys
*/
public async populateDB(): Promise<void> {
this.client.on('connect', async () => {
console.log('Connected to Redis');
try {
await this.createCompressedKeys();
} catch (error) {
console.error('Error during key creation:', error);
} finally {
await this.client.quit();
}
});
}

/**
* create a hash
* @param prefix prefix of the key name
* @param values values of the hash
*/
protected async createHash(
prefix: string,
values: Buffer[]
): Promise<void> {
let fields: string[] = [];

const randomNumber = Array.from({ length: 5 }).map(() => Math.random());

values.forEach((value) => {
const field = `${value.toString()}:${randomNumber.toString()}`;
const fieldValue = `${value.toString()}:${randomNumber.toString()}`;
fields.push(field, fieldValue);
});

try {
await this.client.hset(`${prefix}:hash`, ...fields);
console.log(`Hash created with prefix: ${prefix}`);
} catch (error) {
console.error(`Error creating hash with prefix ${prefix}:`, error);
throw error;
}
}

/**
* create a string
* @param prefix prefix of the key name
* @param value values of the string
*/
protected async createString(prefix: string, value: Buffer): Promise<void> {
this.client.set(`${prefix}:string`, value, (error: Error | null) => {
if (error) {
console.error(`Error saving key ${prefix}:`, error);
throw error;
}
console.log(`Key ${prefix} successfully saved.`);
});
}
}
134 changes: 134 additions & 0 deletions tests/e2e/helpers/decompressors/brotli-database-populator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
import * as proto from 'protobufjs';
import { pack as msgpackrPack } from 'msgpackr';
import * as brotli from 'brotli-unicode';
import * as fflate from 'fflate';
import * as fs from 'fs';
import { BaseDatabasePopulator } from './base-decompressors-populator';

const COMPRESSED_PREFIX = 'Comp';
const BROTLI_PREFIX = 'BROTLI';

export class BrotliDatabasePopulator extends BaseDatabasePopulator {

/**
* Create keys with all types of Bolti compression
*/
protected async createCompressedKeys(): Promise<void> {
await this.createBrotliUnicodeKeys();
await this.createBrotliASCIIKeys();
await this.createBrotliVectorKeys();
await this.createBrotliJSONKeys();
await this.createBrotliPHPUnserializedJSONKeys();
await this.createBrotliJavaSerializedObjectKeys();
await this.createBrotliMsgpackKeys();
await this.createBrotliProtobufKeys();
await this.createBrotliPickleKeys();
}

private async createBrotliUnicodeKeys() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what about comments with description for all your methods?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe that for private methods, especially those with self-explanatory names, comments may not always add significant value. it just clutters up the code

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

okay

const encoder = new TextEncoder();
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Unicode`;
const rawValue = '漢字';
const buf = encoder.encode(rawValue);
const value = Buffer.from(await brotli.compress(buf));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createBrotliASCIIKeys() {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:ASCII`;
const rawValue = '\xac\xed\x00\x05t\x0a4102';
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(await brotli.compress(buf));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createBrotliVectorKeys() {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Vector`;
const rawValue = JSON.parse(fs.readFileSync('./test-data/decompressors/vector.json', 'utf8'));
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(await brotli.compress(buf));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createBrotliJSONKeys() {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:JSON`;
const rawValue = '{"test":"test"}';
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(await brotli.compress(buf));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createBrotliPHPUnserializedJSONKeys() {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:PHP`;
const rawValue = 'a:2:{i:0;s:12:"Sample array";i:1;a:2:{i:0;s:5:"Apple";i:1;s:6:"Orange";}}';
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(await brotli.compress(buf));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createBrotliPickleKeys() {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Pickle`;
const rawValue = fs.readFileSync('./test-data/decompressors/pickleFile1.pickle');
const value = Buffer.from(await brotli.compress(rawValue));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
};

private async createBrotliJavaSerializedObjectKeys() {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Java`;
const rawValue = fs.readFileSync('./test-data/decompressors/test_serialised_obj.ser');
const rawValue2 = fs.readFileSync('./test-data/decompressors/test_annotated_obj.ser');

const value = Buffer.from(await brotli.compress(rawValue));
const value2 = Buffer.from(await brotli.compress(rawValue2));

await this.createString(prefix, value);
await this.createString(prefix, value2);
await this.createHash(prefix, [value,value2]);
}

private async createBrotliMsgpackKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Msgpack`;
const rawValue = msgpackrPack({
hello: 'World',
array: [1, 2],
obj: {test: 'test'},
boolean: false,
});
const value = Buffer.from(await brotli.compress(rawValue));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private createBrotliProtobufKeys(): Promise<void> {
return new Promise((resolve, reject) => {
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Proto`;
proto.load('./test-data/decompressors/awesome.proto', async (err, root) => {
if (err || !root) {
console.error('Error loading protobuf:', err);
return reject(err);
}

try {
const Book = root.lookupType('com.book.BookStore');
const payload = {name: 'Test name', books: {0: 'book 1', 1: 'book 2'}};
const message = Book.create(payload);
const rawValue = Book.encode(message).finish();

const value = Buffer.from(await brotli.compress(rawValue));
await this.createString(prefix, value);
await this.createHash(prefix, [value]);
resolve();
} catch (error) {
reject(error);
}
});
});
}
}

141 changes: 141 additions & 0 deletions tests/e2e/helpers/decompressors/gzip-database-populator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import { pack as msgpackrPack } from 'msgpackr';
import * as fs from 'fs';
import * as fflate from 'fflate';
import * as proto from 'protobufjs';
import { BaseDatabasePopulator } from './base-decompressors-populator';


const COMPRESSED_PREFIX = 'Comp';
const GZIP_PREFIX = 'GZIP';

export class GzipDatabasePopulator extends BaseDatabasePopulator {

/**
* Create keys with all types of Gzip compression
*/
protected async createCompressedKeys(): Promise<void> {
await this.createGZIPUnicodeKeys();
await this.createGZIPASCIIKeys();
await this.createGZIPJSONKeys();
await this.createGZIPPHPUnserializedJSONKeys();
await this.createGZIPMsgpackKeys();
await this.createGZIPProtobufKeys();
await this.createGZIPPickleKeys();
await this.createGZIPJavaSerializedObjectKeys();
await this.createGZIPVectorKeys();
}

private async createGZIPUnicodeKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Unicode`;
const rawValue = '漢字';
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(fflate.compressSync(buf));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createGZIPASCIIKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:ASCII`;
const rawValue = '\xac\xed\x00\x05t\x0a4102';
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(fflate.compressSync(buf));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createGZIPJSONKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:JSON`;
const rawValue = '{"test":"test"}';
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(fflate.compressSync(buf));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createGZIPPHPUnserializedJSONKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:PHP`;
const rawValue =
'a:2:{i:0;s:12:"Sample array";i:1;a:2:{i:0;s:5:"Apple";i:1;s:6:"Orange";}}';
const buf = fflate.strToU8(rawValue);
const value = Buffer.from(fflate.compressSync(buf));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createGZIPJavaSerializedObjectKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Java`;
const rawValue = fs.readFileSync('./test-data/decompressors/test_serialised_obj.ser');
const rawValue2 = fs.readFileSync('./test-data/decompressors/test_annotated_obj.ser');

const value = Buffer.from(fflate.compressSync(rawValue));
const value2 = Buffer.from(fflate.compressSync(rawValue2));

await this.createString(prefix, value);
await this.createString(prefix, value2);
await this.createHash(prefix, [value,value2]);
}

private async createGZIPMsgpackKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Msgpack`;
const rawValue = msgpackrPack({
hello: 'World',
array: [1, 2],
obj: {test: 'test'},
boolean: false,
});

const value = Buffer.from(fflate.compressSync(rawValue));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private async createGZIPVectorKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Vector`;
const rawValue = JSON.parse(fs.readFileSync('./test-data/decompressors/vector.json', 'utf8'));
const value = Buffer.from(fflate.compressSync(rawValue));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}

private createGZIPProtobufKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Proto`;

return new Promise((resolve, reject) => {
proto.load('./test-data/decompressors/awesome.proto', async (err, root) => {
if (err || !root) {
console.error('Error loading protobuf:', err);
return reject(err);
}

const Book = root.lookupType('com.book.BookStore');
const payloadBookStore = {
name: 'Test name',
books: {0: 'book 1', 1: 'book 2'},
};
const message = Book.create(payloadBookStore);
const rawValue = Book.encode(message).finish();
const value = Buffer.from(fflate.compressSync(rawValue));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);

resolve();
});
});
}

private async createGZIPPickleKeys(): Promise<void> {
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Pickle`;
const rawValue = fs.readFileSync('./test-data/decompressors/pickleFile1.pickle');
const value = Buffer.from(fflate.compressSync(rawValue));

await this.createString(prefix, value);
await this.createHash(prefix, [value]);
}
}
Loading
Loading