Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core-snapshots): implement core-snapshots package #3672

Merged
merged 46 commits into from May 5, 2020
Merged
Show file tree
Hide file tree
Changes from 43 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
e32ca08
feat(core-snapshots): implements truncate method in new core-snapshot…
sebastijankuzner Apr 3, 2020
f47d787
feat(core-snapshots): implemented rollback method for core-snapshot p…
sebastijankuzner Apr 7, 2020
0fb7be8
feat(core-snapshots): implemented dump, verify, restore, truncate, re…
sebastijankuzner Apr 14, 2020
5b19907
chore(core-snapshots): remove dependency on core-database
sebastijankuzner Apr 14, 2020
aa567fc
chore(core-snapshots): render progress every 100 ms
sebastijankuzner Apr 14, 2020
33ef9b3
chore(core-snapshots): add packageVersion and network into meta data
sebastijankuzner Apr 14, 2020
a0a126b
chore(core-snapshots): add get read and write stream methods, add ski…
sebastijankuzner Apr 14, 2020
27abe5a
chore(core-snapshots): add support for different codecs
sebastijankuzner Apr 14, 2020
e93a04d
chore(core-snapshots): codec return stream instead of codec
sebastijankuzner Apr 14, 2020
e416241
chore(core-snapshots): add support for workers
sebastijankuzner Apr 15, 2020
1f08369
refactor(core-snapshots): clean code and renames
sebastijankuzner Apr 15, 2020
f92f988
chore(core-snapshots): add test and fix code
sebastijankuzner Apr 20, 2020
a02ca1f
chore(core-snapshots): add ReadProcessor
sebastijankuzner Apr 21, 2020
ab71653
chore(core-snapshots): remove init state and fix dump
sebastijankuzner Apr 21, 2020
05d7ae8
chore(core-snapshots): all methods working correctly
sebastijankuzner Apr 22, 2020
34193fe
chore(core-snapshots): add custom encoder decoder and different fixes
sebastijankuzner Apr 23, 2020
3d222a5
chore(core-snapshots): detect end of items in file
sebastijankuzner Apr 23, 2020
d695ccd
chore(core-snapshots): use streamWriter in dump action
sebastijankuzner Apr 28, 2020
a06245d
chore(core-snapshots): fix restore, verify and change codec for trans…
sebastijankuzner Apr 29, 2020
a465e83
chore(core-snapshots): dump in range
sebastijankuzner Apr 29, 2020
aeaf1aa
chore(core-snapshots): cleanup cli
sebastijankuzner Apr 29, 2020
5aa8c98
test(core-snapshots): add cli tests
sebastijankuzner Apr 29, 2020
dfeab90
chore(core-snapshots): fix codec and add zlib support
sebastijankuzner Apr 30, 2020
d24a096
chore(core-snapshots): fix stream reader and writer to use compression
sebastijankuzner Apr 30, 2020
ab896b7
chore(core-snapshots): fix EOF handling, ...
sebastijankuzner Apr 30, 2020
be242b7
test(core-snapshots): add core-snapshot tests
sebastijankuzner Apr 30, 2020
0d3af67
chore(core-snapshots): add support for verification inside restore, c…
sebastijankuzner Apr 30, 2020
1dfdefb
chore(core-snapshots): istanbul ignore
sebastijankuzner Apr 30, 2020
0748836
Merge branch 'develop' into feature/core-snapshots
sebastijankuzner Apr 30, 2020
22a46d2
chore(core-snapshots): after migration fixes
sebastijankuzner Apr 30, 2020
8d27a34
chore(core-snapshots): fix lint problems
sebastijankuzner Apr 30, 2020
0a6abb0
chore(core-snapshot): fix lint problems
sebastijankuzner May 1, 2020
aa0f5ef
chore(core-snapshot): terminate app after execution
sebastijankuzner May 1, 2020
e181fff
test(core-snapshot): fix core/command tests
sebastijankuzner May 4, 2020
6a28684
Merge branch 'develop' into feature/core-snapshots
sebastijankuzner May 4, 2020
870a00e
chore(core-snapshot): fix problems highlighted in the PR
sebastijankuzner May 4, 2020
e4e7532
test(core-snapshot): add tests
sebastijankuzner May 4, 2020
29c5d60
Merge branch 'develop' into feature/core-snapshots
sebastijankuzner May 4, 2020
f654e30
chore(core-snapshot): add additional exports required by core-snapsho…
sebastijankuzner May 4, 2020
fe8bb6d
test(core): add tests for core-snapshots
sebastijankuzner May 4, 2020
894a633
chore(core-snapshots): change codec properties style to decode/encode…
sebastijankuzner May 4, 2020
9332474
chore(core): add counter and progress object into progressRenderer
sebastijankuzner May 4, 2020
1b68e2d
chore(core-snapshots): add istanbul ignore
sebastijankuzner May 4, 2020
79d2411
chore(core-snapshots): fixes suggested in PR
sebastijankuzner May 4, 2020
99aefdf
refactor: reduced nesting
faustbrian May 5, 2020
7c87bfb
Merge branch 'develop' into feature/core-snapshots
faustbrian May 5, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Binary file not shown.
@@ -0,0 +1 @@
{"blocks":{"count":51,"start":1,"end":52},"transactions":{"count":153,"start":0,"end":98136840},"rounds":{"count":51,"start":1,"end":2},"folder":"1-52","skipCompression":true,"network":"testnet","packageVersion":"3.0.0-next.0","codec":"json"}
Binary file not shown.
Binary file not shown.
134 changes: 134 additions & 0 deletions __tests__/unit/core-snapshots/__fixtures__/assets.ts
@@ -0,0 +1,134 @@
import { Meta } from "@packages/core-snapshots/src/contracts";
import { Models } from "@arkecosystem/core-database";
import { BigNumber } from "@arkecosystem/utils";

export const metaData: Meta.MetaData = {
blocks: {
count: 261,
start: 1,
end: 261,
},
transactions: {
count: 153,
start: 1,
end: 153,
},
rounds: {
count: 306,
start: 1,
end: 306,
},
network: "testnet",
packageVersion: "3.0.0-next.0",
folder: "1-261",
skipCompression: false,
codec: "default"
};

export const blocksBigNumber: Models.Block[] = [
{
id: "17184958558311101492",
version: 0,
timestamp: 0,
previousBlock: "0",
height: 1,
numberOfTransactions: 153,
totalAmount: BigNumber.make(12500000000000000),
totalFee: BigNumber.make(0),
reward: BigNumber.make(0),
payloadLength: 35960,
payloadHash: "d9acd04bde4234a81addb8482333b4ac906bed7be5a9970ce8ada428bd083192",
generatorPublicKey: "03b47f6b6719c76bad46a302d9cff7be9b1c2b2a20602a0d880f139b5b8901f068",
blockSignature: "304402202fe5de5697fa25d3d3c0cb24617ac02ddfb1c915ee9194a89f8392f948c6076402200d07c5244642fe36afa53fb2d048735f1adfa623e8fa4760487e5f72e17d253b"
},
{
id: "12294213091819080736",
version:0,
timestamp:96928320,
previousBlock: "17184958558311101492",
height:2,
numberOfTransactions:0,
totalAmount: BigNumber.make(0),
totalFee: BigNumber.make(0),
reward: BigNumber.make(0),
payloadLength: 0,
payloadHash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
generatorPublicKey: "022f2978d57f95c021b9d4bf082b482738ce392bcf6bc213710e7a21504cfeb5a0",
blockSignature: "3045022100a08a6322e6dca4b1be83ce0bbd14c4335be1f72fcbb806af24e88c9db5b9f05e0220289eb4db6a2fb46f4c9b577828dd4e63e4b76f7707d947135b879a94e9c988fa"
}
];

export const blocks: Models.Block[] = [
{
id: "17184958558311101492",
version: 0,
timestamp: 0,
previousBlock: "0",
height: 1,
numberOfTransactions: 153,
// totalAmount: BigNumber.make(12500000000000000),
// totalFee: BigNumber.make(0),
// reward: BigNumber.make(0),
// @ts-ignore
totalAmount: BigInt(12500000000000000),
// @ts-ignore
totalFee: BigInt(0),
// @ts-ignore
reward: BigInt(0),
payloadLength: 35960,
payloadHash: "d9acd04bde4234a81addb8482333b4ac906bed7be5a9970ce8ada428bd083192",
generatorPublicKey: "03b47f6b6719c76bad46a302d9cff7be9b1c2b2a20602a0d880f139b5b8901f068",
blockSignature: "304402202fe5de5697fa25d3d3c0cb24617ac02ddfb1c915ee9194a89f8392f948c6076402200d07c5244642fe36afa53fb2d048735f1adfa623e8fa4760487e5f72e17d253b"
},
{
id: "12294213091819080736",
version:0,
timestamp:96928320,
previousBlock: "17184958558311101492",
height:2,
numberOfTransactions:0,
totalAmount: BigNumber.make(0),
totalFee: BigNumber.make(0),
reward: BigNumber.make(0),
// @ts-ignore
// totalAmount: BigNumber(0),
// // @ts-ignore
// totalFee: BigInt(0),
// // @ts-ignore
// reward: BigInt(0),
payloadLength: 0,
payloadHash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
generatorPublicKey: "022f2978d57f95c021b9d4bf082b482738ce392bcf6bc213710e7a21504cfeb5a0",
blockSignature: "3045022100a08a6322e6dca4b1be83ce0bbd14c4335be1f72fcbb806af24e88c9db5b9f05e0220289eb4db6a2fb46f4c9b577828dd4e63e4b76f7707d947135b879a94e9c988fa"
}
];

export const transactions: Models.Transaction[] = [
{
id: "db1aa687737858cc9199bfa336f9b1c035915c30aaee60b1e0f8afadfdb946bd",
version:1,
blockId: "17184958558311101492",
sequence:0,
timestamp:0,
nonce: BigNumber.make(1),
senderPublicKey:"035b63b4668ee261c16ca91443f3371e2fe349e131cb7bf5f8a3e93a3ddfdfc788",
recipientId:"AHXtmB84sTZ9Zd35h9Y1vfFvPE2Xzqj8ri",
type:0,
typeGroup:1,
vendorField: undefined,
amount: BigNumber.make(245098000000000),
fee: BigNumber.make(0),
serialized: Buffer.from([255,1,23,0,0,0,0,0,3,91,99,180,102,142,226,97,193,108,169,20,67,243,55,30,47,227,73,225,49,203,123,245,248,163,233,58,61,223,223,199,136,0,0,0,0,0,0,0,0,0,0,164,57,83,234,222,0,0,0,0,0,0,23,19,75,91,228,179,39,221,249,194,187,71,254,200,161,164,65,137,233,15,116,48,68,2,32,95,203,6,119,224,107,222,122,172,61,199,118,102,86,21,244,185,62,248,195,237,15,221,236,239,153,0,231,79,203,0,243,2,32,105,88,160,201,134,142,161,177,243,209,81,189,250,146,218,28,226,77,224,177,252,217,25,51,230,79,183,151,30,146,244,141]),
asset: {}
}
];

export const rounds: Models.Round[] = [
{
publicKey: "021770413ad01c60b94e1d3ed44c00e0145fe7897e40f5f6265e220f4e65cf427f",
// @ts-ignore
round: 1,
// @ts-ignore
balance: 245098000000000,
},
];
3 changes: 3 additions & 0 deletions __tests__/unit/core-snapshots/__fixtures__/index.ts
@@ -0,0 +1,3 @@
import * as Assets from "./assets";

export { Assets };
113 changes: 113 additions & 0 deletions __tests__/unit/core-snapshots/codecs/codec.test.ts
@@ -0,0 +1,113 @@
import "jest-extended";
import {decamelize} from "xcase";

import { MessagePackCodec } from "@packages/core-snapshots/src/codecs";
import { Assets } from "../__fixtures__";

const appendPrefix = (prefix: string, entity: any) => {
let itemToReturn = {};

let item = entity;

for(let key of Object.keys(item)) {
itemToReturn[prefix + decamelize(key)] = item[key];
}

return itemToReturn;
}

let codec;

beforeEach(() => {
codec = new MessagePackCodec();
})

describe("Codec", () => {
describe("encodeBlock", () => {
it("should be ok", async () => {
let encoded = codec.encodeBlock(appendPrefix("Block_", Assets.blocks[1]));

expect(encoded).toBeDefined();
});

it("should throw error", async () => {
let corruptedBlock = {
Block_id: "123"
}

expect(() => {codec.encodeBlock(corruptedBlock)}).toThrow();
});
})

describe("decodeBlock", () => {
it("should be ok", async () => {
let encoded = codec.encodeBlock(appendPrefix("Block_", Assets.blocks[1]));

expect(encoded).toBeDefined();

let decoded = codec.decodeBlock(encoded);

expect(decoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.decodeBlock(Buffer.from(""))}).toThrow();
});
})

describe("encodeTransaction", () => {
it("should be ok", async () => {
let encoded = codec.encodeTransaction(Assets.transactions[0]);

expect(encoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.encodeTransaction(undefined)}).toThrow();
});
})

describe("decodeTransaction", () => {
it("should be ok", async () => {
let encoded = codec.encodeTransaction(appendPrefix("Transaction_", Assets.transactions[0]));

expect(encoded).toBeDefined();

let decoded = codec.decodeTransaction(encoded);

expect(decoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.decodeTransaction(Buffer.from(""))}).toThrow();
});
})

describe("encodeRound", () => {
it("should be ok", async () => {
let encoded = codec.encodeRound(appendPrefix("Round_", Assets.rounds[0]));

expect(encoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.encodeRound(undefined)}).toThrow();
});
})

describe("decodeRound", () => {
it("should be ok", async () => {
let encoded = codec.encodeRound(appendPrefix("Round_", Assets.rounds[0]));

expect(encoded).toBeDefined();

let decoded = codec.decodeRound(encoded);

expect(decoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.decodeRound(Buffer.from(""))}).toThrow();
});
})
});
109 changes: 109 additions & 0 deletions __tests__/unit/core-snapshots/codecs/json-codec.test.ts
@@ -0,0 +1,109 @@
import "jest-extended";
import {decamelize} from "xcase";

import { JSONCodec } from "@packages/core-snapshots/src/codecs";
import { Assets } from "../__fixtures__";

const appendPrefix = (prefix: string, entity: any) => {
let itemToReturn = {};

let item = entity;

for(let key of Object.keys(item)) {
itemToReturn[prefix + decamelize(key)] = item[key];
}

return itemToReturn;
}

let codec;

beforeEach(() => {
codec = new JSONCodec();
})

describe("Codec", () => {
describe("encodeBlock", () => {
it("should be ok", async () => {
let encoded = codec.encodeBlock(appendPrefix("Block_", Assets.blocks[1]));

expect(encoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.encodeBlock(undefined)}).toThrow();
});
})

describe("decodeBlock", () => {
it("should be ok", async () => {
let encoded = codec.encodeBlock(appendPrefix("Block_", Assets.blocks[1]));

expect(encoded).toBeDefined();

let decoded = codec.decodeBlock(encoded);

expect(decoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.decodeBlock(Buffer.from(""))}).toThrow();
});
})

describe("encodeTransaction", () => {
it("should be ok", async () => {
let encoded = codec.encodeTransaction(Assets.transactions[0]);

expect(encoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.encodeTransaction(undefined)}).toThrow();
});
})

describe("decodeTransaction", () => {
it("should be ok", async () => {
let encoded = codec.encodeTransaction(appendPrefix("Transaction_", Assets.transactions[0]));

expect(encoded).toBeDefined();

let decoded = codec.decodeTransaction(encoded);

expect(decoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.decodeTransaction(Buffer.from(""))}).toThrow();
});
})

describe("encodeRound", () => {
it("should be ok", async () => {
let encoded = codec.encodeRound(appendPrefix("Round_", Assets.rounds[0]));

expect(encoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.encodeRound(undefined)}).toThrow();
});
})

describe("decodeRound", () => {
it("should be ok", async () => {
let encoded = codec.encodeRound(appendPrefix("Round_", Assets.rounds[0]));

expect(encoded).toBeDefined();

let decoded = codec.decodeRound(encoded);

expect(decoded).toBeDefined();
});

it("should throw error", async () => {
expect(() => {codec.decodeRound(Buffer.from(""))}).toThrow();
});
})
});