From 483b1a95ada53e465486556456dac76769d8176c Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Tue, 9 Oct 2018 11:01:33 +0300 Subject: [PATCH 01/14] Broken commit (not done) for attestation routine --- .../sharding/config/BeaconConfig.java | 13 +- .../org/ethereum/sharding/domain/Beacon.java | 21 ++- .../sharding/domain/BeaconGenesis.java | 3 +- .../processing/state/ActiveState.java | 125 +++++++++++++ .../processing/state/AttestationRecord.java | 168 ++++++++++++++++++ .../processing/state/BeaconState.java | 21 ++- .../sharding/pubsub/BeaconBlockAttested.java | 42 +++++ .../org/ethereum/sharding/util/Bitfield.java | 108 +++++++++++ .../java/org/ethereum/sharding/util/Sign.java | 51 ++++++ .../sharding/validator/BeaconAttester.java | 39 ++++ .../validator/BeaconAttesterImpl.java | 78 ++++++++ .../sharding/validator/BeaconProposer.java | 8 +- .../validator/BeaconProposerImpl.java | 26 ++- .../sharding/validator/ValidatorService.java | 5 +- .../validator/ValidatorServiceImpl.java | 40 +++-- 15 files changed, 718 insertions(+), 30 deletions(-) create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/pubsub/BeaconBlockAttested.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/util/Sign.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java b/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java index 837d039b83..337288b3e7 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java @@ -43,6 +43,8 @@ import org.ethereum.sharding.processing.db.IndexedBeaconStore; import org.ethereum.sharding.processing.state.BeaconStateRepository; import org.ethereum.sharding.processing.state.StateRepository; +import org.ethereum.sharding.validator.BeaconAttester; +import org.ethereum.sharding.validator.BeaconAttesterImpl; import org.ethereum.sharding.validator.BeaconProposer; import org.ethereum.sharding.validator.BeaconProposerImpl; import org.ethereum.sharding.validator.ValidatorService; @@ -127,8 +129,8 @@ public ValidatorRegistrationService validatorRegistrationService() { @Bean public ValidatorService validatorService() { if (validatorConfig().isEnabled()) { - ValidatorService validatorService = new ValidatorServiceImpl(beaconProposer(), beaconChain(), - publisher(), validatorConfig(), ethereum, blockStore); + ValidatorService validatorService = new ValidatorServiceImpl(beaconProposer(), beaconAttester(), + beaconChain(), publisher(), validatorConfig(), ethereum, blockStore); shardingWorldManager.setProposerService(validatorService); return validatorService; } else { @@ -216,10 +218,15 @@ public Publisher publisher() { @Bean public BeaconProposer beaconProposer() { - return new BeaconProposerImpl(randao(), beaconStateRepository(), + return new BeaconProposerImpl(randao(), beaconStateRepository(), beaconStore(), BeaconChainFactory.stateTransition(validatorRepository()), validatorConfig()); } + @Bean + public BeaconAttester beaconAttester() { + return new BeaconAttesterImpl(beaconStateRepository(), beaconStore(), validatorConfig()); + } + @Bean public Randao randao() { DbSource src = commonConfig.keyValueDataSource("randao"); diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java index 59729ae4ff..73488779b3 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java @@ -18,6 +18,7 @@ package org.ethereum.sharding.domain; import org.ethereum.datasource.Serializer; +import org.ethereum.sharding.processing.state.AttestationRecord; import org.ethereum.util.ByteUtil; import org.ethereum.util.FastByteComparisons; import org.ethereum.util.RLP; @@ -46,13 +47,17 @@ public class Beacon { private byte[] stateHash; /* Slot number */ private long slotNumber; + /* Attestations */ + private AttestationRecord[] attestations; - public Beacon(byte[] parentHash, byte[] randaoReveal, byte[] mainChainRef, byte[] stateHash, long slotNumber) { + public Beacon(byte[] parentHash, byte[] randaoReveal, byte[] mainChainRef, byte[] stateHash, + long slotNumber, AttestationRecord[] attestations) { this.parentHash = parentHash; this.randaoReveal = randaoReveal; this.mainChainRef = mainChainRef; this.stateHash = stateHash; this.slotNumber = slotNumber; + this.attestations = attestations; } public Beacon(byte[] rlp) { @@ -62,6 +67,12 @@ public Beacon(byte[] rlp) { this.mainChainRef = items.get(2).getRLPData(); this.stateHash = items.get(3).getRLPData(); this.slotNumber = ByteUtil.bytesToBigInteger(items.get(4).getRLPData()).longValue(); + + RLPList attestationsRlp = RLP.unwrapList(items.get(5).getRLPData()); + this.attestations = new AttestationRecord[attestationsRlp.size()]; + for (int i = 0; i < attestationsRlp.size(); ++i) { + attestations[i] = new AttestationRecord(attestationsRlp.get(i).getRLPData()); + } } public byte[] getEncoded() { @@ -93,6 +104,10 @@ public long getSlotNumber() { return slotNumber; } + public AttestationRecord[] getAttestations() { + return attestations; + } + public boolean isParentOf(Beacon other) { return FastByteComparisons.equal(this.getHash(), other.getParentHash()); } @@ -101,6 +116,10 @@ public void setStateHash(byte[] stateHash) { this.stateHash = stateHash; } + public void setAttestations(AttestationRecord[] attestations) { + this.attestations = attestations; + } + @Override public boolean equals(Object other) { if (this == other) return true; diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java index c0ae4fb5b6..37ec460526 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.io.ByteStreams; +import org.ethereum.sharding.processing.state.AttestationRecord; import org.ethereum.util.Utils; import java.io.InputStream; @@ -53,7 +54,7 @@ public class BeaconGenesis extends Beacon { private List initialValidators = new ArrayList<>(); BeaconGenesis(Json json) { - super(json.parentHash(), json.randaoReveal(), json.mainChainRef(), EMPTY, SLOT); + super(json.parentHash(), json.randaoReveal(), json.mainChainRef(), EMPTY, SLOT, new AttestationRecord[0]); this.timestamp = json.timestamp(); this.initialValidators = json.validatorSet(); } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java new file mode 100644 index 0000000000..d39f2b5d23 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.processing.state; + +import org.ethereum.util.ByteUtil; +import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; + +import static org.ethereum.crypto.HashUtil.blake2b; +import static org.ethereum.sharding.processing.consensus.BeaconConstants.CYCLE_LENGTH; +import static org.ethereum.util.ByteUtil.toHexString; + +/** + * Active beacon chain state + */ +public class ActiveState { + // Attestations that have not yet been processed + private final AttestationRecord[] pendingAttestations; + // Most recent 2 * CYCLE_LENGTH block hashes, older to newer + private final byte[][] recentBlockHashes; + // RANDAO state + private final byte[] randaoMix; + + // TODO: Add pending_specials + + + public ActiveState(AttestationRecord[] pendingAttestations, byte[][] recentBlockHashes, byte[] randaoMix) { + this.pendingAttestations = pendingAttestations; + this.recentBlockHashes = recentBlockHashes; + this.randaoMix = randaoMix; + } + + /** + * Creates active state with empty pending attestations + * and block hashes filled with `00`*32 + */ + public static ActiveState createEmpty() { + AttestationRecord[] pendingAttestations = new AttestationRecord[0]; + int size = CYCLE_LENGTH * 2; + byte[][] recentBlockHashes = new byte[size][]; + for (int i = 0; i < size; ++i) { + recentBlockHashes[i] = new byte[32]; + } + + return new ActiveState(pendingAttestations, recentBlockHashes, new byte[32]); + } + + public ActiveState(byte[] encoded) { + RLPList list = RLP.unwrapList(encoded); + + RLPList attestationList = RLP.unwrapList(list.get(0).getRLPData()); + this.pendingAttestations = new AttestationRecord[attestationList.size()]; + for (int i = 0; i < attestationList.size(); i++) + pendingAttestations[i] = new AttestationRecord(attestationList.get(i).getRLPData()); + + RLPList hashesList = RLP.unwrapList(list.get(1).getRLPData()); + this.recentBlockHashes = new byte[hashesList.size()][]; + for (int i = 0; i < hashesList.size(); i++) + recentBlockHashes[i] = hashesList.get(i).getRLPData(); + + this.randaoMix = list.get(2).getRLPData(); + } + + public AttestationRecord[] getPendingAttestations() { + return pendingAttestations; + } + + public byte[][] getRecentBlockHashes() { + return recentBlockHashes; + } + + public byte[] getRandaoMix() { + return randaoMix; + } + + public byte[] getEncoded() { + byte[][] encodedAttestations = new byte[pendingAttestations.length][]; + + for (int i = 0; i < pendingAttestations.length; i++) + encodedAttestations[i] = pendingAttestations[i].getEncoded(); + + return RLP.wrapList( + pendingAttestations.length > 0 ? RLP.wrapList(encodedAttestations) : ByteUtil.ZERO_BYTE_ARRAY, + recentBlockHashes.length > 0 ? RLP.wrapList(recentBlockHashes) : ByteUtil.ZERO_BYTE_ARRAY, + RLP.encodeElement(randaoMix) + ); + } + + public byte[] getHash() { + return blake2b(getEncoded()); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder().append("ActiveState{") + .append("pendingAttestations=[").append(pendingAttestations.length).append(" item(s)]") + .append(", recentBlockHashes=[...") + + for (int i = Math.max(0, recentBlockHashes.length - 3); i < recentBlockHashes.length; ++i) { + builder.append(", ").append(toHexString(recentBlockHashes[i])); + } + builder.append("]"); + + builder.append(", randaoMix=") + .append(ByteUtil.toHexString(randaoMix)) + .append('}'); + + return builder.toString(); + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java new file mode 100644 index 0000000000..c5d765bbae --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java @@ -0,0 +1,168 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.processing.state; + +import org.ethereum.util.ByteUtil; +import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; + +import java.math.BigInteger; +import java.util.Arrays; + +import static org.ethereum.util.ByteUtil.bigIntegerToBytes; +import static org.ethereum.util.ByteUtil.byteArrayToLong; +import static org.ethereum.util.ByteUtil.bytesToBigInteger; +import static org.ethereum.util.ByteUtil.intToBytesNoLeadZeroes; +import static org.ethereum.util.ByteUtil.longToBytesNoLeadZeroes; +import static org.ethereum.util.ByteUtil.toHexString; + +/** + * Slot attestation data + */ +public class AttestationRecord { + + // Slot number + private final long slot; + // Shard ID + private final int shardId; + // List of block hashes that this signature is signing over that + // are NOT part of the current chain, in order of oldest to newest + private final byte[][] obliqueParentHashes; + // Block hash in the shard that we are attesting to + private final byte[] shardBlockHash; + // Who is participating + private final byte[] attesterBitfield; + // Last justified block + private final long justifiedSlot; + private final byte[] justifiedBlockHash; + // The actual signature + private final BigInteger[] aggregateSig; // Defined by two BigIntegers? + + public AttestationRecord(long slot, int shardId, byte[][] obliqueParentHashes, byte[] shardBlockHash, + byte[] attesterBitfield, long justifiedSlot, byte[] justifiedBlockHash, + BigInteger[] aggregateSig) { + this.slot = slot; + this.shardId = shardId; + this.obliqueParentHashes = obliqueParentHashes; + this.shardBlockHash = shardBlockHash; + this.attesterBitfield = attesterBitfield; + this.justifiedSlot = justifiedSlot; + this.justifiedBlockHash = justifiedBlockHash; + this.aggregateSig = aggregateSig; + } + + public AttestationRecord(byte[] encoded) { + RLPList list = RLP.unwrapList(encoded); + + this.slot = byteArrayToLong(list.get(0).getRLPData()); + this.shardId = bytesToBigInteger(list.get(1).getRLPData()).shortValue(); + + RLPList hashesList = RLP.unwrapList(list.get(2).getRLPData()); + this.obliqueParentHashes = new byte[hashesList.size()][]; + for (int i = 0; i < hashesList.size(); i++) + this.obliqueParentHashes[i] = hashesList.get(i).getRLPData(); + + this.shardBlockHash = list.get(3).getRLPData(); + this.attesterBitfield = list.get(4).getRLPData(); + this.justifiedSlot = byteArrayToLong(list.get(5).getRLPData()); + this.justifiedBlockHash = list.get(6).getRLPData(); + + RLPList sigList = RLP.unwrapList(list.get(7).getRLPData()); + this.aggregateSig = new BigInteger[sigList.size()]; + for (int i = 0; i < sigList.size(); i++) + this.aggregateSig[i] = ByteUtil.bytesToBigInteger(hashesList.get(i).getRLPData()); + } + + public long getSlot() { + return slot; + } + + public int getShardId() { + return shardId; + } + + public byte[][] getObliqueParentHashes() { + return obliqueParentHashes; + } + + public byte[] getShardBlockHash() { + return shardBlockHash; + } + + public byte[] getAttesterBitfield() { + return attesterBitfield; + } + + public long getJustifiedSlot() { + return justifiedSlot; + } + + public byte[] getJustifiedBlockHash() { + return justifiedBlockHash; + } + + public BigInteger[] getAggregateSig() { + return aggregateSig; + } + + public byte[] getEncoded() { + byte[][] encodedAggSig = new byte[aggregateSig.length][]; + for (int i = 0; i < aggregateSig.length; i++) + encodedAggSig[i] = bigIntegerToBytes(aggregateSig[i]); + + return RLP.wrapList(longToBytesNoLeadZeroes(slot), + intToBytesNoLeadZeroes(shardId), + RLP.wrapList(obliqueParentHashes), + shardBlockHash, + attesterBitfield, + longToBytesNoLeadZeroes(justifiedSlot), + justifiedBlockHash, + RLP.wrapList(encodedAggSig)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AttestationRecord that = (AttestationRecord) o; + return slot == that.slot && + shardId == that.shardId && + Arrays.equals(attesterBitfield, that.attesterBitfield) && + justifiedSlot == that.justifiedSlot && + Arrays.equals(obliqueParentHashes, that.obliqueParentHashes) && + Arrays.equals(shardBlockHash, that.shardBlockHash) && + Arrays.equals(justifiedBlockHash, that.justifiedBlockHash) && + Arrays.equals(aggregateSig, that.aggregateSig); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder() + .append("AttestationRecord{") + .append("slot=").append(slot) + .append(", shardId=").append(shardId) + .append(", obliqueParentHashes=[").append(obliqueParentHashes.length).append(" item(s)]") + .append(", shardBlockHash=").append(toHexString(shardBlockHash)) + .append(", attesterBitfield=").append(toHexString(attesterBitfield)) + .append(", justifiedSlot=").append(justifiedSlot) + .append(", justifiedBlockHash=").append(toHexString(justifiedBlockHash)) + .append(", aggregateSig=[").append(aggregateSig.length).append(" item(s)]}"); + + return builder.toString(); + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconState.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconState.java index ee2b68c4ea..1d0479120f 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconState.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconState.java @@ -35,21 +35,27 @@ public class BeaconState { private final CrystallizedState crystallizedState; + private final ActiveState activeState; - public BeaconState(CrystallizedState crystallizedState) { + public BeaconState(CrystallizedState crystallizedState, ActiveState activeState) { this.crystallizedState = crystallizedState; + this.activeState = activeState; } public CrystallizedState getCrystallizedState() { return crystallizedState; } + public ActiveState getActiveState() { + return activeState; + } + public byte[] getHash() { return flatten().getHash(); } public Flattened flatten() { - return new Flattened(crystallizedState); + return new Flattened(crystallizedState, activeState); } public Committee[][] getCommittees() { @@ -70,26 +76,33 @@ public boolean equals(Object o) { public static class Flattened { private final byte[] crystallizedStateHash; + private final byte[] activeStateHash; - public Flattened(CrystallizedState crystallizedState) { + public Flattened(CrystallizedState crystallizedState, ActiveState activeState) { this.crystallizedStateHash = crystallizedState.getHash(); + this.activeStateHash = activeState.getHash(); } public Flattened(byte[] encoded) { RLPList list = RLP.unwrapList(encoded); this.crystallizedStateHash = list.get(0).getRLPData(); + this.activeStateHash = list.get(1).getRLPData(); } public byte[] getCrystallizedStateHash() { return crystallizedStateHash; } + public byte[] getActiveStateHash() { + return activeStateHash; + } + public byte[] getHash() { return blake2b(encode()); } public byte[] encode() { - return RLP.wrapList(crystallizedStateHash); + return RLP.wrapList(crystallizedStateHash, activeStateHash); } public static final org.ethereum.datasource.Serializer Serializer = new Serializer() { diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/pubsub/BeaconBlockAttested.java b/ethereumj-core/src/main/java/org/ethereum/sharding/pubsub/BeaconBlockAttested.java new file mode 100644 index 0000000000..bdc4797980 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/pubsub/BeaconBlockAttested.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.pubsub; + +import org.ethereum.sharding.processing.state.AttestationRecord; + +/** + * When new {@link AttestationRecord} received + */ +public class BeaconBlockAttested extends Event { + + public static class Data { + private final AttestationRecord attestationRecord; + + public Data(AttestationRecord attestationRecord) { + this.attestationRecord = attestationRecord; + } + + public AttestationRecord getAttestationRecord() { + return attestationRecord; + } + } + + public BeaconBlockAttested(AttestationRecord attestationRecord) { + super(new Data(attestationRecord)); + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java new file mode 100644 index 0000000000..1fc4961086 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.util; + +import java.util.Arrays; + +/** + * Bitfield utility methods + * + * Bitfield is bit array where every bit represents status of + * attester with corresponding index + */ +public class Bitfield { + + /** + * Creates empty bitfield for estimated number of attesters + * @param validatorsCount Number of attesters + * @return empty bitfield with correct length + */ + public static byte[] createEmpty(int validatorsCount) { + return new byte[calcLength(validatorsCount)]; + } + + // TODO: Add test + /** + * Calculates attesters bitfield length + * @param num Number of attesters + * @return Bitfield length in bytes + */ + public static int calcLength(int num) { + return (num + 7) / Byte.SIZE; + } + + /** + * Modifies bitfield to represent attester's vote + * Should place its bit on the right place + * Doesn't modify original bitfield + * @param bitfield Original bitfield + * @param index Index number of attester + * @return bitfield with vote in place + */ + public static byte[] markVote(final byte[] bitfield, int index) { + byte[] newBitfield = Arrays.copyOf(bitfield, bitfield.length); + int byteIndex = index / Byte.SIZE; + int bitIndex = index % Byte.SIZE; + newBitfield[byteIndex] |= 128 >> bitIndex; + return newBitfield; + } + + /** + * Checks whether validator with provided index did his vote + * @param bitfield Bitfield + * @param index Index number of attester + */ + public static boolean hasVoted(byte[] bitfield, int index) { + int byteIndex = index / Byte.SIZE; + int bitIndex = index % Byte.SIZE; + + return (bitfield[byteIndex] & (128 >> bitIndex)) == 1; + } + + /** + * Calculate number of votes in provided bitfield + * @param bitfield Bitfield + * @return number of votes + */ + public static int calcVotes(byte[] bitfield) { + int votes = 0; + for (int i = 0; i < (bitfield.length * Byte.SIZE); ++i) { + if (hasVoted(bitfield, i)) ++votes; + } + + return votes; + } + + /** + * OR aggregation function + * OR aggregation of input bitfields + * @param bitfields Bitfields + * @return All bitfields aggregated using OR + */ + public static byte[] orBitfield(byte[][] bitfields) { + int bitfieldLen = bitfields[0].length; + byte[] aggBitfield = new byte[bitfieldLen]; + for (int i = 0; i < bitfieldLen; ++i) { + for (byte[] bitfield : bitfields) { + aggBitfield[i] |= bitfield[i]; + } + } + + return aggBitfield; + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Sign.java b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Sign.java new file mode 100644 index 0000000000..1bcbfe27c3 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Sign.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.util; + +import java.math.BigInteger; + +import static org.ethereum.crypto.HashUtil.sha3; + +/** + * Signature utilities + */ +public class Sign { + + public static byte[] sign(byte[] msg, byte[] privateKey) { + // TODO: BLS should be here + return sha3(msg, privateKey); + } + + public static boolean verify(byte[] signature, byte[] publicKey) { + // TODO: real BLS verification should be here + return true; + } + + public static BigInteger[] aggSigns(byte[][] signatures) { + // TODO: real BLS signature aggregation instead of XOR + int signatureLen = signatures[0].length; + byte[] aggSignature = new byte[signatureLen]; + for (int i = 0; i < signatureLen; ++i) { + for (byte[] signature : signatures) { + aggSignature[i] ^= signature[i]; + } + } + + return new BigInteger[] {new BigInteger(1, aggSignature), BigInteger.ONE}; + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java new file mode 100644 index 0000000000..6154363e16 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.validator; + +import org.ethereum.sharding.domain.Beacon; +import org.ethereum.sharding.processing.state.AttestationRecord; +import org.ethereum.sharding.processing.state.Committee; + +/** + * Beacon chain block attester + */ +public interface BeaconAttester { + + /** + * Attests block + * + * @param slotNumber Current slot number + * @param index Commitee index with validator id etc + * @param block Block to attest + * @param pubKey Public key of the attestation validator + * @return Attestation record + */ + AttestationRecord attestBlock(long slotNumber, Committee.Index index, Beacon block, byte[] pubKey); +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java new file mode 100644 index 0000000000..7a482f431d --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.validator; + +import org.ethereum.sharding.config.ValidatorConfig; +import org.ethereum.sharding.domain.Beacon; +import org.ethereum.sharding.processing.db.BeaconStore; +import org.ethereum.sharding.processing.state.AttestationRecord; +import org.ethereum.sharding.processing.state.BeaconState; +import org.ethereum.sharding.processing.state.Committee; +import org.ethereum.sharding.processing.state.StateRepository; +import org.ethereum.sharding.util.Bitfield; +import org.ethereum.sharding.util.Sign; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Default implementation of {@link BeaconAttester}. + */ +public class BeaconAttesterImpl implements BeaconAttester { + + private static final Logger logger = LoggerFactory.getLogger("attester"); + + StateRepository repository; + BeaconStore store; + ValidatorConfig config; + + public BeaconAttesterImpl(StateRepository repository, BeaconStore store, ValidatorConfig config) { + this.repository = repository; + this.store = store; + this.config = config; + } + + @Override + public AttestationRecord attestBlock(long slotNumber, Committee.Index index, Beacon block, byte[] pubKey) { + if (block.getAttestations().length == 0) { + throw new RuntimeException("Block should have at least 1 attestation of its proposer"); + } + + // validate that block doesn't contain our attestation + for (AttestationRecord attestationRecord : block.getAttestations()) { + if (Bitfield.hasVoted(attestationRecord.getAttesterBitfield(), index.getValidatorIdx())) { + throw new RuntimeException("Shouldn't attest block again"); + } + } + + BeaconState headState = repository.get(store.getCanonicalHead().getHash()); + long lastJustified = headState.getCrystallizedState().getFinality().getLastJustifiedSlot(); + AttestationRecord attestationRecord = new AttestationRecord( + slotNumber, + index.getShardId(), + new byte[0][0], // FIXME: obliqueParentHashes + new byte[32], // FIXME: shardBlockHash?? + Bitfield.markVote(Bitfield.createEmpty(index.getCommitteeSize()), index.getValidatorIdx()), + lastJustified, + store.getCanonicalByNumber(lastJustified) == null ? new byte[32] : store.getCanonicalByNumber(lastJustified).getHash(), + Sign.aggSigns(new byte[][] {Sign.sign(block.getEncoded(), pubKey)}) + ); + + logger.info("Block {} attested by #{} in slot {} ", block, index.getValidatorIdx(), slotNumber); + return attestationRecord; + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java index e80c870fa5..e7ac46ed9b 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java @@ -19,6 +19,7 @@ import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.processing.state.BeaconState; +import org.ethereum.sharding.processing.state.Committee; /** * Beacon chain block proposer. @@ -48,19 +49,22 @@ public interface BeaconProposer { class Input { long slotNumber; + Committee.Index index; Beacon parent; BeaconState state; byte[] mainChainRef; - public Input(long slotNumber, Beacon parent, BeaconState state, byte[] mainChainRef) { + public Input(long slotNumber, Committee.Index index, Beacon parent, BeaconState state, byte[] mainChainRef) { this.slotNumber = slotNumber; + this.index = index; this.parent = parent; this.state = state; this.mainChainRef = mainChainRef; } - public Input(long slotNumber, ValidatorService.ChainHead head, byte[] mainChainRef) { + public Input(long slotNumber, Committee.Index index, ValidatorService.ChainHead head, byte[] mainChainRef) { this.slotNumber = slotNumber; + this.index = index; this.parent = head.block; this.state = head.state; this.mainChainRef = mainChainRef; diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java index 02775b96ed..01c9231417 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java @@ -21,11 +21,16 @@ import org.ethereum.sharding.config.ValidatorConfig; import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.domain.Validator; +import org.ethereum.sharding.processing.db.BeaconStore; +import org.ethereum.sharding.processing.state.AttestationRecord; import org.ethereum.sharding.pubsub.BeaconChainSynced; import org.ethereum.sharding.processing.consensus.StateTransition; import org.ethereum.sharding.processing.state.BeaconState; import org.ethereum.sharding.processing.state.StateRepository; +import org.ethereum.sharding.util.Bitfield; import org.ethereum.sharding.util.Randao; +import org.ethereum.sharding.util.Sign; +import org.ethereum.util.ByteUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,11 +52,13 @@ public class BeaconProposerImpl implements BeaconProposer { StateTransition stateTransition; StateRepository repository; ValidatorConfig config; + BeaconStore store; - public BeaconProposerImpl(Randao randao, StateRepository repository, + public BeaconProposerImpl(Randao randao, StateRepository repository, BeaconStore store, StateTransition stateTransition, ValidatorConfig config) { this.randao = randao; this.repository = repository; + this.store = store; this.stateTransition = stateTransition; this.config = config; } @@ -74,10 +81,25 @@ byte[] randaoReveal(BeaconState state, byte[] pubKey) { @Override public Beacon createNewBlock(Input in, byte[] pubKey) { Beacon block = new Beacon(in.parent.getHash(), randaoReveal(in.state, pubKey), in.mainChainRef, - HashUtil.EMPTY_DATA_HASH, in.slotNumber); + HashUtil.EMPTY_DATA_HASH, in.slotNumber, new AttestationRecord[0]); BeaconState newState = stateTransition.applyBlock(block, in.state); block.setStateHash(newState.getHash()); + AttestationRecord[] attestationRecords = new AttestationRecord[1]; + byte[] emptyBitfield = Bitfield.createEmpty(in.index.getCommitteeSize()); + long lastJustified = in.state.getCrystallizedState().getFinality().getLastJustifiedSlot(); + attestationRecords[0] = new AttestationRecord( + in.slotNumber, + in.index.getShardId(), + new byte[0][0], + new byte[32], // FIXME: shardBlockHash?? + Bitfield.markVote(emptyBitfield, in.index.getValidatorIdx()), + lastJustified, + store.getCanonicalByNumber(lastJustified) == null ? new byte[32] : store.getCanonicalByNumber(lastJustified).getHash(), + Sign.aggSigns(new byte[][] {Sign.sign(block.getEncoded(), pubKey)}) + ); + block.setAttestations(attestationRecords); + logger.info("New block created {}", block); return block; } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java index d0c41dd839..1342f8aa14 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java @@ -19,6 +19,7 @@ import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.processing.state.BeaconState; +import org.ethereum.sharding.processing.state.Committee; /** * Service that is responsible for scheduling attestation and proposal tasks for the beacon chain validator. @@ -45,13 +46,13 @@ default void init(ChainHead head, byte[]... pubKeys) {} * Submits a task to propose block with given slot number. * Thread safe. */ - default void propose(long slotNumber, int validatorIdx) {} + default void propose(long slotNumber, Committee.Index index) {} /** * Submits a task to make an attestation in a given slot number. * Thread safe. */ - default void attest(long slotNumber, int validatorIdx) {} + default void attest(long slotNumber, Committee.Index index) {} /** * Handy aggregator diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java index 12b547a6ab..11eec17aab 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java @@ -26,7 +26,9 @@ import org.ethereum.sharding.config.ValidatorConfig; import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.domain.Validator; +import org.ethereum.sharding.processing.state.AttestationRecord; import org.ethereum.sharding.processing.state.Committee; +import org.ethereum.sharding.pubsub.BeaconBlockAttested; import org.ethereum.sharding.pubsub.BeaconBlockImported; import org.ethereum.sharding.pubsub.Publisher; import org.ethereum.sharding.processing.BeaconChain; @@ -63,6 +65,7 @@ public class ValidatorServiceImpl implements ValidatorService { private static final Logger logger = LoggerFactory.getLogger("validator"); BeaconProposer proposer; + BeaconAttester attester; BeaconChain beaconChain; Publisher publisher; ValidatorConfig config; @@ -77,9 +80,10 @@ public class ValidatorServiceImpl implements ValidatorService { private ChainHead head; private byte[] mainChainRef; - public ValidatorServiceImpl(BeaconProposer proposer, BeaconChain beaconChain, Publisher publisher, - ValidatorConfig config, Ethereum ethereum, BlockStore blockStore) { + public ValidatorServiceImpl(BeaconProposer proposer, BeaconAttester attester, BeaconChain beaconChain, + Publisher publisher, ValidatorConfig config, Ethereum ethereum, BlockStore blockStore) { this.proposer = proposer; + this.attester = attester; this.beaconChain = beaconChain; this.publisher = publisher; this.config = config; @@ -166,35 +170,41 @@ private void submitIfAssigned(Committee[][] committees) { // proposer = committee[X % len(committee)], X = slotNumber // taken from the spec if (slotNumber % index.getCommitteeSize() == index.getArrayIdx()) { - this.propose(slotNumber, index.getValidatorIdx()); + this.propose(slotNumber, index); } else { - this.attest(slotNumber, index.getValidatorIdx()); + this.attest(slotNumber, index); } } } @Override - public void propose(long slotNumber, int validatorIdx) { - long delay = submit(0L, slotNumber, validatorIdx, () -> { - BeaconProposer.Input input = new BeaconProposer.Input(slotNumber, head, mainChainRef); - Beacon newBlock = proposer.createNewBlock(input, pubKeysMap.get(validatorIdx)); + public void propose(long slotNumber, Committee.Index index) { + long delay = submit(0L, slotNumber, index.getValidatorIdx(), () -> { + BeaconProposer.Input input = new BeaconProposer.Input(slotNumber, index, head, mainChainRef); + Beacon newBlock = proposer.createNewBlock(input, pubKeysMap.get(index.getValidatorIdx())); beaconChain.insert(newBlock); return newBlock; }); - if (delay >= 0) logger.info("Proposer {}: schedule new slot #{} in {}ms", validatorIdx, slotNumber, delay); + if (delay >= 0) logger.info("Proposer {}: schedule new slot #{} in {}ms", index.getValidatorIdx(), slotNumber, delay); } @Override - public void attest(long slotNumber, int validatorIdx) { + public void attest(long slotNumber, Committee.Index index) { // attester's job should be triggered in the middle of slot's time period - long delay = submit(SLOT_DURATION / 2, slotNumber, validatorIdx, () -> { - // TODO add attester routine here - logger.info("Fake attestation on slot #{}", slotNumber); - return (Void) null; + long delay = submit(SLOT_DURATION / 2, slotNumber, index.getValidatorIdx(), () -> { + AttestationRecord attestation = attester.attestBlock( + slotNumber, + index, + beaconChain.getCanonicalHead(), // FIXME: how could we be sure that it's a correct block? + pubKeysMap.get(index.getValidatorIdx()) + ); + logger.info("Attestation by #{} on slot #{}", index.getValidatorIdx(), slotNumber); + publisher.publish(new BeaconBlockAttested(attestation)); + return attestation; }); - if (delay >= 0) logger.info("Attester {}: schedule new slot #{} in {}ms", validatorIdx, slotNumber, delay); + if (delay >= 0) logger.info("Attester {}: schedule new slot #{} in {}ms", index.getValidatorIdx(), slotNumber, delay); } long submit(long delayShiftMillis, long slotNumber, int validatorIdx, Supplier supplier) { From 1eb2533b3ece88ab10117feebf38da6955c9c2e9 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Thu, 11 Oct 2018 23:26:30 +0300 Subject: [PATCH 02/14] Fix missed semicolomn --- .../org/ethereum/sharding/processing/state/ActiveState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java index d39f2b5d23..ffca93100b 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java @@ -109,7 +109,7 @@ public byte[] getHash() { public String toString() { StringBuilder builder = new StringBuilder().append("ActiveState{") .append("pendingAttestations=[").append(pendingAttestations.length).append(" item(s)]") - .append(", recentBlockHashes=[...") + .append(", recentBlockHashes=[..."); for (int i = Math.max(0, recentBlockHashes.length - 3); i < recentBlockHashes.length; ++i) { builder.append(", ").append(toHexString(recentBlockHashes[i])); From 056178d4cd1eaf178dbb2915810fa9b804535054 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Thu, 11 Oct 2018 23:26:53 +0300 Subject: [PATCH 03/14] Store AttestationRecord as List + add AttestationRecords to encoded beacon --- .../org/ethereum/sharding/domain/Beacon.java | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java index 73488779b3..9520fc6af4 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/Beacon.java @@ -22,12 +22,17 @@ import org.ethereum.util.ByteUtil; import org.ethereum.util.FastByteComparisons; import org.ethereum.util.RLP; +import org.ethereum.util.RLPElement; import org.ethereum.util.RLPList; import org.spongycastle.util.encoders.Hex; import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; import static org.ethereum.crypto.HashUtil.blake2b; +import static org.ethereum.util.ByteUtil.ZERO_BYTE_ARRAY; +import static org.ethereum.util.ByteUtil.isSingleZero; /** * Beacon chain block structure. @@ -48,10 +53,10 @@ public class Beacon { /* Slot number */ private long slotNumber; /* Attestations */ - private AttestationRecord[] attestations; + private List attestations; public Beacon(byte[] parentHash, byte[] randaoReveal, byte[] mainChainRef, byte[] stateHash, - long slotNumber, AttestationRecord[] attestations) { + long slotNumber,List attestations) { this.parentHash = parentHash; this.randaoReveal = randaoReveal; this.mainChainRef = mainChainRef; @@ -68,16 +73,23 @@ public Beacon(byte[] rlp) { this.stateHash = items.get(3).getRLPData(); this.slotNumber = ByteUtil.bytesToBigInteger(items.get(4).getRLPData()).longValue(); - RLPList attestationsRlp = RLP.unwrapList(items.get(5).getRLPData()); - this.attestations = new AttestationRecord[attestationsRlp.size()]; - for (int i = 0; i < attestationsRlp.size(); ++i) { - attestations[i] = new AttestationRecord(attestationsRlp.get(i).getRLPData()); + this.attestations = new ArrayList<>(); + if (!isSingleZero(items.get(5).getRLPData())) { + RLPList attestationsRlp = RLP.unwrapList(items.get(5).getRLPData()); + for (RLPElement anAttestationsRlp : attestationsRlp) { + attestations.add(new AttestationRecord(anAttestationsRlp.getRLPData())); + } } } public byte[] getEncoded() { + byte[][] encodedAttestations = new byte[attestations.size()][]; + for (int i = 0; i < attestations.size(); i++) + encodedAttestations[i] = attestations.get(i).getEncoded(); + return RLP.wrapList(parentHash, randaoReveal, mainChainRef, stateHash, - BigInteger.valueOf(slotNumber).toByteArray()); + BigInteger.valueOf(slotNumber).toByteArray(), + encodedAttestations.length == 0 ? ZERO_BYTE_ARRAY : RLP.wrapList(encodedAttestations)); } public byte[] getHash() { @@ -104,7 +116,7 @@ public long getSlotNumber() { return slotNumber; } - public AttestationRecord[] getAttestations() { + public List getAttestations() { return attestations; } @@ -116,14 +128,14 @@ public void setStateHash(byte[] stateHash) { this.stateHash = stateHash; } - public void setAttestations(AttestationRecord[] attestations) { + public void setAttestations(List attestations) { this.attestations = attestations; } @Override public boolean equals(Object other) { if (this == other) return true; - if (other == null || !(other instanceof Beacon)) return false; + if (!(other instanceof Beacon)) return false; return FastByteComparisons.equal(this.getHash(), ((Beacon) other).getHash()); } From 0eddf7b5a11a6dea2f4c1ef8b9abbba6b77d2b43 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Thu, 11 Oct 2018 23:47:16 +0300 Subject: [PATCH 04/14] Moved to list instead of array --- .../processing/state/ActiveState.java | 63 +++++++++++-------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java index ffca93100b..b7619d6933 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java @@ -19,10 +19,15 @@ import org.ethereum.util.ByteUtil; import org.ethereum.util.RLP; +import org.ethereum.util.RLPElement; import org.ethereum.util.RLPList; +import java.util.ArrayList; +import java.util.List; + import static org.ethereum.crypto.HashUtil.blake2b; import static org.ethereum.sharding.processing.consensus.BeaconConstants.CYCLE_LENGTH; +import static org.ethereum.util.ByteUtil.isSingleZero; import static org.ethereum.util.ByteUtil.toHexString; /** @@ -30,16 +35,16 @@ */ public class ActiveState { // Attestations that have not yet been processed - private final AttestationRecord[] pendingAttestations; + private final List pendingAttestations; // Most recent 2 * CYCLE_LENGTH block hashes, older to newer - private final byte[][] recentBlockHashes; + private final List recentBlockHashes; // RANDAO state private final byte[] randaoMix; // TODO: Add pending_specials - public ActiveState(AttestationRecord[] pendingAttestations, byte[][] recentBlockHashes, byte[] randaoMix) { + public ActiveState(List pendingAttestations, List recentBlockHashes, byte[] randaoMix) { this.pendingAttestations = pendingAttestations; this.recentBlockHashes = recentBlockHashes; this.randaoMix = randaoMix; @@ -50,11 +55,10 @@ public ActiveState(AttestationRecord[] pendingAttestations, byte[][] recentBlock * and block hashes filled with `00`*32 */ public static ActiveState createEmpty() { - AttestationRecord[] pendingAttestations = new AttestationRecord[0]; - int size = CYCLE_LENGTH * 2; - byte[][] recentBlockHashes = new byte[size][]; - for (int i = 0; i < size; ++i) { - recentBlockHashes[i] = new byte[32]; + List pendingAttestations = new ArrayList<>(); + List recentBlockHashes = new ArrayList<>(); + for (int i = 0; i < (CYCLE_LENGTH * 2); ++i) { + recentBlockHashes.add(new byte[32]); } return new ActiveState(pendingAttestations, recentBlockHashes, new byte[32]); @@ -63,24 +67,28 @@ public static ActiveState createEmpty() { public ActiveState(byte[] encoded) { RLPList list = RLP.unwrapList(encoded); - RLPList attestationList = RLP.unwrapList(list.get(0).getRLPData()); - this.pendingAttestations = new AttestationRecord[attestationList.size()]; - for (int i = 0; i < attestationList.size(); i++) - pendingAttestations[i] = new AttestationRecord(attestationList.get(i).getRLPData()); + this.pendingAttestations = new ArrayList<>(); + if (!isSingleZero(list.get(0).getRLPData())) { + RLPList attestationList = RLP.unwrapList(list.get(0).getRLPData()); + for (RLPElement attestationRlp : attestationList) + pendingAttestations.add(new AttestationRecord(attestationRlp.getRLPData())); + } - RLPList hashesList = RLP.unwrapList(list.get(1).getRLPData()); - this.recentBlockHashes = new byte[hashesList.size()][]; - for (int i = 0; i < hashesList.size(); i++) - recentBlockHashes[i] = hashesList.get(i).getRLPData(); + this.recentBlockHashes = new ArrayList<>(); + if (!isSingleZero(list.get(1).getRLPData())) { + RLPList hashesList = RLP.unwrapList(list.get(1).getRLPData()); + for (RLPElement hashRlp : hashesList) + recentBlockHashes.add(hashRlp.getRLPData()); + } this.randaoMix = list.get(2).getRLPData(); } - public AttestationRecord[] getPendingAttestations() { + public List getPendingAttestations() { return pendingAttestations; } - public byte[][] getRecentBlockHashes() { + public List getRecentBlockHashes() { return recentBlockHashes; } @@ -89,14 +97,17 @@ public byte[] getRandaoMix() { } public byte[] getEncoded() { - byte[][] encodedAttestations = new byte[pendingAttestations.length][]; + byte[][] encodedAttestations = new byte[pendingAttestations.size()][]; + for (int i = 0; i < pendingAttestations.size(); i++) + encodedAttestations[i] = pendingAttestations.get(i).getEncoded(); - for (int i = 0; i < pendingAttestations.length; i++) - encodedAttestations[i] = pendingAttestations[i].getEncoded(); + byte[][] encodedHashes = new byte[recentBlockHashes.size()][]; + for (int i = 0; i < recentBlockHashes.size(); i++) + encodedHashes[i] = RLP.encodeElement(recentBlockHashes.get(i)); return RLP.wrapList( - pendingAttestations.length > 0 ? RLP.wrapList(encodedAttestations) : ByteUtil.ZERO_BYTE_ARRAY, - recentBlockHashes.length > 0 ? RLP.wrapList(recentBlockHashes) : ByteUtil.ZERO_BYTE_ARRAY, + pendingAttestations.size() > 0 ? RLP.wrapList(encodedAttestations) : ByteUtil.ZERO_BYTE_ARRAY, + recentBlockHashes.size() > 0 ? RLP.wrapList(encodedHashes) : ByteUtil.ZERO_BYTE_ARRAY, RLP.encodeElement(randaoMix) ); } @@ -108,11 +119,11 @@ public byte[] getHash() { @Override public String toString() { StringBuilder builder = new StringBuilder().append("ActiveState{") - .append("pendingAttestations=[").append(pendingAttestations.length).append(" item(s)]") + .append("pendingAttestations=[").append(pendingAttestations.size()).append(" item(s)]") .append(", recentBlockHashes=[..."); - for (int i = Math.max(0, recentBlockHashes.length - 3); i < recentBlockHashes.length; ++i) { - builder.append(", ").append(toHexString(recentBlockHashes[i])); + for (int i = Math.max(0, recentBlockHashes.size() - 3); i < recentBlockHashes.size(); ++i) { + builder.append(", ").append(toHexString(recentBlockHashes.get(i))); } builder.append("]"); From c330b0e4366b856eb7bac37503899662fdcfcf3b Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Thu, 11 Oct 2018 23:47:32 +0300 Subject: [PATCH 05/14] Moved to BitSet instead of byte[] --- .../org/ethereum/sharding/util/Bitfield.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java index 1fc4961086..5791c406e5 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java @@ -17,7 +17,8 @@ */ package org.ethereum.sharding.util; -import java.util.Arrays; +import java.util.BitSet; +import java.util.List; /** * Bitfield utility methods @@ -32,8 +33,8 @@ public class Bitfield { * @param validatorsCount Number of attesters * @return empty bitfield with correct length */ - public static byte[] createEmpty(int validatorsCount) { - return new byte[calcLength(validatorsCount)]; + public static BitSet createEmpty(int validatorsCount) { + return new BitSet(calcLength(validatorsCount)); } // TODO: Add test @@ -54,11 +55,9 @@ public static int calcLength(int num) { * @param index Index number of attester * @return bitfield with vote in place */ - public static byte[] markVote(final byte[] bitfield, int index) { - byte[] newBitfield = Arrays.copyOf(bitfield, bitfield.length); - int byteIndex = index / Byte.SIZE; - int bitIndex = index % Byte.SIZE; - newBitfield[byteIndex] |= 128 >> bitIndex; + public static BitSet markVote(final BitSet bitfield, int index) { + BitSet newBitfield = (BitSet) bitfield.clone(); + newBitfield.set(index); return newBitfield; } @@ -67,11 +66,8 @@ public static byte[] markVote(final byte[] bitfield, int index) { * @param bitfield Bitfield * @param index Index number of attester */ - public static boolean hasVoted(byte[] bitfield, int index) { - int byteIndex = index / Byte.SIZE; - int bitIndex = index % Byte.SIZE; - - return (bitfield[byteIndex] & (128 >> bitIndex)) == 1; + public static boolean hasVoted(BitSet bitfield, int index) { + return bitfield.get(index); } /** @@ -79,9 +75,9 @@ public static boolean hasVoted(byte[] bitfield, int index) { * @param bitfield Bitfield * @return number of votes */ - public static int calcVotes(byte[] bitfield) { + public static int calcVotes(BitSet bitfield) { int votes = 0; - for (int i = 0; i < (bitfield.length * Byte.SIZE); ++i) { + for (int i = 0; i < bitfield.size(); ++i) { if (hasVoted(bitfield, i)) ++votes; } @@ -94,12 +90,16 @@ public static int calcVotes(byte[] bitfield) { * @param bitfields Bitfields * @return All bitfields aggregated using OR */ - public static byte[] orBitfield(byte[][] bitfields) { - int bitfieldLen = bitfields[0].length; - byte[] aggBitfield = new byte[bitfieldLen]; + public static BitSet orBitfield(List bitfields) { + if (bitfields.isEmpty()) return null; + + int bitfieldLen = bitfields.get(0).size(); + BitSet aggBitfield = new BitSet(bitfieldLen); for (int i = 0; i < bitfieldLen; ++i) { - for (byte[] bitfield : bitfields) { - aggBitfield[i] |= bitfield[i]; + for (BitSet bitfield : bitfields) { + if (aggBitfield.get(i) | bitfield.get(i)) { + aggBitfield.set(i); + } } } From 9081fc1f155785a62b64ef484ff7d4b3c5021ddb Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Thu, 11 Oct 2018 23:51:42 +0300 Subject: [PATCH 06/14] Move bitfield to BitSet --- .../processing/state/AttestationRecord.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java index c5d765bbae..e04c11ae01 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java @@ -23,6 +23,7 @@ import java.math.BigInteger; import java.util.Arrays; +import java.util.BitSet; import static org.ethereum.util.ByteUtil.bigIntegerToBytes; import static org.ethereum.util.ByteUtil.byteArrayToLong; @@ -46,7 +47,7 @@ public class AttestationRecord { // Block hash in the shard that we are attesting to private final byte[] shardBlockHash; // Who is participating - private final byte[] attesterBitfield; + private final BitSet attesterBitfield; // Last justified block private final long justifiedSlot; private final byte[] justifiedBlockHash; @@ -54,7 +55,7 @@ public class AttestationRecord { private final BigInteger[] aggregateSig; // Defined by two BigIntegers? public AttestationRecord(long slot, int shardId, byte[][] obliqueParentHashes, byte[] shardBlockHash, - byte[] attesterBitfield, long justifiedSlot, byte[] justifiedBlockHash, + BitSet attesterBitfield, long justifiedSlot, byte[] justifiedBlockHash, BigInteger[] aggregateSig) { this.slot = slot; this.shardId = shardId; @@ -78,7 +79,7 @@ public AttestationRecord(byte[] encoded) { this.obliqueParentHashes[i] = hashesList.get(i).getRLPData(); this.shardBlockHash = list.get(3).getRLPData(); - this.attesterBitfield = list.get(4).getRLPData(); + this.attesterBitfield = BitSet.valueOf(list.get(4).getRLPData()); this.justifiedSlot = byteArrayToLong(list.get(5).getRLPData()); this.justifiedBlockHash = list.get(6).getRLPData(); @@ -104,7 +105,7 @@ public byte[] getShardBlockHash() { return shardBlockHash; } - public byte[] getAttesterBitfield() { + public BitSet getAttesterBitfield() { return attesterBitfield; } @@ -129,7 +130,7 @@ public byte[] getEncoded() { intToBytesNoLeadZeroes(shardId), RLP.wrapList(obliqueParentHashes), shardBlockHash, - attesterBitfield, + attesterBitfield.toByteArray(), longToBytesNoLeadZeroes(justifiedSlot), justifiedBlockHash, RLP.wrapList(encodedAggSig)); @@ -142,7 +143,7 @@ public boolean equals(Object o) { AttestationRecord that = (AttestationRecord) o; return slot == that.slot && shardId == that.shardId && - Arrays.equals(attesterBitfield, that.attesterBitfield) && + attesterBitfield == that.attesterBitfield && justifiedSlot == that.justifiedSlot && Arrays.equals(obliqueParentHashes, that.obliqueParentHashes) && Arrays.equals(shardBlockHash, that.shardBlockHash) && @@ -158,7 +159,7 @@ public String toString() { .append(", shardId=").append(shardId) .append(", obliqueParentHashes=[").append(obliqueParentHashes.length).append(" item(s)]") .append(", shardBlockHash=").append(toHexString(shardBlockHash)) - .append(", attesterBitfield=").append(toHexString(attesterBitfield)) + .append(", attesterBitfield=").append(attesterBitfield) .append(", justifiedSlot=").append(justifiedSlot) .append(", justifiedBlockHash=").append(toHexString(justifiedBlockHash)) .append(", aggregateSig=[").append(aggregateSig.length).append(" item(s)]}"); From 33654d93b8c3838d821b80acfc461e87bf20bb4f Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Fri, 12 Oct 2018 00:20:20 +0300 Subject: [PATCH 07/14] Fixing attestation logic --- .../sharding/validator/BeaconAttester.java | 29 +++++++++++++++--- .../validator/BeaconAttesterImpl.java | 30 ++++++------------- .../sharding/validator/BeaconProposer.java | 8 ++--- .../validator/BeaconProposerImpl.java | 20 ++----------- .../sharding/validator/ValidatorService.java | 2 +- .../validator/ValidatorServiceImpl.java | 20 +++++-------- .../validator/BeaconProposerTest.java | 5 +++- 7 files changed, 52 insertions(+), 62 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java index 6154363e16..6756359e66 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttester.java @@ -19,6 +19,7 @@ import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.processing.state.AttestationRecord; +import org.ethereum.sharding.processing.state.BeaconState; import org.ethereum.sharding.processing.state.Committee; /** @@ -29,11 +30,31 @@ public interface BeaconAttester { /** * Attests block * - * @param slotNumber Current slot number - * @param index Commitee index with validator id etc - * @param block Block to attest + * @param in Input data * @param pubKey Public key of the attestation validator * @return Attestation record */ - AttestationRecord attestBlock(long slotNumber, Committee.Index index, Beacon block, byte[] pubKey); + AttestationRecord attestBlock(Input in, byte[] pubKey); + + + class Input { + long slotNumber; + Committee.Index index; + Beacon block; + BeaconState state; + + public Input(long slotNumber, Committee.Index index, Beacon block, BeaconState state) { + this.slotNumber = slotNumber; + this.index = index; + this.block = block; + this.state = state; + } + + public Input(long slotNumber, Committee.Index index, ValidatorService.ChainHead head) { + this.slotNumber = slotNumber; + this.index = index; + this.block = head.block; + this.state = head.state; + } + } } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java index 7a482f431d..5ea4e15cc8 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java @@ -47,32 +47,20 @@ public BeaconAttesterImpl(StateRepository repository, BeaconStore store, Validat } @Override - public AttestationRecord attestBlock(long slotNumber, Committee.Index index, Beacon block, byte[] pubKey) { - if (block.getAttestations().length == 0) { - throw new RuntimeException("Block should have at least 1 attestation of its proposer"); - } - - // validate that block doesn't contain our attestation - for (AttestationRecord attestationRecord : block.getAttestations()) { - if (Bitfield.hasVoted(attestationRecord.getAttesterBitfield(), index.getValidatorIdx())) { - throw new RuntimeException("Shouldn't attest block again"); - } - } - - BeaconState headState = repository.get(store.getCanonicalHead().getHash()); - long lastJustified = headState.getCrystallizedState().getFinality().getLastJustifiedSlot(); + public AttestationRecord attestBlock(Input in, byte[] pubKey) { + long lastJustified = in.state.getCrystallizedState().getFinality().getLastJustifiedSlot(); AttestationRecord attestationRecord = new AttestationRecord( - slotNumber, - index.getShardId(), - new byte[0][0], // FIXME: obliqueParentHashes - new byte[32], // FIXME: shardBlockHash?? - Bitfield.markVote(Bitfield.createEmpty(index.getCommitteeSize()), index.getValidatorIdx()), + in.slotNumber, + in.index.getShardId(), + new byte[0][0], + in.block.getHash(), + Bitfield.markVote(Bitfield.createEmpty(in.index.getCommitteeSize()), in.index.getValidatorIdx()), lastJustified, store.getCanonicalByNumber(lastJustified) == null ? new byte[32] : store.getCanonicalByNumber(lastJustified).getHash(), - Sign.aggSigns(new byte[][] {Sign.sign(block.getEncoded(), pubKey)}) + Sign.aggSigns(new byte[][] {Sign.sign(in.block.getEncoded(), pubKey)}) ); - logger.info("Block {} attested by #{} in slot {} ", block, index.getValidatorIdx(), slotNumber); + logger.info("Block {} attested by #{} in slot {} ", in.block, in.index.getValidatorIdx(), in.slotNumber); return attestationRecord; } } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java index e7ac46ed9b..e80c870fa5 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposer.java @@ -19,7 +19,6 @@ import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.processing.state.BeaconState; -import org.ethereum.sharding.processing.state.Committee; /** * Beacon chain block proposer. @@ -49,22 +48,19 @@ public interface BeaconProposer { class Input { long slotNumber; - Committee.Index index; Beacon parent; BeaconState state; byte[] mainChainRef; - public Input(long slotNumber, Committee.Index index, Beacon parent, BeaconState state, byte[] mainChainRef) { + public Input(long slotNumber, Beacon parent, BeaconState state, byte[] mainChainRef) { this.slotNumber = slotNumber; - this.index = index; this.parent = parent; this.state = state; this.mainChainRef = mainChainRef; } - public Input(long slotNumber, Committee.Index index, ValidatorService.ChainHead head, byte[] mainChainRef) { + public Input(long slotNumber, ValidatorService.ChainHead head, byte[] mainChainRef) { this.slotNumber = slotNumber; - this.index = index; this.parent = head.block; this.state = head.state; this.mainChainRef = mainChainRef; diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java index 01c9231417..91c0344996 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java @@ -34,6 +34,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; + /** * Default implementation of {@link BeaconProposer}. * @@ -81,25 +83,9 @@ byte[] randaoReveal(BeaconState state, byte[] pubKey) { @Override public Beacon createNewBlock(Input in, byte[] pubKey) { Beacon block = new Beacon(in.parent.getHash(), randaoReveal(in.state, pubKey), in.mainChainRef, - HashUtil.EMPTY_DATA_HASH, in.slotNumber, new AttestationRecord[0]); + HashUtil.EMPTY_DATA_HASH, in.slotNumber, new ArrayList<>()); BeaconState newState = stateTransition.applyBlock(block, in.state); block.setStateHash(newState.getHash()); - - AttestationRecord[] attestationRecords = new AttestationRecord[1]; - byte[] emptyBitfield = Bitfield.createEmpty(in.index.getCommitteeSize()); - long lastJustified = in.state.getCrystallizedState().getFinality().getLastJustifiedSlot(); - attestationRecords[0] = new AttestationRecord( - in.slotNumber, - in.index.getShardId(), - new byte[0][0], - new byte[32], // FIXME: shardBlockHash?? - Bitfield.markVote(emptyBitfield, in.index.getValidatorIdx()), - lastJustified, - store.getCanonicalByNumber(lastJustified) == null ? new byte[32] : store.getCanonicalByNumber(lastJustified).getHash(), - Sign.aggSigns(new byte[][] {Sign.sign(block.getEncoded(), pubKey)}) - ); - block.setAttestations(attestationRecords); - logger.info("New block created {}", block); return block; } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java index 1342f8aa14..9a941827b4 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorService.java @@ -46,7 +46,7 @@ default void init(ChainHead head, byte[]... pubKeys) {} * Submits a task to propose block with given slot number. * Thread safe. */ - default void propose(long slotNumber, Committee.Index index) {} + default void propose(long slotNumber, int validatorIdx) {} /** * Submits a task to make an attestation in a given slot number. diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java index b1aefa3714..3cdfde89ec 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/ValidatorServiceImpl.java @@ -164,7 +164,7 @@ private void submitIfAssigned(Committee[][] committees) { // proposer = committee[X % len(committee)], X = slotNumber // taken from the spec if (slotNumber % index.getCommitteeSize() == index.getArrayIdx()) { - this.propose(slotNumber, index); + this.propose(slotNumber, index.getValidatorIdx()); } else { this.attest(slotNumber, index); } @@ -172,27 +172,23 @@ private void submitIfAssigned(Committee[][] committees) { } @Override - public void propose(long slotNumber, Committee.Index index) { - long delay = submit(0L, slotNumber, index.getValidatorIdx(), () -> { - BeaconProposer.Input input = new BeaconProposer.Input(slotNumber, index, head, mainChainRef); - Beacon newBlock = proposer.createNewBlock(input, pubKeysMap.get(index.getValidatorIdx())); + public void propose(long slotNumber, int validatorIdx) { + long delay = submit(0L, slotNumber, validatorIdx, () -> { + BeaconProposer.Input input = new BeaconProposer.Input(slotNumber, head, mainChainRef); + Beacon newBlock = proposer.createNewBlock(input, pubKeysMap.get(validatorIdx)); beaconChain.insert(newBlock); return newBlock; }); - if (delay >= 0) logger.info("Proposer {}: schedule new slot #{} in {}ms", index.getValidatorIdx(), slotNumber, delay); + if (delay >= 0) logger.info("Proposer {}: schedule new slot #{} in {}ms", validatorIdx, slotNumber, delay); } @Override public void attest(long slotNumber, Committee.Index index) { // attester's job should be triggered in the middle of slot's time period long delay = submit(SLOT_DURATION / 2, slotNumber, index.getValidatorIdx(), () -> { - AttestationRecord attestation = attester.attestBlock( - slotNumber, - index, - beaconChain.getCanonicalHead(), // FIXME: how could we be sure that it's a correct block? - pubKeysMap.get(index.getValidatorIdx()) - ); + BeaconAttester.Input input = new BeaconAttester.Input(slotNumber, index, head); + AttestationRecord attestation = attester.attestBlock(input, pubKeysMap.get(index.getValidatorIdx())); logger.info("Attestation by #{} on slot #{}", index.getValidatorIdx(), slotNumber); publisher.publish(new BeaconBlockAttested(attestation)); return attestation; diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java index 32c607f2da..83275034f0 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java @@ -24,6 +24,8 @@ import org.ethereum.sharding.domain.BeaconGenesis; import org.ethereum.sharding.processing.consensus.NoTransition; import org.ethereum.sharding.processing.consensus.StateTransition; +import org.ethereum.sharding.processing.db.BeaconStore; +import org.ethereum.sharding.processing.db.IndexedBeaconStore; import org.ethereum.sharding.processing.state.BeaconState; import org.ethereum.sharding.processing.state.BeaconStateRepository; import org.ethereum.sharding.processing.state.StateRepository; @@ -106,10 +108,11 @@ static Helper newInstance() { StateRepository repository = new BeaconStateRepository(new HashMapDB<>(), new HashMapDB<>(), new HashMapDB<>(), new HashMapDB<>()); + BeaconStore store = new IndexedBeaconStore(new HashMapDB<>(), new HashMapDB<>()); StateTransition stateTransition = new NoTransition(); Helper helper = new Helper(); - helper.proposer = new BeaconProposerImpl(randao, repository, stateTransition, + helper.proposer = new BeaconProposerImpl(randao, repository, store, stateTransition, ValidatorConfig.DISABLED) { @Override From 93dbd508eefb6480a5860c186becca7e77e91260 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Sun, 14 Oct 2018 23:17:22 +0300 Subject: [PATCH 08/14] Signature refactored to simplify adding real signature impl in future --- .../sharding/config/BeaconConfig.java | 8 +++- .../{util/Sign.java => crypto/DummySign.java} | 34 +++++++++------- .../org/ethereum/sharding/crypto/Sign.java | 39 +++++++++++++++++++ .../processing/state/AttestationRecord.java | 24 ++++++------ .../validator/BeaconAttesterImpl.java | 12 +++--- .../validator/BeaconProposerImpl.java | 4 -- 6 files changed, 83 insertions(+), 38 deletions(-) rename ethereumj-core/src/main/java/org/ethereum/sharding/{util/Sign.java => crypto/DummySign.java} (53%) create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java b/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java index 337288b3e7..aebeefdc1c 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java @@ -35,6 +35,8 @@ import org.ethereum.db.TransactionStore; import org.ethereum.facade.Ethereum; import org.ethereum.manager.WorldManager; +import org.ethereum.sharding.crypto.DummySign; +import org.ethereum.sharding.crypto.Sign; import org.ethereum.sharding.pubsub.Publisher; import org.ethereum.sharding.manager.ShardingWorldManager; import org.ethereum.sharding.processing.BeaconChain; @@ -224,7 +226,7 @@ public BeaconProposer beaconProposer() { @Bean public BeaconAttester beaconAttester() { - return new BeaconAttesterImpl(beaconStateRepository(), beaconStore(), validatorConfig()); + return new BeaconAttesterImpl(beaconStateRepository(), beaconStore(), validatorConfig(), sign()); } @Bean @@ -251,6 +253,10 @@ public DepositAuthority depositAuthority() { return new UnsecuredDepositAuthority(validatorConfig()); } + public Sign sign() { + return new DummySign(); + } + public static class Enabled implements Condition { @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Sign.java b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java similarity index 53% rename from ethereumj-core/src/main/java/org/ethereum/sharding/util/Sign.java rename to ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java index 1bcbfe27c3..20691d236a 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Sign.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java @@ -15,37 +15,41 @@ * You should have received a copy of the GNU Lesser General Public License * along with the ethereumJ library. If not, see . */ -package org.ethereum.sharding.util; +package org.ethereum.sharding.crypto; import java.math.BigInteger; import static org.ethereum.crypto.HashUtil.sha3; /** - * Signature utilities + * Dummy signature implementation without real crypto underneath */ -public class Sign { +public class DummySign implements Sign { - public static byte[] sign(byte[] msg, byte[] privateKey) { - // TODO: BLS should be here - return sha3(msg, privateKey); + public Signature sign(byte[] msg, byte[] privateKey) { + byte[] rSource = sha3(privateKey); + byte[] sSource = sha3(msg, privateKey); + Signature res = new Signature(); + res.r = new BigInteger(rSource); + res.s = new BigInteger(sSource); + + return res; } - public static boolean verify(byte[] signature, byte[] publicKey) { - // TODO: real BLS verification should be here + public boolean verify(Signature signature, byte[] publicKey) { return true; } - public static BigInteger[] aggSigns(byte[][] signatures) { - // TODO: real BLS signature aggregation instead of XOR - int signatureLen = signatures[0].length; - byte[] aggSignature = new byte[signatureLen]; + public Signature aggSigns(Signature[] signatures) { + int signatureLen = signatures.length; + Signature aggSignature = new Signature(); for (int i = 0; i < signatureLen; ++i) { - for (byte[] signature : signatures) { - aggSignature[i] ^= signature[i]; + for (Signature signature : signatures) { + aggSignature.r = aggSignature.r.xor(signature.r); + aggSignature.s = aggSignature.s.xor(signature.s); } } - return new BigInteger[] {new BigInteger(1, aggSignature), BigInteger.ONE}; + return aggSignature; } } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java new file mode 100644 index 0000000000..5b45cd4345 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java @@ -0,0 +1,39 @@ +package org.ethereum.sharding.crypto; + +import java.math.BigInteger; +import java.util.Objects; + +/** + * Signature utilities + * Signature should be implemented using BLS + */ +public interface Sign { + + Signature sign(byte[] msg, byte[] privateKey); + + boolean verify(Signature signature, byte[] publicKey); + + Signature aggSigns(Signature[] signatures); + + class Signature { + public BigInteger r; + public BigInteger s; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Signature signature = (Signature) o; + return Objects.equals(r, signature.r) && + Objects.equals(s, signature.s); + } + + @Override + public String toString() { + return "Signature{" + + "r=" + r + + ", s=" + s + + '}'; + } + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java index e04c11ae01..f05e27a7a5 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java @@ -17,11 +17,11 @@ */ package org.ethereum.sharding.processing.state; +import org.ethereum.sharding.crypto.Sign; import org.ethereum.util.ByteUtil; import org.ethereum.util.RLP; import org.ethereum.util.RLPList; -import java.math.BigInteger; import java.util.Arrays; import java.util.BitSet; @@ -52,11 +52,11 @@ public class AttestationRecord { private final long justifiedSlot; private final byte[] justifiedBlockHash; // The actual signature - private final BigInteger[] aggregateSig; // Defined by two BigIntegers? + private final Sign.Signature aggregateSig; public AttestationRecord(long slot, int shardId, byte[][] obliqueParentHashes, byte[] shardBlockHash, BitSet attesterBitfield, long justifiedSlot, byte[] justifiedBlockHash, - BigInteger[] aggregateSig) { + Sign.Signature aggregateSig) { this.slot = slot; this.shardId = shardId; this.obliqueParentHashes = obliqueParentHashes; @@ -84,9 +84,9 @@ public AttestationRecord(byte[] encoded) { this.justifiedBlockHash = list.get(6).getRLPData(); RLPList sigList = RLP.unwrapList(list.get(7).getRLPData()); - this.aggregateSig = new BigInteger[sigList.size()]; - for (int i = 0; i < sigList.size(); i++) - this.aggregateSig[i] = ByteUtil.bytesToBigInteger(hashesList.get(i).getRLPData()); + this.aggregateSig = new Sign.Signature(); + this.aggregateSig.r = ByteUtil.bytesToBigInteger(sigList.get(0).getRLPData()); + this.aggregateSig.s = ByteUtil.bytesToBigInteger(sigList.get(1).getRLPData()); } public long getSlot() { @@ -117,14 +117,14 @@ public byte[] getJustifiedBlockHash() { return justifiedBlockHash; } - public BigInteger[] getAggregateSig() { + public Sign.Signature getAggregateSig() { return aggregateSig; } public byte[] getEncoded() { - byte[][] encodedAggSig = new byte[aggregateSig.length][]; - for (int i = 0; i < aggregateSig.length; i++) - encodedAggSig[i] = bigIntegerToBytes(aggregateSig[i]); + byte[][] encodedAggSig = new byte[2][]; + encodedAggSig[0] = bigIntegerToBytes(aggregateSig.r); + encodedAggSig[1] = bigIntegerToBytes(aggregateSig.s); return RLP.wrapList(longToBytesNoLeadZeroes(slot), intToBytesNoLeadZeroes(shardId), @@ -148,7 +148,7 @@ public boolean equals(Object o) { Arrays.equals(obliqueParentHashes, that.obliqueParentHashes) && Arrays.equals(shardBlockHash, that.shardBlockHash) && Arrays.equals(justifiedBlockHash, that.justifiedBlockHash) && - Arrays.equals(aggregateSig, that.aggregateSig); + aggregateSig == that.aggregateSig; } @Override @@ -162,7 +162,7 @@ public String toString() { .append(", attesterBitfield=").append(attesterBitfield) .append(", justifiedSlot=").append(justifiedSlot) .append(", justifiedBlockHash=").append(toHexString(justifiedBlockHash)) - .append(", aggregateSig=[").append(aggregateSig.length).append(" item(s)]}"); + .append(", aggregateSig=[").append(aggregateSig).append("}"); return builder.toString(); } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java index 5ea4e15cc8..eddfe73f8e 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconAttesterImpl.java @@ -18,14 +18,11 @@ package org.ethereum.sharding.validator; import org.ethereum.sharding.config.ValidatorConfig; -import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.processing.db.BeaconStore; import org.ethereum.sharding.processing.state.AttestationRecord; -import org.ethereum.sharding.processing.state.BeaconState; -import org.ethereum.sharding.processing.state.Committee; import org.ethereum.sharding.processing.state.StateRepository; import org.ethereum.sharding.util.Bitfield; -import org.ethereum.sharding.util.Sign; +import org.ethereum.sharding.crypto.Sign; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,11 +36,14 @@ public class BeaconAttesterImpl implements BeaconAttester { StateRepository repository; BeaconStore store; ValidatorConfig config; + Sign sign; - public BeaconAttesterImpl(StateRepository repository, BeaconStore store, ValidatorConfig config) { + public BeaconAttesterImpl(StateRepository repository, BeaconStore store, ValidatorConfig config, + Sign sign) { this.repository = repository; this.store = store; this.config = config; + this.sign = sign; } @Override @@ -57,7 +57,7 @@ public AttestationRecord attestBlock(Input in, byte[] pubKey) { Bitfield.markVote(Bitfield.createEmpty(in.index.getCommitteeSize()), in.index.getValidatorIdx()), lastJustified, store.getCanonicalByNumber(lastJustified) == null ? new byte[32] : store.getCanonicalByNumber(lastJustified).getHash(), - Sign.aggSigns(new byte[][] {Sign.sign(in.block.getEncoded(), pubKey)}) + sign.aggSigns(new Sign.Signature[]{sign.sign(in.block.getEncoded(), pubKey)}) ); logger.info("Block {} attested by #{} in slot {} ", in.block, in.index.getValidatorIdx(), in.slotNumber); diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java index 91c0344996..cc7ef97eb2 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/validator/BeaconProposerImpl.java @@ -22,15 +22,11 @@ import org.ethereum.sharding.domain.Beacon; import org.ethereum.sharding.domain.Validator; import org.ethereum.sharding.processing.db.BeaconStore; -import org.ethereum.sharding.processing.state.AttestationRecord; import org.ethereum.sharding.pubsub.BeaconChainSynced; import org.ethereum.sharding.processing.consensus.StateTransition; import org.ethereum.sharding.processing.state.BeaconState; import org.ethereum.sharding.processing.state.StateRepository; -import org.ethereum.sharding.util.Bitfield; import org.ethereum.sharding.util.Randao; -import org.ethereum.sharding.util.Sign; -import org.ethereum.util.ByteUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 2e48da418d68561d287c81e95beca1e3217cf880 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Sun, 14 Oct 2018 23:59:00 +0300 Subject: [PATCH 09/14] Fixed broken compilation and tests --- .../org/ethereum/sharding/config/BeaconConfig.java | 3 ++- .../ethereum/sharding/domain/BeaconGenesis.java | 2 +- .../consensus/BeaconStateTransition.java | 2 +- .../processing/consensus/GenesisTransition.java | 2 +- .../sharding/processing/state/ActiveState.java | 13 +++++++++++++ .../processing/state/BeaconStateRepository.java | 13 +++++++++++-- .../sharding/domain/GenesisTransitionTest.java | 2 +- .../sharding/processing/BeaconChainTest.java | 5 +++-- .../processing/consensus/StateTransitionTest.java | 14 ++++++++------ .../processing/db/IndexedBeaconStoreTest.java | 5 +++-- .../sharding/validator/BeaconProposerTest.java | 2 +- 11 files changed, 45 insertions(+), 18 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java b/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java index aebeefdc1c..f3b76ee86a 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/config/BeaconConfig.java @@ -164,7 +164,8 @@ public StateRepository beaconStateRepository() { Source validatorSrc = cachedBeaconChainSource("validator_set"); Source validatorIndexSrc = cachedBeaconChainSource("validator_index"); Source crystallizedSrc = cachedBeaconChainSource("crystallized_state"); - return new BeaconStateRepository(src, crystallizedSrc, validatorSrc, validatorIndexSrc); + Source activeSrc = cachedBeaconChainSource("active_state"); + return new BeaconStateRepository(src, crystallizedSrc, activeSrc, validatorSrc, validatorIndexSrc); } @Bean diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java index 37ec460526..0080b24102 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java @@ -54,7 +54,7 @@ public class BeaconGenesis extends Beacon { private List initialValidators = new ArrayList<>(); BeaconGenesis(Json json) { - super(json.parentHash(), json.randaoReveal(), json.mainChainRef(), EMPTY, SLOT, new AttestationRecord[0]); + super(json.parentHash(), json.randaoReveal(), json.mainChainRef(), EMPTY, SLOT, new ArrayList<>()); this.timestamp = json.timestamp(); this.initialValidators = json.validatorSet(); } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/BeaconStateTransition.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/BeaconStateTransition.java index bee585d5ae..d69c83f797 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/BeaconStateTransition.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/BeaconStateTransition.java @@ -69,6 +69,6 @@ public BeaconState applyBlock(Beacon block, BeaconState to) { .withFinality(finality); } - return new BeaconState(crystallized); + return new BeaconState(crystallized, to.getActiveState()); } } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/GenesisTransition.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/GenesisTransition.java index 59c1e292b6..6263f2f353 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/GenesisTransition.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/consensus/GenesisTransition.java @@ -80,7 +80,7 @@ public BeaconState applyBlock(Beacon block, BeaconState to) { .withLastStateRecalc(0L) .withCrosslinks(Crosslink.empty(SHARD_COUNT)); - return new BeaconState(crystallizedState); + return new BeaconState(crystallizedState, to.getActiveState()); } class ValidatorSetInitiator implements StateTransition { diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java index b7619d6933..d06ce7f728 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java @@ -17,6 +17,7 @@ */ package org.ethereum.sharding.processing.state; +import org.ethereum.datasource.Serializer; import org.ethereum.util.ByteUtil; import org.ethereum.util.RLP; import org.ethereum.util.RLPElement; @@ -133,4 +134,16 @@ public String toString() { return builder.toString(); } + + public static final org.ethereum.datasource.Serializer Serializer = new Serializer() { + @Override + public byte[] serialize(ActiveState state) { + return state == null ? null : state.getEncoded(); + } + + @Override + public ActiveState deserialize(byte[] stream) { + return stream == null ? null : new ActiveState(stream); + } + }; } diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconStateRepository.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconStateRepository.java index 06ed4f9c38..c271ae825f 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconStateRepository.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/BeaconStateRepository.java @@ -35,10 +35,12 @@ public class BeaconStateRepository implements StateRepository { ObjectDataSource stateDS; Source crystallizedSrc; ObjectDataSource crystallizedDS; + ObjectDataSource activeDS; Source validatorSrc; Source validatorIndexSrc; public BeaconStateRepository(Source src, Source crystallizedSrc, + Source activeSrc, Source validatorSrc, Source validatorIndexSrc) { this.src = src; this.crystallizedSrc = crystallizedSrc; @@ -48,12 +50,16 @@ public BeaconStateRepository(Source src, Source this.stateDS = new ObjectDataSource<>(src, BeaconState.Flattened.Serializer, BeaconStore.BLOCKS_IN_MEM); this.crystallizedDS = new ObjectDataSource<>(crystallizedSrc, CrystallizedState.Flattened.Serializer, BeaconStore.BLOCKS_IN_MEM); + this.activeDS = new ObjectDataSource<>(activeSrc, + ActiveState.Serializer, BeaconStore.BLOCKS_IN_MEM); } @Override public void insert(BeaconState state) { CrystallizedState crystallized = state.getCrystallizedState(); crystallizedDS.put(crystallized.getHash(), crystallized.flatten()); + ActiveState activeState = state.getActiveState(); + activeDS.put(activeState.getHash(), activeState); stateDS.put(state.getHash(), state.flatten()); } @@ -70,15 +76,18 @@ public BeaconState get(byte[] hash) { public BeaconState getEmpty() { CrystallizedState.Flattened crystallizedFlattened = CrystallizedState.Flattened.empty(); CrystallizedState crystallizedState = fromFlattened(crystallizedFlattened); + ActiveState activeState = ActiveState.createEmpty(); - return new BeaconState(crystallizedState); + return new BeaconState(crystallizedState, activeState); } BeaconState fromFlattened(BeaconState.Flattened flattened) { CrystallizedState.Flattened crystallizedFlattened = crystallizedDS.get(flattened.getCrystallizedStateHash()); CrystallizedState crystallizedState = fromFlattened(crystallizedFlattened); - return new BeaconState(crystallizedState); + ActiveState activeState = activeDS.get(flattened.getActiveStateHash()); + + return new BeaconState(crystallizedState, activeState); } CrystallizedState fromFlattened(CrystallizedState.Flattened flattened) { diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/domain/GenesisTransitionTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/domain/GenesisTransitionTest.java index 87fadb6561..46592d87c2 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/domain/GenesisTransitionTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/domain/GenesisTransitionTest.java @@ -56,7 +56,7 @@ public void testInitialValidatorSet() { BeaconGenesis genesis = new BeaconGenesis(getJson(v1, v3, v4)); StateRepository stateRepository = new BeaconStateRepository(new HashMapDB<>(), new HashMapDB<>(), - new HashMapDB<>(), new HashMapDB<>()); + new HashMapDB<>(), new HashMapDB<>(), new HashMapDB<>()); ValidatorRepository validatorRepository = new PredefinedValidatorRepository(v1, v2, v3, v4); GenesisTransition transition = new GenesisTransition(validatorRepository); diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/processing/BeaconChainTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/processing/BeaconChainTest.java index 3b125612eb..36771a437b 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/processing/BeaconChainTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/processing/BeaconChainTest.java @@ -32,6 +32,7 @@ import org.junit.Test; import java.math.BigInteger; +import java.util.ArrayList; import java.util.Collections; import java.util.Random; import java.util.concurrent.Future; @@ -183,7 +184,7 @@ static Helper newInstance() { Helper inst = new Helper(); inst.store = new IndexedBeaconStore(new HashMapDB<>(), new HashMapDB<>()); inst.repository = new BeaconStateRepository(new HashMapDB<>(), new HashMapDB<>(), - new HashMapDB<>(), new HashMapDB<>()); + new HashMapDB<>(), new HashMapDB<>(), new HashMapDB<>()); inst.beaconChain = (BeaconChainImpl) BeaconChainFactory.create( new DummyFlusher(), inst.store, inst.repository, new NoTransition(), new NoTransition()); inst.beaconChain.scoreFunction = (block, state) -> BigInteger.valueOf(block.getMainChainRef()[0]); @@ -208,7 +209,7 @@ Beacon createBlock(Beacon parent, long score) { BeaconState parentState = repository.get(parent.getStateHash()); Beacon newBlock = new Beacon(parent.getHash(), - randaoReveal, mainChainRef, null, parent.getSlotNumber() + 1); + randaoReveal, mainChainRef, null, parent.getSlotNumber() + 1, new ArrayList<>()); BeaconState newState = beaconChain.transitionFunction.applyBlock(newBlock, parentState); newBlock.setStateHash(newState.getHash()); diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/processing/consensus/StateTransitionTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/processing/consensus/StateTransitionTest.java index 53b9a13b30..688930520b 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/processing/consensus/StateTransitionTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/processing/consensus/StateTransitionTest.java @@ -6,6 +6,7 @@ import org.ethereum.sharding.domain.Validator; import org.ethereum.sharding.processing.db.TrieValidatorSet; import org.ethereum.sharding.processing.db.ValidatorSet; +import org.ethereum.sharding.processing.state.ActiveState; import org.ethereum.sharding.processing.state.BeaconState; import org.ethereum.sharding.processing.state.Committee; import org.ethereum.sharding.processing.state.Crosslink; @@ -14,6 +15,7 @@ import org.ethereum.sharding.processing.state.Finality; import org.junit.Test; +import java.util.ArrayList; import java.util.Random; import static org.ethereum.crypto.HashUtil.randomHash; @@ -32,10 +34,10 @@ public void testBasics() { StateTransition transitionFunction = new BeaconStateTransition( dynastyTransition(validatorTransition(validator)), finalityTransition()); - Beacon b1 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 1L); - Beacon b2 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 63L); - Beacon b3 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 64L); - Beacon b4 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 72L); + Beacon b1 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 1L, new ArrayList<>()); + Beacon b2 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 63L, new ArrayList<>()); + Beacon b3 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 64L, new ArrayList<>()); + Beacon b4 = new Beacon(new byte[32], new byte[32], new byte[32], new byte[32], 72L, new ArrayList<>()); assertEquals(getOrigin(), transitionFunction.applyBlock(b1, getOrigin())); assertEquals(getOrigin(), transitionFunction.applyBlock(b2, getOrigin())); @@ -72,7 +74,7 @@ BeaconState getExpected(Beacon block, Validator validator) { dynasty, finality, new Crosslink[0] ); - return new BeaconState(crystallized); + return new BeaconState(crystallized, getOrigin().getActiveState()); } BeaconState getOrigin() { @@ -81,7 +83,7 @@ BeaconState getOrigin() { Finality finality = Finality.empty(); CrystallizedState crystallized = new CrystallizedState(0L, dynasty, finality, new Crosslink[0]); - return new BeaconState(crystallized); + return new BeaconState(crystallized, ActiveState.createEmpty()); } Validator getRandomValidator() { diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/processing/db/IndexedBeaconStoreTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/processing/db/IndexedBeaconStoreTest.java index 4f20eddec0..abc527d9f4 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/processing/db/IndexedBeaconStoreTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/processing/db/IndexedBeaconStoreTest.java @@ -22,6 +22,7 @@ import org.junit.Test; import java.math.BigInteger; +import java.util.ArrayList; import java.util.Random; import static org.junit.Assert.assertArrayEquals; @@ -227,8 +228,8 @@ private Beacon createBlock(Beacon parent) { rnd.nextBytes(mainChainRef); rnd.nextBytes(stateHash); - return new Beacon(parent == null ? new byte[32] : parent.getHash(), - randaoReveal, mainChainRef, stateHash, parent == null ? 0 : parent.getSlotNumber() + 1); + return new Beacon(parent == null ? new byte[32] : parent.getHash(), randaoReveal, + mainChainRef, stateHash, parent == null ? 0 : parent.getSlotNumber() + 1, new ArrayList<>()); } static class StoreHelper { diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java index 83275034f0..679d09a8d6 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/validator/BeaconProposerTest.java @@ -107,7 +107,7 @@ static Helper newInstance() { randao.generate(1000); StateRepository repository = new BeaconStateRepository(new HashMapDB<>(), new HashMapDB<>(), - new HashMapDB<>(), new HashMapDB<>()); + new HashMapDB<>(), new HashMapDB<>(), new HashMapDB<>()); BeaconStore store = new IndexedBeaconStore(new HashMapDB<>(), new HashMapDB<>()); StateTransition stateTransition = new NoTransition(); From 6147ee85710dac53c7ab05e79828efac15366f52 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Mon, 15 Oct 2018 22:45:44 +0300 Subject: [PATCH 10/14] Added Bitfield utils test --- .../org/ethereum/sharding/util/Bitfield.java | 12 +-- .../ethereum/sharding/util/BitfieldTest.java | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java index 5791c406e5..5207014d31 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java @@ -34,17 +34,7 @@ public class Bitfield { * @return empty bitfield with correct length */ public static BitSet createEmpty(int validatorsCount) { - return new BitSet(calcLength(validatorsCount)); - } - - // TODO: Add test - /** - * Calculates attesters bitfield length - * @param num Number of attesters - * @return Bitfield length in bytes - */ - public static int calcLength(int num) { - return (num + 7) / Byte.SIZE; + return new BitSet(validatorsCount); } /** diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java new file mode 100644 index 0000000000..3141ba11cc --- /dev/null +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.util; + +import org.junit.Test; + +import java.util.BitSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link Bitfield} + */ +public class BitfieldTest { + + @Test + public void testSize() { + BitSet bitfield = Bitfield.createEmpty(13); + assertEquals(64, bitfield.size()); // 64 is minimum size for BitSet + + BitSet bitfield2 = Bitfield.createEmpty(113); + assertEquals(128, bitfield2.size()); // incremented by long words, which is 64 bits + } + + @Test + public void testBasics() { + BitSet bitfield = Bitfield.createEmpty(35); + assertEquals(0, Bitfield.calcVotes(bitfield)); + + bitfield = Bitfield.markVote(bitfield, 32); + assertEquals(1, Bitfield.calcVotes(bitfield)); + for (int i = 0; i < 35; ++i) { + if (i != 32) { + assertFalse(Bitfield.hasVoted(bitfield, i)); + } else { + assertTrue(Bitfield.hasVoted(bitfield, i)); + } + } + + bitfield = Bitfield.markVote(bitfield, 0); + assertEquals(2, Bitfield.calcVotes(bitfield)); + assertTrue(Bitfield.hasVoted(bitfield, 0)); + assertFalse(Bitfield.hasVoted(bitfield, 1)); + assertTrue(Bitfield.hasVoted(bitfield, 32)); + } + + @Test + public void testSerialize() { + BitSet bitfield = Bitfield.createEmpty(125); + bitfield = Bitfield.markVote(bitfield, 124); + bitfield = Bitfield.markVote(bitfield, 116); + byte[] serialized = bitfield.toByteArray(); + + BitSet restored = BitSet.valueOf(serialized); + assertTrue(Bitfield.hasVoted(restored, 124)); + assertTrue(Bitfield.hasVoted(restored, 116)); + assertEquals(2, Bitfield.calcVotes(restored)); + } +} From 05038f752912cee8b8af21ba096f17f34232e723 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Mon, 15 Oct 2018 23:04:01 +0300 Subject: [PATCH 11/14] Wrap BitSet to have normal size etc --- .../processing/state/AttestationRecord.java | 11 ++-- .../org/ethereum/sharding/util/Bitfield.java | 62 +++++++++++++++---- .../ethereum/sharding/util/BitfieldTest.java | 16 ++--- 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java index f05e27a7a5..b2e87e34c9 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java @@ -18,6 +18,7 @@ package org.ethereum.sharding.processing.state; import org.ethereum.sharding.crypto.Sign; +import org.ethereum.sharding.util.Bitfield; import org.ethereum.util.ByteUtil; import org.ethereum.util.RLP; import org.ethereum.util.RLPList; @@ -47,7 +48,7 @@ public class AttestationRecord { // Block hash in the shard that we are attesting to private final byte[] shardBlockHash; // Who is participating - private final BitSet attesterBitfield; + private final Bitfield attesterBitfield; // Last justified block private final long justifiedSlot; private final byte[] justifiedBlockHash; @@ -55,7 +56,7 @@ public class AttestationRecord { private final Sign.Signature aggregateSig; public AttestationRecord(long slot, int shardId, byte[][] obliqueParentHashes, byte[] shardBlockHash, - BitSet attesterBitfield, long justifiedSlot, byte[] justifiedBlockHash, + Bitfield attesterBitfield, long justifiedSlot, byte[] justifiedBlockHash, Sign.Signature aggregateSig) { this.slot = slot; this.shardId = shardId; @@ -79,7 +80,7 @@ public AttestationRecord(byte[] encoded) { this.obliqueParentHashes[i] = hashesList.get(i).getRLPData(); this.shardBlockHash = list.get(3).getRLPData(); - this.attesterBitfield = BitSet.valueOf(list.get(4).getRLPData()); + this.attesterBitfield = new Bitfield(list.get(4).getRLPData()); this.justifiedSlot = byteArrayToLong(list.get(5).getRLPData()); this.justifiedBlockHash = list.get(6).getRLPData(); @@ -105,7 +106,7 @@ public byte[] getShardBlockHash() { return shardBlockHash; } - public BitSet getAttesterBitfield() { + public Bitfield getAttesterBitfield() { return attesterBitfield; } @@ -130,7 +131,7 @@ public byte[] getEncoded() { intToBytesNoLeadZeroes(shardId), RLP.wrapList(obliqueParentHashes), shardBlockHash, - attesterBitfield.toByteArray(), + attesterBitfield.getData(), longToBytesNoLeadZeroes(justifiedSlot), justifiedBlockHash, RLP.wrapList(encodedAggSig)); diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java index 5207014d31..3112e7f638 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java @@ -17,6 +17,7 @@ */ package org.ethereum.sharding.util; +import java.util.Arrays; import java.util.BitSet; import java.util.List; @@ -28,13 +29,35 @@ */ public class Bitfield { + private final BitSet payload; + private final int size; // in Bits + + private Bitfield(int size) { + this.size = calcLength(size) * Byte.SIZE; + this.payload = new BitSet(size); + } + + public Bitfield(byte[] data) { + this.size = data.length * Byte.SIZE; + this.payload = BitSet.valueOf(data); + } + + /** + * Calculates attesters bitfield length + * @param num Number of attesters + * @return Bitfield length in bytes + */ + private static int calcLength(int num) { + return (num + 7) / Byte.SIZE; + } + /** * Creates empty bitfield for estimated number of attesters * @param validatorsCount Number of attesters * @return empty bitfield with correct length */ - public static BitSet createEmpty(int validatorsCount) { - return new BitSet(validatorsCount); + public static Bitfield createEmpty(int validatorsCount) { + return new Bitfield(validatorsCount); } /** @@ -45,9 +68,9 @@ public static BitSet createEmpty(int validatorsCount) { * @param index Index number of attester * @return bitfield with vote in place */ - public static BitSet markVote(final BitSet bitfield, int index) { - BitSet newBitfield = (BitSet) bitfield.clone(); - newBitfield.set(index); + public static Bitfield markVote(final Bitfield bitfield, int index) { + Bitfield newBitfield = bitfield.clone(); + newBitfield.payload.set(index); return newBitfield; } @@ -56,8 +79,8 @@ public static BitSet markVote(final BitSet bitfield, int index) { * @param bitfield Bitfield * @param index Index number of attester */ - public static boolean hasVoted(BitSet bitfield, int index) { - return bitfield.get(index); + public static boolean hasVoted(Bitfield bitfield, int index) { + return bitfield.payload.get(index); } /** @@ -65,7 +88,7 @@ public static boolean hasVoted(BitSet bitfield, int index) { * @param bitfield Bitfield * @return number of votes */ - public static int calcVotes(BitSet bitfield) { + public static int calcVotes(Bitfield bitfield) { int votes = 0; for (int i = 0; i < bitfield.size(); ++i) { if (hasVoted(bitfield, i)) ++votes; @@ -80,19 +103,32 @@ public static int calcVotes(BitSet bitfield) { * @param bitfields Bitfields * @return All bitfields aggregated using OR */ - public static BitSet orBitfield(List bitfields) { + public static Bitfield orBitfield(List bitfields) { if (bitfields.isEmpty()) return null; int bitfieldLen = bitfields.get(0).size(); - BitSet aggBitfield = new BitSet(bitfieldLen); + Bitfield aggBitfield = new Bitfield(bitfieldLen); for (int i = 0; i < bitfieldLen; ++i) { - for (BitSet bitfield : bitfields) { - if (aggBitfield.get(i) | bitfield.get(i)) { - aggBitfield.set(i); + for (Bitfield bitfield : bitfields) { + if (aggBitfield.payload.get(i) | bitfield.payload.get(i)) { + aggBitfield.payload.set(i); } } } return aggBitfield; } + + public int size() { + return size; + } + + public byte[] getData() { + return Arrays.copyOf(payload.toByteArray(), size * Byte.SIZE); + } + + public Bitfield clone() { + return new Bitfield(getData()); + } + } diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java index 3141ba11cc..6061799547 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java @@ -31,16 +31,16 @@ public class BitfieldTest { @Test public void testSize() { - BitSet bitfield = Bitfield.createEmpty(13); - assertEquals(64, bitfield.size()); // 64 is minimum size for BitSet + Bitfield bitfield = Bitfield.createEmpty(13); + assertEquals(16, bitfield.size()); - BitSet bitfield2 = Bitfield.createEmpty(113); - assertEquals(128, bitfield2.size()); // incremented by long words, which is 64 bits + Bitfield bitfield2 = Bitfield.createEmpty(113); + assertEquals(120, bitfield2.size()); } @Test public void testBasics() { - BitSet bitfield = Bitfield.createEmpty(35); + Bitfield bitfield = Bitfield.createEmpty(35); assertEquals(0, Bitfield.calcVotes(bitfield)); bitfield = Bitfield.markVote(bitfield, 32); @@ -62,12 +62,12 @@ public void testBasics() { @Test public void testSerialize() { - BitSet bitfield = Bitfield.createEmpty(125); + Bitfield bitfield = Bitfield.createEmpty(125); bitfield = Bitfield.markVote(bitfield, 124); bitfield = Bitfield.markVote(bitfield, 116); - byte[] serialized = bitfield.toByteArray(); + byte[] serialized = bitfield.getData(); - BitSet restored = BitSet.valueOf(serialized); + Bitfield restored = new Bitfield(serialized); assertTrue(Bitfield.hasVoted(restored, 124)); assertTrue(Bitfield.hasVoted(restored, 116)); assertEquals(2, Bitfield.calcVotes(restored)); From 529632fb27de21d3484392a6964bb71d53bc021c Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Mon, 15 Oct 2018 23:20:17 +0300 Subject: [PATCH 12/14] Add signature tests --- .../ethereum/sharding/crypto/DummySign.java | 2 + .../ethereum/sharding/crypto/SignTest.java | 52 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 ethereumj-core/src/test/java/org/ethereum/sharding/crypto/SignTest.java diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java index 20691d236a..e0928d8701 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java @@ -43,6 +43,8 @@ public boolean verify(Signature signature, byte[] publicKey) { public Signature aggSigns(Signature[] signatures) { int signatureLen = signatures.length; Signature aggSignature = new Signature(); + aggSignature.r = BigInteger.ZERO; + aggSignature.s = BigInteger.ZERO; for (int i = 0; i < signatureLen; ++i) { for (Signature signature : signatures) { aggSignature.r = aggSignature.r.xor(signature.r); diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/crypto/SignTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/crypto/SignTest.java new file mode 100644 index 0000000000..06b98a0b1a --- /dev/null +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/crypto/SignTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.crypto; + +import org.junit.Test; +import org.spongycastle.util.encoders.Hex; + +import static org.ethereum.crypto.HashUtil.sha3; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link Sign} + */ +public class SignTest { + + Sign sign = new DummySign(); + + @Test + public void testDummy() { + byte[] msg = Hex.decode("0737626387abcdef"); + byte[] privKey = sha3(msg); + byte[] pubKey = sha3(privKey); + Sign.Signature signature = sign.sign(msg, privKey); + assertTrue(sign.verify(signature, pubKey)); + + byte[] privKey2 = sha3(Hex.decode("abcdef")); + byte[] pubKey2 = sha3(privKey); + // FIXME: Dummy sign is always correct, not true for real implementation + // assertFalse(sign.verify(signature, pubKey2)); + + Sign.Signature signature2 = sign.sign(msg, privKey2); + Sign.Signature aggSign = sign.aggSigns(new Sign.Signature[] {signature, signature2}); + assertTrue(sign.verify(aggSign, pubKey)); + assertTrue(sign.verify(aggSign, pubKey2)); + } +} From 81dc46759002ae84744410df71c7b16a371d7ff9 Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Mon, 15 Oct 2018 23:43:13 +0300 Subject: [PATCH 13/14] Beacon attester code cleanup --- .../java/org/ethereum/sharding/crypto/DummySign.java | 9 +++++++++ .../src/main/java/org/ethereum/sharding/crypto/Sign.java | 9 +++++++++ .../java/org/ethereum/sharding/domain/BeaconGenesis.java | 1 - .../sharding/processing/state/AttestationRecord.java | 1 - .../main/java/org/ethereum/sharding/util/Bitfield.java | 2 -- .../java/org/ethereum/sharding/util/BitfieldTest.java | 1 - 6 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java index e0928d8701..988481464e 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/DummySign.java @@ -26,6 +26,9 @@ */ public class DummySign implements Sign { + /** + * Sign the message + */ public Signature sign(byte[] msg, byte[] privateKey) { byte[] rSource = sha3(privateKey); byte[] sSource = sha3(msg, privateKey); @@ -36,10 +39,16 @@ public Signature sign(byte[] msg, byte[] privateKey) { return res; } + /** + * Verifies whether signature is made by signer with publicKey + */ public boolean verify(Signature signature, byte[] publicKey) { return true; } + /** + * Aggregates several signatures in one + */ public Signature aggSigns(Signature[] signatures) { int signatureLen = signatures.length; Signature aggSignature = new Signature(); diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java index 5b45cd4345..04e25a9955 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/crypto/Sign.java @@ -9,10 +9,19 @@ */ public interface Sign { + /** + * Sign the message + */ Signature sign(byte[] msg, byte[] privateKey); + /** + * Verifies whether signature is made by signer with publicKey + */ boolean verify(Signature signature, byte[] publicKey); + /** + * Aggregates several signatures in one + */ Signature aggSigns(Signature[] signatures); class Signature { diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java index 0080b24102..94738cb429 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/domain/BeaconGenesis.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.io.ByteStreams; -import org.ethereum.sharding.processing.state.AttestationRecord; import org.ethereum.util.Utils; import java.io.InputStream; diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java index b2e87e34c9..0d56b0fb9b 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/AttestationRecord.java @@ -24,7 +24,6 @@ import org.ethereum.util.RLPList; import java.util.Arrays; -import java.util.BitSet; import static org.ethereum.util.ByteUtil.bigIntegerToBytes; import static org.ethereum.util.ByteUtil.byteArrayToLong; diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java index 3112e7f638..b3e635b287 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/util/Bitfield.java @@ -22,8 +22,6 @@ import java.util.List; /** - * Bitfield utility methods - * * Bitfield is bit array where every bit represents status of * attester with corresponding index */ diff --git a/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java b/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java index 6061799547..3ee53328c9 100644 --- a/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java +++ b/ethereumj-core/src/test/java/org/ethereum/sharding/util/BitfieldTest.java @@ -19,7 +19,6 @@ import org.junit.Test; -import java.util.BitSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; From d4f2f02783c90c5377a0b307f93ebe6588dfa28a Mon Sep 17 00:00:00 2001 From: Dmitry Shmatko Date: Mon, 15 Oct 2018 23:54:13 +0300 Subject: [PATCH 14/14] Added SpecialRecord object --- .../processing/state/ActiveState.java | 32 ++++++-- .../processing/state/SpecialRecord.java | 80 +++++++++++++++++++ 2 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/SpecialRecord.java diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java index d06ce7f728..b693de845c 100644 --- a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/ActiveState.java @@ -37,6 +37,8 @@ public class ActiveState { // Attestations that have not yet been processed private final List pendingAttestations; + // Special objects that have not yet been processed + private final List pendingSpecials; // Most recent 2 * CYCLE_LENGTH block hashes, older to newer private final List recentBlockHashes; // RANDAO state @@ -45,8 +47,10 @@ public class ActiveState { // TODO: Add pending_specials - public ActiveState(List pendingAttestations, List recentBlockHashes, byte[] randaoMix) { + public ActiveState(List pendingAttestations, List pendingSpecials, + List recentBlockHashes, byte[] randaoMix) { this.pendingAttestations = pendingAttestations; + this.pendingSpecials = pendingSpecials; this.recentBlockHashes = recentBlockHashes; this.randaoMix = randaoMix; } @@ -57,12 +61,13 @@ public ActiveState(List pendingAttestations, List rec */ public static ActiveState createEmpty() { List pendingAttestations = new ArrayList<>(); + List pendingSpecials = new ArrayList<>(); List recentBlockHashes = new ArrayList<>(); for (int i = 0; i < (CYCLE_LENGTH * 2); ++i) { recentBlockHashes.add(new byte[32]); } - return new ActiveState(pendingAttestations, recentBlockHashes, new byte[32]); + return new ActiveState(pendingAttestations, pendingSpecials, recentBlockHashes, new byte[32]); } public ActiveState(byte[] encoded) { @@ -75,20 +80,31 @@ public ActiveState(byte[] encoded) { pendingAttestations.add(new AttestationRecord(attestationRlp.getRLPData())); } - this.recentBlockHashes = new ArrayList<>(); + this.pendingSpecials = new ArrayList<>(); if (!isSingleZero(list.get(1).getRLPData())) { - RLPList hashesList = RLP.unwrapList(list.get(1).getRLPData()); + RLPList specialsList = RLP.unwrapList(list.get(1).getRLPData()); + for (RLPElement specialRlp : specialsList) + pendingSpecials.add(new SpecialRecord(specialRlp.getRLPData())); + } + + this.recentBlockHashes = new ArrayList<>(); + if (!isSingleZero(list.get(2).getRLPData())) { + RLPList hashesList = RLP.unwrapList(list.get(2).getRLPData()); for (RLPElement hashRlp : hashesList) recentBlockHashes.add(hashRlp.getRLPData()); } - this.randaoMix = list.get(2).getRLPData(); + this.randaoMix = list.get(3).getRLPData(); } public List getPendingAttestations() { return pendingAttestations; } + public List getPendingSpecials() { + return pendingSpecials; + } + public List getRecentBlockHashes() { return recentBlockHashes; } @@ -102,12 +118,17 @@ public byte[] getEncoded() { for (int i = 0; i < pendingAttestations.size(); i++) encodedAttestations[i] = pendingAttestations.get(i).getEncoded(); + byte[][] encodedSpecials = new byte[pendingSpecials.size()][]; + for (int i = 0; i < pendingSpecials.size(); i++) + encodedSpecials[i] = pendingSpecials.get(i).getEncoded(); + byte[][] encodedHashes = new byte[recentBlockHashes.size()][]; for (int i = 0; i < recentBlockHashes.size(); i++) encodedHashes[i] = RLP.encodeElement(recentBlockHashes.get(i)); return RLP.wrapList( pendingAttestations.size() > 0 ? RLP.wrapList(encodedAttestations) : ByteUtil.ZERO_BYTE_ARRAY, + pendingSpecials.size() > 0 ? RLP.wrapList(encodedSpecials) : ByteUtil.ZERO_BYTE_ARRAY, recentBlockHashes.size() > 0 ? RLP.wrapList(encodedHashes) : ByteUtil.ZERO_BYTE_ARRAY, RLP.encodeElement(randaoMix) ); @@ -121,6 +142,7 @@ public byte[] getHash() { public String toString() { StringBuilder builder = new StringBuilder().append("ActiveState{") .append("pendingAttestations=[").append(pendingAttestations.size()).append(" item(s)]") + .append("pendingSpecials=[").append(pendingSpecials.size()).append(" item(s)]") .append(", recentBlockHashes=[..."); for (int i = Math.max(0, recentBlockHashes.size() - 3); i < recentBlockHashes.size(); ++i) { diff --git a/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/SpecialRecord.java b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/SpecialRecord.java new file mode 100644 index 0000000000..760ba6f59c --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/sharding/processing/state/SpecialRecord.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) [2016] [ ] + * This file is part of the ethereumJ library. + * + * The ethereumJ library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The ethereumJ library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the ethereumJ library. If not, see . + */ +package org.ethereum.sharding.processing.state; + +import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; +import org.spongycastle.util.encoders.Hex; + +import java.util.Arrays; + +import static org.ethereum.util.ByteUtil.byteArrayToInt; +import static org.ethereum.util.ByteUtil.intToBytesNoLeadZeroes; + +/** + * Special record, sort of container + */ +public class SpecialRecord { + + // Kind of object + private final int kind; + // Payload + private final byte[] data; + + public SpecialRecord(int kind, byte[] data) { + this.kind = kind; + this.data = data; + } + + public SpecialRecord(byte[] encoded) { + RLPList list = RLP.unwrapList(encoded); + + this.kind = byteArrayToInt(list.get(0).getRLPData()); + this.data = list.get(1).getRLPData(); + } + + public int getKind() { + return kind; + } + + public byte[] getData() { + return data; + } + + public byte[] getEncoded() { + return RLP.wrapList(intToBytesNoLeadZeroes(kind), + data); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SpecialRecord that = (SpecialRecord) o; + return kind == that.kind && + Arrays.equals(data, that.data); + } + + @Override + public String toString() { + return "SpecialRecord{" + + "kind=" + kind + + ", data=" + Hex.toHexString(data) + + '}'; + } +}