From 3ea2bf3ce26233af12f04813076fbd87902f17ae Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 21 May 2019 15:04:28 +0200 Subject: [PATCH 001/223] Model: Milestone->Round --- src/main/java/net/helix/hlx/Helix.java | 5 +- .../hlx/controllers/MilestoneViewModel.java | 118 +++++++++--------- .../helix/hlx/model/persistables/Hashes.java | 2 +- .../hlx/model/persistables/Milestone.java | 45 ------- .../helix/hlx/model/persistables/Round.java | 37 ++++++ .../jobs/MilestonePrunerJob.java | 6 +- .../java/net/helix/hlx/storage/Tangle.java | 2 +- 7 files changed, 104 insertions(+), 111 deletions(-) delete mode 100644 src/main/java/net/helix/hlx/model/persistables/Milestone.java create mode 100644 src/main/java/net/helix/hlx/model/persistables/Round.java diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index 6f61a2ce..b3d2c401 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -4,6 +4,7 @@ import net.helix.hlx.conf.TipSelConfig; import net.helix.hlx.controllers.TipsViewModel; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.network.Node; import net.helix.hlx.network.TransactionRequester; import net.helix.hlx.network.UDPReceiver; @@ -175,7 +176,7 @@ public void init() throws Exception { } if (configuration.isRevalidate()) { - tangle.clearColumn(net.helix.hlx.model.persistables.Milestone.class); + tangle.clearColumn(Round.class); tangle.clearColumn(net.helix.hlx.model.StateDiff.class); tangle.clearMetadata(net.helix.hlx.model.persistables.Transaction.class); } @@ -237,7 +238,7 @@ private void rescanDb() throws Exception { tangle.clearColumn(net.helix.hlx.model.persistables.Approvee.class); tangle.clearColumn(net.helix.hlx.model.persistables.BundleNonce.class); tangle.clearColumn(net.helix.hlx.model.persistables.Tag.class); - tangle.clearColumn(net.helix.hlx.model.persistables.Milestone.class); + tangle.clearColumn(Round.class); tangle.clearColumn(net.helix.hlx.model.StateDiff.class); tangle.clearMetadata(net.helix.hlx.model.persistables.Transaction.class); diff --git a/src/main/java/net/helix/hlx/controllers/MilestoneViewModel.java b/src/main/java/net/helix/hlx/controllers/MilestoneViewModel.java index 3fc6a252..5cbda46b 100644 --- a/src/main/java/net/helix/hlx/controllers/MilestoneViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/MilestoneViewModel.java @@ -2,7 +2,7 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.IntegerIndex; -import net.helix.hlx.model.persistables.Milestone; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.storage.Indexable; import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.Tangle; @@ -12,19 +12,19 @@ import java.util.concurrent.ConcurrentHashMap; /** - * Acts as a controller interface for a {@link Milestone} hash object. This controller is used by the - * {@link net.helix.hlx.service.milestone.LatestMilestoneTracker} to manipulate a {@link Milestone} object. + * Acts as a controller interface for a {@link Round} hash object. This controller is used by the + * {@link net.helix.hlx.service.milestone.LatestMilestoneTracker} to manipulate a {@link Round} object. */ public class MilestoneViewModel { - private final Milestone milestone; + private final Round round; private static final Map milestones = new ConcurrentHashMap<>(); - private MilestoneViewModel(final Milestone milestone) { - this.milestone = milestone; + private MilestoneViewModel(final Round round) { + this.round = round; } /** - * Removes the contents of the stored {@link Milestone} object set. + * Removes the contents of the stored {@link Round} object set. */ public static void clear() { milestones.clear(); @@ -43,27 +43,27 @@ public static void clear(int milestoneIndex) { } /** - * Constructor for a {@link Milestone} set controller. This controller is generated from a finalized - * {@link Milestone} hash identifier, indexing this object to the integer {@link Milestone} index. + * Constructor for a {@link Round} set controller. This controller is generated from a finalized + * {@link Round} hash identifier, indexing this object to the integer {@link Round} index. * - * @param index The finalized numerical index the {@link Milestone} object will be referenced by in the set - * @param milestoneHash The finalized {@link Hash} identifier for the {@link Milestone} object + * @param index The finalized numerical index the {@link Round} object will be referenced by in the set + * @param milestoneHash The finalized {@link Hash} identifier for the {@link Round} object */ public MilestoneViewModel(final int index, final Hash milestoneHash) { - this.milestone = new Milestone(); - this.milestone.index = new IntegerIndex(index); - milestone.hash = milestoneHash; + this.round = new Round(); + this.round.index = new IntegerIndex(index); + round.hash = milestoneHash; } /** * Fetches an existing {@link MilestoneViewModel} if its index reference can be found in the controller. If the - * {@link MilestoneViewModel} is null, but the indexed {@link Milestone} object exists in the database, a new - * controller is created for the {@link Milestone} object. + * {@link MilestoneViewModel} is null, but the indexed {@link Round} object exists in the database, a new + * controller is created for the {@link Round} object. * * @param tangle The tangle reference for the database - * @param index The integer index of the {@link Milestone} object that the controller should be returned for - * @return The {@link MilestoneViewModel} for the indexed {@link Milestone} object - * @throws Exception Thrown if the database fails to load the indexed {@link Milestone} object + * @param index The integer index of the {@link Round} object that the controller should be returned for + * @return The {@link MilestoneViewModel} for the indexed {@link Round} object + * @throws Exception Thrown if the database fails to load the indexed {@link Round} object */ public static MilestoneViewModel get(Tangle tangle, int index) throws Exception { MilestoneViewModel milestoneViewModel = milestones.get(index); @@ -74,96 +74,96 @@ public static MilestoneViewModel get(Tangle tangle, int index) throws Exception } /** - * Fetches a {@link Milestone} object from the database using its integer index. If the {@link Milestone} and the + * Fetches a {@link Round} object from the database using its integer index. If the {@link Round} and the * associated {@link Hash} identifier are not null, a new {@link MilestoneViewModel} is created for the - * {@link Milestone} object, and it is placed into the Milestones set, indexed by the provided integer + * {@link Round} object, and it is placed into the Milestones set, indexed by the provided integer * index. * * @param tangle The tangle reference for the database - * @param index The integer index reference for the {@link Milestone} object - * @return True if the {@link Milestone} object is stored in the Milestones set, False if not - * @throws Exception Thrown if the database fails to load the {@link Milestone} object + * @param index The integer index reference for the {@link Round} object + * @return True if the {@link Round} object is stored in the Milestones set, False if not + * @throws Exception Thrown if the database fails to load the {@link Round} object */ public static boolean load(Tangle tangle, int index) throws Exception { - Milestone milestone = (Milestone) tangle.load(Milestone.class, new IntegerIndex(index)); - if(milestone != null && milestone.hash != null) { - milestones.put(index, new MilestoneViewModel(milestone)); + Round round = (Round) tangle.load(Round.class, new IntegerIndex(index)); + if(round != null && round.hash != null) { + milestones.put(index, new MilestoneViewModel(round)); return true; } return false; } /** - * Fetches the first persistable {@link Milestone} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * Fetches the first persistable {@link Round} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle the tangle reference for the database * @return The new {@link MilestoneViewModel} * @throws Exception Thrown if the database fails to return a first object */ public static MilestoneViewModel first(Tangle tangle) throws Exception { - Pair milestonePair = tangle.getFirst(Milestone.class, IntegerIndex.class); + Pair milestonePair = tangle.getFirst(Round.class, IntegerIndex.class); if(milestonePair != null && milestonePair.hi != null) { - Milestone milestone = (Milestone) milestonePair.hi; - return new MilestoneViewModel(milestone); + Round round = (Round) milestonePair.hi; + return new MilestoneViewModel(round); } return null; } /** - * Fetches the most recent persistable {@link Milestone} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * Fetches the most recent persistable {@link Round} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle the tangle reference for the database * @return The new {@link MilestoneViewModel} * @throws Exception Thrown if the database fails to return a first object */ public static MilestoneViewModel latest(Tangle tangle) throws Exception { - Pair milestonePair = tangle.getLatest(Milestone.class, IntegerIndex.class); + Pair milestonePair = tangle.getLatest(Round.class, IntegerIndex.class); if(milestonePair != null && milestonePair.hi != null) { - Milestone milestone = (Milestone) milestonePair.hi; - return new MilestoneViewModel(milestone); + Round round = (Round) milestonePair.hi; + return new MilestoneViewModel(round); } return null; } /** - * Fetches the previously indexed persistable {@link Milestone} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * Fetches the previously indexed persistable {@link Round} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle the tangle reference for the database * @return The new {@link MilestoneViewModel} * @throws Exception Thrown if the database fails to return a first object */ public MilestoneViewModel previous(Tangle tangle) throws Exception { - Pair milestonePair = tangle.previous(Milestone.class, this.milestone.index); + Pair milestonePair = tangle.previous(Round.class, this.round.index); if(milestonePair != null && milestonePair.hi != null) { - Milestone milestone = (Milestone) milestonePair.hi; - return new MilestoneViewModel((Milestone) milestone); + Round round = (Round) milestonePair.hi; + return new MilestoneViewModel((Round) round); } return null; } /** - * Fetches the next indexed persistable {@link Milestone} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * Fetches the next indexed persistable {@link Round} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle The tangle reference for the database * @return The new {@link MilestoneViewModel} * @throws Exception Thrown if the database fails to return a first object */ public MilestoneViewModel next(Tangle tangle) throws Exception { - Pair milestonePair = tangle.next(Milestone.class, this.milestone.index); + Pair milestonePair = tangle.next(Round.class, this.round.index); if(milestonePair != null && milestonePair.hi != null) { - Milestone milestone = (Milestone) milestonePair.hi; - return new MilestoneViewModel((Milestone) milestone); + Round round = (Round) milestonePair.hi; + return new MilestoneViewModel((Round) round); } return null; } /** - * Fetches a {@link MilestoneViewModel} for the closest {@link Milestone} object previously indexed in the + * Fetches a {@link MilestoneViewModel} for the closest {@link Round} object previously indexed in the * database. The method starts at the provided index and works backwards through the database to try and find a * {@link MilestoneViewModel} for the previous indexes until a non null controller is found. * @@ -208,34 +208,34 @@ public static MilestoneViewModel findClosestNextMilestone(Tangle tangle, int ind } /** - * Save the {@link Milestone} object, indexed by its integer index, to the database. + * Save the {@link Round} object, indexed by its integer index, to the database. * * @param tangle The tangle reference for the database - * @return True if the {@link Milestone} object is saved correctly, False if not - * @throws Exception Thrown if there is an error while saving the {@link Milestone} object + * @return True if the {@link Round} object is saved correctly, False if not + * @throws Exception Thrown if there is an error while saving the {@link Round} object */ public boolean store(Tangle tangle) throws Exception { - return tangle.save(milestone, milestone.index); + return tangle.save(round, round.index); } - /**@return The {@link Hash} identifier of the {@link Milestone} object*/ + /**@return The {@link Hash} identifier of the {@link Round} object*/ public Hash getHash() { - return milestone.hash; + return round.hash; } - /**@return The integer index of the {@link Milestone} object*/ + /**@return The integer index of the {@link Round} object*/ public Integer index() { - return milestone.index.getValue(); + return round.index.getValue(); } /** - * Removes the {@link Milestone} object from the database. + * Removes the {@link Round} object from the database. * * @param tangle The tangle reference for the database - * @throws Exception Thrown if there is an error removing the {@link Milestone} object + * @throws Exception Thrown if there is an error removing the {@link Round} object */ public void delete(Tangle tangle) throws Exception { - tangle.delete(Milestone.class, milestone.index); + tangle.delete(Round.class, round.index); } /** diff --git a/src/main/java/net/helix/hlx/model/persistables/Hashes.java b/src/main/java/net/helix/hlx/model/persistables/Hashes.java index c6feff21..5169d57d 100644 --- a/src/main/java/net/helix/hlx/model/persistables/Hashes.java +++ b/src/main/java/net/helix/hlx/model/persistables/Hashes.java @@ -20,7 +20,7 @@ */ public class Hashes implements Persistable { public Set set = new LinkedHashSet<>(); - private static final byte delimiter = ",".getBytes()[0]; + static final byte delimiter = ",".getBytes()[0]; /** * Get byte array of the set. diff --git a/src/main/java/net/helix/hlx/model/persistables/Milestone.java b/src/main/java/net/helix/hlx/model/persistables/Milestone.java deleted file mode 100644 index f6dde73a..00000000 --- a/src/main/java/net/helix/hlx/model/persistables/Milestone.java +++ /dev/null @@ -1,45 +0,0 @@ -package net.helix.hlx.model.persistables; - -import net.helix.hlx.model.Hash; -import net.helix.hlx.model.HashFactory; -import net.helix.hlx.model.IntegerIndex; -import net.helix.hlx.storage.Persistable; -import net.helix.hlx.utils.Serializer; -import org.apache.commons.lang3.ArrayUtils; - -/** - * Created by paul on 4/11/17. - */ - - /** - * The MilestoneTracker model class is an implementation of the Persistable interface. - * It consists of the milestoneTracker index, an IntegerIndex , and the transaction hash of the milestoneTracker transaction. - */ -public class Milestone implements Persistable { - public IntegerIndex index; - public Hash hash; - - public byte[] bytes() { - return ArrayUtils.addAll(index.bytes(), hash.bytes()); - } - - public void read(byte[] bytes) { - if(bytes != null) { - index = new IntegerIndex(Serializer.getInteger(bytes)); - hash = HashFactory.TRANSACTION.create(bytes, Integer.BYTES, Hash.SIZE_IN_BYTES); - } - } - - @Override - public byte[] metadata() { - return new byte[0]; - } - - @Override - public void readMetadata(byte[] bytes) { } - - @Override - public boolean merge() { - return false; - } -} diff --git a/src/main/java/net/helix/hlx/model/persistables/Round.java b/src/main/java/net/helix/hlx/model/persistables/Round.java new file mode 100644 index 00000000..2cc59f2c --- /dev/null +++ b/src/main/java/net/helix/hlx/model/persistables/Round.java @@ -0,0 +1,37 @@ +package net.helix.hlx.model.persistables; + +import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; +import net.helix.hlx.model.IntegerIndex; +import net.helix.hlx.utils.Serializer; + +import org.apache.commons.lang3.ArrayUtils; + +import java.util.LinkedHashSet; + + /** + * The Round model class consists of a set of milestone hashes and a corresponding roundIndex. + */ +public class Round extends Hashes { + public IntegerIndex roundIndex; + + @Override + public byte[] bytes() { + byte[] hashes = set.parallelStream() + .map(Hash::bytes) + .reduce((a,b) -> ArrayUtils.addAll(ArrayUtils.add(a, delimiter), b)) + .orElse(new byte[0]); + return ArrayUtils.addAll(roundIndex.bytes(), hashes); + } + + @Override + public void read(byte[] bytes) { + if(bytes != null) { + roundIndex = new IntegerIndex(Serializer.getInteger(bytes)); + set = new LinkedHashSet<>((bytes.length - Integer.BYTES) / (1 + Hash.SIZE_IN_BYTES) + 1); + for (int i = Integer.BYTES; i < bytes.length; i += 1 + Hash.SIZE_IN_BYTES) { + set.add(HashFactory.TRANSACTION.create(bytes, i, Hash.SIZE_IN_BYTES)); + } + } + } +} diff --git a/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java index 7886e36b..f2a7f4ba 100644 --- a/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -3,7 +3,7 @@ import net.helix.hlx.controllers.MilestoneViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.model.IntegerIndex; -import net.helix.hlx.model.persistables.Milestone; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.model.persistables.Transaction; import net.helix.hlx.service.transactionpruning.TransactionPrunerJobStatus; import net.helix.hlx.service.transactionpruning.TransactionPruningException; @@ -235,7 +235,7 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { throw new RuntimeException(e); } } - } else if(Milestone.class.equals(element.hi)) { + } else if(Round.class.equals(element.hi)) { MilestoneViewModel.clear(((IntegerIndex) element.low).getValue()); } }); @@ -264,7 +264,7 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { MilestoneViewModel milestoneViewModel = MilestoneViewModel.get(getTangle(), getCurrentIndex()); if (milestoneViewModel != null) { elementsToDelete.add(new Pair<>(milestoneViewModel.getHash(), Transaction.class)); - elementsToDelete.add(new Pair<>(new IntegerIndex(milestoneViewModel.index()), Milestone.class)); + elementsToDelete.add(new Pair<>(new IntegerIndex(milestoneViewModel.index()), Round.class)); DAGHelper.get(getTangle()).traverseApprovees(milestoneViewModel.getHash(), approvedTransaction -> approvedTransaction.snapshotIndex() >= milestoneViewModel.index(), diff --git a/src/main/java/net/helix/hlx/storage/Tangle.java b/src/main/java/net/helix/hlx/storage/Tangle.java index d82ce948..ef7e3cef 100644 --- a/src/main/java/net/helix/hlx/storage/Tangle.java +++ b/src/main/java/net/helix/hlx/storage/Tangle.java @@ -21,7 +21,7 @@ public class Tangle { public static final Map> COLUMN_FAMILIES = new LinkedHashMap>() {{ put("transaction", Transaction.class); - put("milestone", Milestone.class); + put("milestone", Round.class); put("stateDiff", StateDiff.class); put("address", Address.class); put("approvee", Approvee.class); From 1dac88bf8dd4f00bae1ca7a9622563544298da1b Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 21 May 2019 15:22:15 +0200 Subject: [PATCH 002/223] ViewModel: MilestoneViewModel -> RoundViewModel --- ...toneViewModel.java => RoundViewModel.java} | 116 +++++++++--------- .../hlx/controllers/TransactionViewModel.java | 2 +- .../helix/hlx/model/persistables/Round.java | 8 +- src/main/java/net/helix/hlx/service/API.java | 4 +- .../hlx/service/ledger/LedgerService.java | 4 +- .../ledger/impl/LedgerServiceImpl.java | 8 +- .../service/milestone/MilestoneService.java | 4 +- .../impl/LatestMilestoneTrackerImpl.java | 4 +- .../impl/LatestSolidMilestoneTrackerImpl.java | 16 +-- .../milestone/impl/MilestoneServiceImpl.java | 32 ++--- .../hlx/service/snapshot/SnapshotService.java | 8 +- .../snapshot/impl/SnapshotServiceImpl.java | 42 +++---- .../impl/SpentAddressesServiceImpl.java | 4 +- .../impl/EntryPointSelectorImpl.java | 8 +- .../jobs/MilestonePrunerJob.java | 16 +-- 15 files changed, 139 insertions(+), 137 deletions(-) rename src/main/java/net/helix/hlx/controllers/{MilestoneViewModel.java => RoundViewModel.java} (64%) diff --git a/src/main/java/net/helix/hlx/controllers/MilestoneViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java similarity index 64% rename from src/main/java/net/helix/hlx/controllers/MilestoneViewModel.java rename to src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 5cbda46b..2f7b3894 100644 --- a/src/main/java/net/helix/hlx/controllers/MilestoneViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -10,16 +10,17 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.Set; /** * Acts as a controller interface for a {@link Round} hash object. This controller is used by the * {@link net.helix.hlx.service.milestone.LatestMilestoneTracker} to manipulate a {@link Round} object. */ -public class MilestoneViewModel { +public class RoundViewModel { private final Round round; - private static final Map milestones = new ConcurrentHashMap<>(); + private static final Map rounds = new ConcurrentHashMap<>(); - private MilestoneViewModel(final Round round) { + private RoundViewModel(final Round round) { this.round = round; } @@ -27,19 +28,19 @@ private MilestoneViewModel(final Round round) { * Removes the contents of the stored {@link Round} object set. */ public static void clear() { - milestones.clear(); + rounds.clear(); } /** - * This method removes a {@link MilestoneViewModel} from the cache. + * This method removes a {@link RoundViewModel} from the cache. * - * It is used by the {@link net.helix.hlx.service.transactionpruning.TransactionPruner} to remove milestones that + * It is used by the {@link net.helix.hlx.service.transactionpruning.TransactionPruner} to remove rounds that * were deleted in the database, so that the runtime environment correctly reflects the database state. * - * @param milestoneIndex the index of the milestone + * @param index the index of the round */ - public static void clear(int milestoneIndex) { - milestones.remove(milestoneIndex); + public static void clear(int index) { + rounds.remove(index); } /** @@ -47,35 +48,35 @@ public static void clear(int milestoneIndex) { * {@link Round} hash identifier, indexing this object to the integer {@link Round} index. * * @param index The finalized numerical index the {@link Round} object will be referenced by in the set - * @param milestoneHash The finalized {@link Hash} identifier for the {@link Round} object + * @param milestoneHashes The finalized {@link Hash} identifier for the {@link Round} object */ - public MilestoneViewModel(final int index, final Hash milestoneHash) { + public RoundViewModel(final int index, final Set milestoneHashes) { this.round = new Round(); this.round.index = new IntegerIndex(index); - round.hash = milestoneHash; + round.set = milestoneHashes; } /** - * Fetches an existing {@link MilestoneViewModel} if its index reference can be found in the controller. If the - * {@link MilestoneViewModel} is null, but the indexed {@link Round} object exists in the database, a new + * Fetches an existing {@link RoundViewModel} if its index reference can be found in the controller. If the + * {@link RoundViewModel} is null, but the indexed {@link Round} object exists in the database, a new * controller is created for the {@link Round} object. * * @param tangle The tangle reference for the database * @param index The integer index of the {@link Round} object that the controller should be returned for - * @return The {@link MilestoneViewModel} for the indexed {@link Round} object + * @return The {@link RoundViewModel} for the indexed {@link Round} object * @throws Exception Thrown if the database fails to load the indexed {@link Round} object */ - public static MilestoneViewModel get(Tangle tangle, int index) throws Exception { - MilestoneViewModel milestoneViewModel = milestones.get(index); - if(milestoneViewModel == null && load(tangle, index)) { - milestoneViewModel = milestones.get(index); + public static RoundViewModel get(Tangle tangle, int index) throws Exception { + RoundViewModel roundViewModel = rounds.get(index); + if(roundViewModel == null && load(tangle, index)) { + roundViewModel = rounds.get(index); } - return milestoneViewModel; + return roundViewModel; } /** * Fetches a {@link Round} object from the database using its integer index. If the {@link Round} and the - * associated {@link Hash} identifier are not null, a new {@link MilestoneViewModel} is created for the + * associated {@link Hash} identifier are not null, a new {@link RoundViewModel} is created for the * {@link Round} object, and it is placed into the Milestones set, indexed by the provided integer * index. * @@ -86,8 +87,8 @@ public static MilestoneViewModel get(Tangle tangle, int index) throws Exception */ public static boolean load(Tangle tangle, int index) throws Exception { Round round = (Round) tangle.load(Round.class, new IntegerIndex(index)); - if(round != null && round.hash != null) { - milestones.put(index, new MilestoneViewModel(round)); + if(round != null && round.set != null) { + rounds.put(index, new RoundViewModel(round)); return true; } return false; @@ -95,34 +96,34 @@ public static boolean load(Tangle tangle, int index) throws Exception { /** * Fetches the first persistable {@link Round} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. + * {@link RoundViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle the tangle reference for the database - * @return The new {@link MilestoneViewModel} + * @return The new {@link RoundViewModel} * @throws Exception Thrown if the database fails to return a first object */ - public static MilestoneViewModel first(Tangle tangle) throws Exception { + public static RoundViewModel first(Tangle tangle) throws Exception { Pair milestonePair = tangle.getFirst(Round.class, IntegerIndex.class); if(milestonePair != null && milestonePair.hi != null) { Round round = (Round) milestonePair.hi; - return new MilestoneViewModel(round); + return new RoundViewModel(round); } return null; } /** * Fetches the most recent persistable {@link Round} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. + * {@link RoundViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle the tangle reference for the database - * @return The new {@link MilestoneViewModel} + * @return The new {@link RoundViewModel} * @throws Exception Thrown if the database fails to return a first object */ - public static MilestoneViewModel latest(Tangle tangle) throws Exception { + public static RoundViewModel latest(Tangle tangle) throws Exception { Pair milestonePair = tangle.getLatest(Round.class, IntegerIndex.class); if(milestonePair != null && milestonePair.hi != null) { Round round = (Round) milestonePair.hi; - return new MilestoneViewModel(round); + return new RoundViewModel(round); } return null; } @@ -130,62 +131,62 @@ public static MilestoneViewModel latest(Tangle tangle) throws Exception { /** * Fetches the previously indexed persistable {@link Round} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. + * {@link RoundViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle the tangle reference for the database - * @return The new {@link MilestoneViewModel} + * @return The new {@link RoundViewModel} * @throws Exception Thrown if the database fails to return a first object */ - public MilestoneViewModel previous(Tangle tangle) throws Exception { + public RoundViewModel previous(Tangle tangle) throws Exception { Pair milestonePair = tangle.previous(Round.class, this.round.index); if(milestonePair != null && milestonePair.hi != null) { Round round = (Round) milestonePair.hi; - return new MilestoneViewModel((Round) round); + return new RoundViewModel((Round) round); } return null; } /** * Fetches the next indexed persistable {@link Round} object from the database and generates a new - * {@link MilestoneViewModel} from it. If no {@link Round} objects exist in the database, it will return null. + * {@link RoundViewModel} from it. If no {@link Round} objects exist in the database, it will return null. * * @param tangle The tangle reference for the database - * @return The new {@link MilestoneViewModel} + * @return The new {@link RoundViewModel} * @throws Exception Thrown if the database fails to return a first object */ - public MilestoneViewModel next(Tangle tangle) throws Exception { + public RoundViewModel next(Tangle tangle) throws Exception { Pair milestonePair = tangle.next(Round.class, this.round.index); if(milestonePair != null && milestonePair.hi != null) { Round round = (Round) milestonePair.hi; - return new MilestoneViewModel((Round) round); + return new RoundViewModel(round); } return null; } /** - * Fetches a {@link MilestoneViewModel} for the closest {@link Round} object previously indexed in the + * Fetches a {@link RoundViewModel} for the closest {@link Round} object previously indexed in the * database. The method starts at the provided index and works backwards through the database to try and find a - * {@link MilestoneViewModel} for the previous indexes until a non null controller is found. + * {@link RoundViewModel} for the previous indexes until a non null controller is found. * * @param tangle The tangle reference for the database * @param index The beginning index the method will work backwards from * @param minIndex The minimum index that should be found in the database - * @return The {@link MilestoneViewModel} of the closest found controller previously indexed in the database - * @throws Exception Thrown if there is a failure to fetch a previous {@link MilestoneViewModel} + * @return The {@link RoundViewModel} of the closest found controller previously indexed in the database + * @throws Exception Thrown if there is a failure to fetch a previous {@link RoundViewModel} */ - public static MilestoneViewModel findClosestPrevMilestone(Tangle tangle, int index, int minIndex) throws Exception { + public static RoundViewModel findClosestPrevRound(Tangle tangle, int index, int minIndex) throws Exception { // search for the previous milestone preceding our index - MilestoneViewModel previousMilestoneViewModel = null; + RoundViewModel previousRoundViewModel = null; int currentIndex = index; - while(previousMilestoneViewModel == null && --currentIndex >= minIndex) { - previousMilestoneViewModel = MilestoneViewModel.get(tangle, currentIndex); + while(previousRoundViewModel == null && --currentIndex >= minIndex) { + previousRoundViewModel = RoundViewModel.get(tangle, currentIndex); } - return previousMilestoneViewModel; + return previousRoundViewModel; } /** - * This method looks for the next milestone after a given index. + * This method looks for the next round after a given index. * * In contrast to the {@link #next} method we do not rely on the insertion order in the database but actively search * for the milestone that was issued next by the coordinator (coo-order preserved). @@ -196,15 +197,15 @@ public static MilestoneViewModel findClosestPrevMilestone(Tangle tangle, int ind * @return the milestone which follows directly after the given index or null if none was found * @throws Exception if anything goes wrong while loading entries from the database */ - public static MilestoneViewModel findClosestNextMilestone(Tangle tangle, int index, int maxIndex) throws Exception { + public static RoundViewModel findClosestNextRound(Tangle tangle, int index, int maxIndex) throws Exception { // search for the next milestone following our index - MilestoneViewModel nextMilestoneViewModel = null; + RoundViewModel nextRoundViewModel = null; int currentIndex = index; - while(nextMilestoneViewModel == null && ++currentIndex <= maxIndex) { - nextMilestoneViewModel = MilestoneViewModel.get(tangle, currentIndex); + while(nextRoundViewModel == null && ++currentIndex <= maxIndex) { + nextRoundViewModel = RoundViewModel.get(tangle, currentIndex); } - return nextMilestoneViewModel; + return nextRoundViewModel; } /** @@ -219,8 +220,8 @@ public boolean store(Tangle tangle) throws Exception { } /**@return The {@link Hash} identifier of the {@link Round} object*/ - public Hash getHash() { - return round.hash; + public Set getHashes() { + return round.set; } /**@return The integer index of the {@link Round} object*/ @@ -244,9 +245,10 @@ public void delete(Tangle tangle) throws Exception { * It can be used to directly append the milestone in error and debug messages. * * @return human readable string representation of the milestone + * TODO: Either get a specific milestone from the set and cast that to string or cast the whole set to string in a loop and print. */ @Override public String toString() { - return "milestone #" + index() + " (" + getHash().toString() + ")"; + return "milestone #" + index() + " (" + getHashes().toString() + ")"; } } diff --git a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java index c8ff48f9..244a6ff6 100644 --- a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java @@ -653,7 +653,7 @@ public void isMilestone(Tangle tangle, Snapshot initialSnapshot, final boolean i * The {@link Transaction#milestone} flag indicates if the {@link Transaction} is a coordinator issued milestone. It * allows us to differentiate the two types of transactions (normal transactions / milestones) very fast and * efficiently without issuing further database queries or even full verifications of the signature. If it is set to - * true one can for example use the snapshotIndex() method to retrieve the corresponding {@link MilestoneViewModel} + * true one can for example use the snapshotIndex() method to retrieve the corresponding {@link RoundViewModel} * object. * * @return true if the {@link Transaction} is a milestone and false otherwise diff --git a/src/main/java/net/helix/hlx/model/persistables/Round.java b/src/main/java/net/helix/hlx/model/persistables/Round.java index 2cc59f2c..9745c366 100644 --- a/src/main/java/net/helix/hlx/model/persistables/Round.java +++ b/src/main/java/net/helix/hlx/model/persistables/Round.java @@ -10,10 +10,10 @@ import java.util.LinkedHashSet; /** - * The Round model class consists of a set of milestone hashes and a corresponding roundIndex. + * The Round model class consists of a set of milestone hashes and a corresponding index. */ public class Round extends Hashes { - public IntegerIndex roundIndex; + public IntegerIndex index; @Override public byte[] bytes() { @@ -21,13 +21,13 @@ public byte[] bytes() { .map(Hash::bytes) .reduce((a,b) -> ArrayUtils.addAll(ArrayUtils.add(a, delimiter), b)) .orElse(new byte[0]); - return ArrayUtils.addAll(roundIndex.bytes(), hashes); + return ArrayUtils.addAll(index.bytes(), hashes); } @Override public void read(byte[] bytes) { if(bytes != null) { - roundIndex = new IntegerIndex(Serializer.getInteger(bytes)); + index = new IntegerIndex(Serializer.getInteger(bytes)); set = new LinkedHashSet<>((bytes.length - Integer.BYTES) / (1 + Hash.SIZE_IN_BYTES) + 1); for (int i = Integer.BYTES; i < bytes.length; i += 1 + Hash.SIZE_IN_BYTES) { set.add(HashFactory.TRANSACTION.create(bytes, i, Hash.SIZE_IN_BYTES)); diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index d0e2e5d8..a853793b 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -6,7 +6,7 @@ import net.helix.hlx.conf.ConsensusConfig; import net.helix.hlx.controllers.AddressViewModel; import net.helix.hlx.controllers.BundleViewModel; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TagViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.*; @@ -878,7 +878,7 @@ private AbstractResponse interruptAttachingToTangleStatement(){ **/ private AbstractResponse getNodeInfoStatement() throws Exception { String name = instance.configuration.isTestnet() ? HLX.TESTNET_NAME : HLX.MAINNET_NAME; - MilestoneViewModel milestone = MilestoneViewModel.first(instance.tangle); + RoundViewModel milestone = RoundViewModel.first(instance.tangle); return GetNodeInfoResponse.create(name, HLX.VERSION, Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().freeMemory(), diff --git a/src/main/java/net/helix/hlx/service/ledger/LedgerService.java b/src/main/java/net/helix/hlx/service/ledger/LedgerService.java index 257439cf..84b37518 100644 --- a/src/main/java/net/helix/hlx/service/ledger/LedgerService.java +++ b/src/main/java/net/helix/hlx/service/ledger/LedgerService.java @@ -1,6 +1,6 @@ package net.helix.hlx.service.ledger; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.service.Graphstream; @@ -45,7 +45,7 @@ public interface LedgerService { * @return {@code true} if the milestone could be applied to the ledger and {@code false} otherwise * @throws LedgerException if anything goes wrong while modifying the ledger state */ - boolean applyMilestoneToLedger(MilestoneViewModel milestone) throws LedgerException; + boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerException; /** * Checks the consistency of the combined balance changes of the given tips.
diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index f0714d75..c8b528eb 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -1,7 +1,7 @@ package net.helix.hlx.service.ledger.impl; import net.helix.hlx.BundleValidator; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.StateDiffViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; @@ -91,7 +91,7 @@ public boolean updateDiff(Set approvedHashes, final Map diff, @Override public void restoreLedgerState() throws LedgerException { try { - Optional milestone = milestoneService.findLatestProcessedSolidMilestoneInDatabase(); + Optional milestone = milestoneService.findLatestProcessedSolidMilestoneInDatabase(); if (milestone.isPresent()) { snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.get().index()); } @@ -101,7 +101,7 @@ public void restoreLedgerState() throws LedgerException { } @Override - public boolean applyMilestoneToLedger(MilestoneViewModel milestone) throws LedgerException { + public boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerException { if(generateStateDiff(milestone)) { try { snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.index()); @@ -262,7 +262,7 @@ public Map generateBalanceDiff(Set visitedTransactions, Hash s * @return {@code true} if the {@link net.helix.hlx.model.StateDiff} could be generated and {@code false} otherwise * @throws LedgerException if anything unexpected happens while generating the {@link net.helix.hlx.model.StateDiff} */ - private boolean generateStateDiff(MilestoneViewModel milestone) throws LedgerException { + private boolean generateStateDiff(RoundViewModel milestone) throws LedgerException { try { TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, milestone.getHash()); diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index 8734a12b..1d444952 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -1,6 +1,6 @@ package net.helix.hlx.service.milestone; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; @@ -24,7 +24,7 @@ public interface MilestoneService { * processed solid milestone can be found * @throws MilestoneException if anything unexpected happend while performing the search */ - Optional findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException; + Optional findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException; /** * Analyzes the given transaction to determine if it is a valid milestone.
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 0d24a1a2..28f18466 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -2,7 +2,7 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.AddressViewModel; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; @@ -374,7 +374,7 @@ private void bootstrapLatestMilestoneValue() { setLatestMilestone(latestSnapshot.getHash(), latestSnapshot.getIndex()); try { - MilestoneViewModel lastMilestoneInDatabase = MilestoneViewModel.latest(tangle); + RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getLatestMilestoneIndex()) { setLatestMilestone(lastMilestoneInDatabase.getHash(), lastMilestoneInDatabase.index()); } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index af97b42d..e1944ad3 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -1,6 +1,6 @@ package net.helix.hlx.service.milestone.impl; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.service.ledger.LedgerService; @@ -81,7 +81,7 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac private int errorCausingMilestoneIndex = Integer.MAX_VALUE; /** - * Counter for the backoff repair strategy (see {@link #repairCorruptedMilestone(MilestoneViewModel)}.
+ * Counter for the backoff repair strategy (see {@link #repairCorruptedMilestone(RoundViewModel)}.
*/ private int repairBackoffCounter = 0; @@ -139,9 +139,9 @@ public void trackLatestSolidMilestone() throws MilestoneException { try { int currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); if (currentSolidMilestoneIndex < latestMilestoneTracker.getLatestMilestoneIndex()) { - MilestoneViewModel nextMilestone; + RoundViewModel nextMilestone; while (!Thread.currentThread().isInterrupted() && - (nextMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null && + (nextMilestone = RoundViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null && TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { //TODO: graphstream @@ -195,7 +195,7 @@ private void latestSolidMilestoneTrackerThread() { * @param milestone the milestone that shall be applied to the ledger state * @throws Exception if anything unexpected goes wrong while applying the milestone to the ledger */ - private void applySolidMilestoneToLedger(MilestoneViewModel milestone) throws Exception { + private void applySolidMilestoneToLedger(RoundViewModel milestone) throws Exception { if (ledgerService.applyMilestoneToLedger(milestone)) { if (isRepairRunning() && isRepairSuccessful(milestone)) { stopRepair(); @@ -225,7 +225,7 @@ private boolean isRepairRunning() { * @param processedMilestone the currently processed milestone * @return {@code true} if we advanced to a milestone following the corrupted one and {@code false} otherwise */ - private boolean isRepairSuccessful(MilestoneViewModel processedMilestone) { + private boolean isRepairSuccessful(RoundViewModel processedMilestone) { return processedMilestone.index() > errorCausingMilestoneIndex; } @@ -233,7 +233,7 @@ private boolean isRepairSuccessful(MilestoneViewModel processedMilestone) { * Resets the internal variables that are used to keep track of the repair process.
*
* It gets called whenever we advance to a milestone that has a higher milestone index than the milestone that - * initially caused the repair routine to kick in (see {@link #repairCorruptedMilestone(MilestoneViewModel)}.
+ * initially caused the repair routine to kick in (see {@link #repairCorruptedMilestone(RoundViewModel)}.
*/ private void stopRepair() { repairBackoffCounter = 0; @@ -298,7 +298,7 @@ private void logChange(int prevSolidMilestoneIndex) { * @param errorCausingMilestone the milestone that failed to be applied * @throws MilestoneException if we failed to reset the corrupted milestone */ - private void repairCorruptedMilestone(MilestoneViewModel errorCausingMilestone) throws MilestoneException { + private void repairCorruptedMilestone(RoundViewModel errorCausingMilestone) throws MilestoneException { if(repairBackoffCounter++ == 0) { errorCausingMilestoneIndex = errorCausingMilestone.index(); } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 6f4e91c8..0bdee47e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -2,7 +2,7 @@ import net.helix.hlx.BundleValidator; import net.helix.hlx.conf.ConsensusConfig; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.Sha3; import net.helix.hlx.crypto.Winternitz; @@ -102,10 +102,10 @@ public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvide * amount of database requests to a minimum (even with a huge amount of milestones in the database).
*/ @Override - public Optional findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException { + public Optional findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException { try { // if we have no milestone in our database -> abort - MilestoneViewModel latestMilestone = MilestoneViewModel.latest(tangle); + RoundViewModel latestMilestone = RoundViewModel.latest(tangle); if (latestMilestone == null) { return Optional.empty(); } @@ -116,7 +116,7 @@ public Optional findLatestProcessedSolidMilestoneInDatabase( } // trivial case #2: the node was fully synced but the last milestone was not processed, yet - MilestoneViewModel latestMilestonePredecessor = MilestoneViewModel.findClosestPrevMilestone(tangle, + RoundViewModel latestMilestonePredecessor = RoundViewModel.findClosestPrevMilestone(tangle, latestMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex()); if (latestMilestonePredecessor != null && wasMilestoneAppliedToLedger(latestMilestonePredecessor)) { return Optional.of(latestMilestonePredecessor); @@ -161,7 +161,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM } try { - if (MilestoneViewModel.get(tangle, milestoneIndex) != null) { + if (RoundViewModel.get(tangle, milestoneIndex) != null) { // Already validated. return VALID; } @@ -205,9 +205,9 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM (HashFactory.ADDRESS.create(merkleRoot)).equals( HashFactory.ADDRESS.create(config.getCoordinator()))) { - MilestoneViewModel newMilestoneViewModel = new MilestoneViewModel(milestoneIndex, + RoundViewModel newRoundViewModel = new RoundViewModel(milestoneIndex, transactionViewModel.getHash()); - newMilestoneViewModel.store(tangle); + newRoundViewModel.store(tangle); // if we find a NEW milestone that should have been processed before our latest solid // milestone -> reset the ledger state and check the milestones again @@ -268,16 +268,16 @@ public boolean isTransactionConfirmed(TransactionViewModel transaction) { * processed solid milestone can be found * @throws Exception if anything unexpected happens while performing the search */ - private Optional binarySearchLatestProcessedSolidMilestoneInDatabase( - MilestoneViewModel latestMilestone) throws Exception { + private Optional binarySearchLatestProcessedSolidMilestoneInDatabase( + RoundViewModel latestMilestone) throws Exception { - Optional lastAppliedCandidate = Optional.empty(); + Optional lastAppliedCandidate = Optional.empty(); int rangeEnd = latestMilestone.index(); int rangeStart = snapshotProvider.getInitialSnapshot().getIndex() + 1; while (rangeEnd - rangeStart >= 0) { // if no candidate found in range -> return last candidate - MilestoneViewModel currentCandidate = getMilestoneInMiddleOfRange(rangeStart, rangeEnd); + RoundViewModel currentCandidate = getMilestoneInMiddleOfRange(rangeStart, rangeEnd); if (currentCandidate == null) { return lastAppliedCandidate; } @@ -314,13 +314,13 @@ private Optional binarySearchLatestProcessedSolidMilestoneIn * found * @throws Exception if anything unexpected happens while trying to get the milestone */ - private MilestoneViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) throws Exception { + private RoundViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) throws Exception { int range = rangeEnd - rangeStart; int middleOfRange = rangeEnd - range / 2; - MilestoneViewModel milestone = MilestoneViewModel.findClosestNextMilestone(tangle, middleOfRange - 1, rangeEnd); + RoundViewModel milestone = RoundViewModel.findClosestNextMilestone(tangle, middleOfRange - 1, rangeEnd); if (milestone == null) { - milestone = MilestoneViewModel.findClosestPrevMilestone(tangle, middleOfRange, rangeStart); + milestone = RoundViewModel.findClosestPrevMilestone(tangle, middleOfRange, rangeStart); } return milestone; @@ -337,7 +337,7 @@ private MilestoneViewModel getMilestoneInMiddleOfRange(int rangeStart, int range * @return {@code true} if the milestone has been processed by IRI before and {@code false} otherwise * @throws Exception if anything unexpected happens while checking the milestone */ - private boolean wasMilestoneAppliedToLedger(MilestoneViewModel milestone) throws Exception { + private boolean wasMilestoneAppliedToLedger(RoundViewModel milestone) throws Exception { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash()); return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT && milestoneTransaction.snapshotIndex() != 0; @@ -520,7 +520,7 @@ private void resetCorruptedMilestone(int index, Set processedTransactions) } log.info("resetting corrupted milestone #" + index); try { - MilestoneViewModel milestoneToRepair = MilestoneViewModel.get(tangle, index); + RoundViewModel milestoneToRepair = RoundViewModel.get(tangle, index); if(milestoneToRepair != null) { if(milestoneToRepair.index() <= snapshotProvider.getLatestSnapshot().getIndex()) { snapshotService.rollBackMilestones(snapshotProvider.getLatestSnapshot(), milestoneToRepair.index()); diff --git a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java index 909c5145..6158fea5 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java +++ b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java @@ -1,6 +1,6 @@ package net.helix.hlx.service.snapshot; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.transactionpruning.TransactionPruner; @@ -78,7 +78,7 @@ void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, Transactio * @return a local snapshot of the full ledger state at the given milestone * @throws SnapshotException if anything goes wrong while generating the local snapshot */ - Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, MilestoneViewModel targetMilestone) throws + Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundViewModel targetMilestone) throws SnapshotException; /** @@ -92,7 +92,7 @@ Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, Milesto * @return a map of solid entry points associating their hash to the milestone index that confirmed them * @throws SnapshotException if anything goes wrong while generating the solid entry points */ - Map generateSolidEntryPoints(MilestoneViewModel targetMilestone) throws SnapshotException; + Map generateSolidEntryPoints(RoundViewModel targetMilestone) throws SnapshotException; /** * This method generates the map of seen milestones that happened after the given target milestone. @@ -107,5 +107,5 @@ Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, Milesto * @throws SnapshotException if anything goes wrong while generating the solid entry points */ Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - MilestoneViewModel targetMilestone) throws SnapshotException; + RoundViewModel targetMilestone) throws SnapshotException; } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 692210b8..36241dd1 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -2,7 +2,7 @@ import net.helix.hlx.conf.SnapshotConfig; import net.helix.hlx.controllers.ApproveeViewModel; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.StateDiffViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; @@ -126,13 +126,13 @@ public SnapshotServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider public void replayMilestones(Snapshot snapshot, int targetMilestoneIndex) throws SnapshotException { Map balanceChanges = new HashMap<>(); Set skippedMilestones = new HashSet<>(); - MilestoneViewModel lastAppliedMilestone = null; + RoundViewModel lastAppliedMilestone = null; try { for (int currentMilestoneIndex = snapshot.getIndex() + 1; currentMilestoneIndex <= targetMilestoneIndex; currentMilestoneIndex++) { - MilestoneViewModel currentMilestone = MilestoneViewModel.get(tangle, currentMilestoneIndex); + RoundViewModel currentMilestone = RoundViewModel.get(tangle, currentMilestoneIndex); if (currentMilestone != null) { StateDiffViewModel stateDiffViewModel = StateDiffViewModel.load(tangle, currentMilestone.getHash()); if(!stateDiffViewModel.isEmpty()) { @@ -212,7 +212,7 @@ public void rollBackMilestones(Snapshot snapshot, int targetMilestoneIndex) thro public void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, TransactionPruner transactionPruner) throws SnapshotException { - MilestoneViewModel targetMilestone = determineMilestoneForLocalSnapshot(tangle, snapshotProvider, config); + RoundViewModel targetMilestone = determineMilestoneForLocalSnapshot(tangle, snapshotProvider, config); Snapshot newSnapshot = generateSnapshot(latestMilestoneTracker, targetMilestone); @@ -230,7 +230,7 @@ public void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, Tra * {@inheritDoc} */ @Override - public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, MilestoneViewModel targetMilestone) + public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundViewModel targetMilestone) throws SnapshotException { if (targetMilestone == null) { @@ -275,7 +275,7 @@ public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, * {@inheritDoc} */ @Override - public Map generateSolidEntryPoints(MilestoneViewModel targetMilestone) throws SnapshotException { + public Map generateSolidEntryPoints(RoundViewModel targetMilestone) throws SnapshotException { Map solidEntryPoints = new HashMap<>(); solidEntryPoints.put(Hash.NULL_HASH, targetMilestone.index()); @@ -290,7 +290,7 @@ public Map generateSolidEntryPoints(MilestoneViewModel targetMile */ @Override public Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - MilestoneViewModel targetMilestone) throws SnapshotException { + RoundViewModel targetMilestone) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [processing seen milestones]", log) @@ -298,8 +298,8 @@ public Map generateSeenMilestones(LatestMilestoneTracker latestMi Map seenMilestones = new HashMap<>(); try { - MilestoneViewModel seenMilestone = targetMilestone; - while ((seenMilestone = MilestoneViewModel.findClosestNextMilestone(tangle, seenMilestone.index(), + RoundViewModel seenMilestone = targetMilestone; + while ((seenMilestone = RoundViewModel.findClosestNextMilestone(tangle, seenMilestone.index(), latestMilestoneTracker.getLatestMilestoneIndex())) != null) { seenMilestones.put(seenMilestone.getHash(), seenMilestone.index()); @@ -376,7 +376,7 @@ private boolean rollbackLastMilestone(Tangle tangle, Snapshot snapshot) throws S } // otherwise set metadata of the previous milestone - MilestoneViewModel currentMilestone = MilestoneViewModel.get(tangle, currentIndex); + RoundViewModel currentMilestone = RoundViewModel.get(tangle, currentIndex); snapshot.setIndex(currentMilestone.index()); snapshot.setHash(currentMilestone.getHash()); snapshot.setTimestamp(TransactionViewModel.fromHash(tangle, currentMilestone.getHash()).getTimestamp()); @@ -401,14 +401,14 @@ private boolean rollbackLastMilestone(Tangle tangle, Snapshot snapshot) throws S * @return the target milestone for the local snapshot * @throws SnapshotException if anything goes wrong while determining the target milestone for the local snapshot */ - private MilestoneViewModel determineMilestoneForLocalSnapshot(Tangle tangle, SnapshotProvider snapshotProvider, - SnapshotConfig config) throws SnapshotException { + private RoundViewModel determineMilestoneForLocalSnapshot(Tangle tangle, SnapshotProvider snapshotProvider, + SnapshotConfig config) throws SnapshotException { int targetMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex() - config.getLocalSnapshotsDepth(); - MilestoneViewModel targetMilestone; + RoundViewModel targetMilestone; try { - targetMilestone = MilestoneViewModel.findClosestPrevMilestone(tangle, targetMilestoneIndex, + targetMilestone = RoundViewModel.findClosestPrevMilestone(tangle, targetMilestoneIndex, snapshotProvider.getInitialSnapshot().getIndex()); } catch (Exception e) { throw new SnapshotException("could not load the target milestone", e); @@ -468,7 +468,7 @@ private void cleanupExpiredSolidEntryPoints(Tangle tangle, Map ol * @throws SnapshotException if anything goes wrong while issuing the cleanup jobs */ private void cleanupOldData(SnapshotConfig config, TransactionPruner transactionPruner, - MilestoneViewModel targetMilestone) throws SnapshotException { + RoundViewModel targetMilestone) throws SnapshotException { int targetIndex = targetMilestone.index() - config.getLocalSnapshotsPruningDelay(); int startingIndex = config.getMilestoneStartIndex() + 1; @@ -578,7 +578,7 @@ private boolean isOrphaned(Tangle tangle, TransactionViewModel transaction, * @param targetMilestone milestone that is used as an anchor for our checks * @return true if the transaction is a solid entry point and false otherwise */ - private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, MilestoneViewModel targetMilestone) { + private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundViewModel targetMilestone) { Set unconfirmedApprovers = new HashSet<>(); try { @@ -620,7 +620,7 @@ private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, Milestone * @param solidEntryPoints map that is used to collect the solid entry points */ private void processOldSolidEntryPoints(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneViewModel targetMilestone, Map solidEntryPoints) { + RoundViewModel targetMilestone, Map solidEntryPoints) { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [analyzing old solid entry points]", log) @@ -654,7 +654,7 @@ && isSolidEntryPoint(tangle, hash, targetMilestone)) { * @throws SnapshotException if anything goes wrong while determining the solid entry points */ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneViewModel targetMilestone, Map solidEntryPoints) throws SnapshotException { + RoundViewModel targetMilestone, Map solidEntryPoints) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [generating solid entry points]", log); @@ -663,11 +663,11 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot progressLogger.start(Math.min(targetMilestone.index() - snapshotProvider.getInitialSnapshot().getIndex(), OUTER_SHELL_SIZE)); - MilestoneViewModel nextMilestone = targetMilestone; + RoundViewModel nextMilestone = targetMilestone; while (nextMilestone != null && nextMilestone.index() > snapshotProvider.getInitialSnapshot().getIndex() && progressLogger.getCurrentStep() < progressLogger.getStepCount()) { - MilestoneViewModel currentMilestone = nextMilestone; + RoundViewModel currentMilestone = nextMilestone; DAGHelper.get(tangle).traverseApprovees( currentMilestone.getHash(), currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(), @@ -680,7 +680,7 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot solidEntryPoints.put(currentMilestone.getHash(), targetMilestone.index()); - nextMilestone = MilestoneViewModel.findClosestPrevMilestone(tangle, currentMilestone.index(), + nextMilestone = RoundViewModel.findClosestPrevMilestone(tangle, currentMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex()); progressLogger.progress(); diff --git a/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java index 60913941..4e790f9f 100644 --- a/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -2,7 +2,7 @@ import net.helix.hlx.BundleValidator; import net.helix.hlx.controllers.AddressViewModel; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.service.snapshot.SnapshotProvider; @@ -79,7 +79,7 @@ public void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) Set addressesToCheck = new HashSet<>(); try { for (int i = fromMilestoneIndex; i < toMilestoneIndex; i++) { - MilestoneViewModel currentMilestone = MilestoneViewModel.get(tangle, i); + RoundViewModel currentMilestone = RoundViewModel.get(tangle, i); if (currentMilestone != null) { DAGHelper.get(tangle).traverseApprovees( currentMilestone.getHash(), diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index a5eed291..da5da4ac 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -1,6 +1,6 @@ package net.helix.hlx.service.tipselection.impl; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; @@ -35,10 +35,10 @@ public EntryPointSelectorImpl(Tangle tangle, SnapshotProvider snapshotProvider, public Hash getEntryPoint(int depth) throws Exception { int milestoneIndex = Math.max(snapshotProvider.getLatestSnapshot().getIndex() - depth - 1, snapshotProvider.getInitialSnapshot().getIndex()); - MilestoneViewModel milestoneViewModel = MilestoneViewModel.findClosestNextMilestone(tangle, milestoneIndex, + RoundViewModel roundViewModel = RoundViewModel.findClosestNextMilestone(tangle, milestoneIndex, latestMilestoneTracker.getLatestMilestoneIndex()); - if (milestoneViewModel != null && milestoneViewModel.getHash() != null) { - return milestoneViewModel.getHash(); + if (roundViewModel != null && roundViewModel.getHash() != null) { + return roundViewModel.getHash(); } return snapshotProvider.getLatestSnapshot().getHash(); diff --git a/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java index f2a7f4ba..9d191451 100644 --- a/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -1,6 +1,6 @@ package net.helix.hlx.service.transactionpruning.jobs; -import net.helix.hlx.controllers.MilestoneViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.persistables.Round; @@ -236,7 +236,7 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { } } } else if(Round.class.equals(element.hi)) { - MilestoneViewModel.clear(((IntegerIndex) element.low).getValue()); + RoundViewModel.clear(((IntegerIndex) element.low).getValue()); } }); @@ -261,13 +261,13 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { try { List>> elementsToDelete = new ArrayList<>(); - MilestoneViewModel milestoneViewModel = MilestoneViewModel.get(getTangle(), getCurrentIndex()); - if (milestoneViewModel != null) { - elementsToDelete.add(new Pair<>(milestoneViewModel.getHash(), Transaction.class)); - elementsToDelete.add(new Pair<>(new IntegerIndex(milestoneViewModel.index()), Round.class)); + RoundViewModel roundViewModel = RoundViewModel.get(getTangle(), getCurrentIndex()); + if (roundViewModel != null) { + elementsToDelete.add(new Pair<>(roundViewModel.getHash(), Transaction.class)); + elementsToDelete.add(new Pair<>(new IntegerIndex(roundViewModel.index()), Round.class)); - DAGHelper.get(getTangle()).traverseApprovees(milestoneViewModel.getHash(), - approvedTransaction -> approvedTransaction.snapshotIndex() >= milestoneViewModel.index(), + DAGHelper.get(getTangle()).traverseApprovees(roundViewModel.getHash(), + approvedTransaction -> approvedTransaction.snapshotIndex() >= roundViewModel.index(), approvedTransaction -> { /*if (approvedTransaction.value() < 0 && !spentAddressesService.wasAddressSpentFrom(approvedTransaction.getAddressHash())) { From d5fee0a8258264816d9b1fdf4ea10230c2860e42 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 21 May 2019 15:29:23 +0200 Subject: [PATCH 003/223] Added getHashes, addMilestone and size to RoundViewModel --- .../net/helix/hlx/controllers/RoundViewModel.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 2f7b3894..01d7c71d 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -219,6 +219,18 @@ public boolean store(Tangle tangle) throws Exception { return tangle.save(round, round.index); } + public int size() { + return round.set.size(); + } + + public boolean addMilestone(Hash milestoneHash) { + return getHashes().add(milestoneHash); + } + + public boolean update(Tangle tangle) throws Exception { + return tangle.update(round, round.index, "round"); + } + /**@return The {@link Hash} identifier of the {@link Round} object*/ public Set getHashes() { return round.set; From 0c95900f767d0d40f6b783d8130d27e53d471d7d Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 21 May 2019 15:36:46 +0200 Subject: [PATCH 004/223] StateDiffViewModel: milestoneHash -> roundIndex --- .../hlx/controllers/StateDiffViewModel.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/StateDiffViewModel.java b/src/main/java/net/helix/hlx/controllers/StateDiffViewModel.java index 510e3fab..91989dcf 100644 --- a/src/main/java/net/helix/hlx/controllers/StateDiffViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/StateDiffViewModel.java @@ -1,6 +1,7 @@ package net.helix.hlx.controllers; import net.helix.hlx.model.Hash; +import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.StateDiff; import net.helix.hlx.storage.Tangle; @@ -12,18 +13,18 @@ /** * The StateDiffViewModel class interacts with the StateDiff model class. - * It consists of a milestoneTracker hash (transaction hash) and the statediff model. + * It consists of a roundIndex and the statediff model. */ public class StateDiffViewModel { private StateDiff stateDiff; - private Hash hash; + private IntegerIndex roundIndex; - public static StateDiffViewModel load(Tangle tangle, Hash hash) throws Exception { - return new StateDiffViewModel((StateDiff) tangle.load(StateDiff.class, hash), hash); + public static StateDiffViewModel load(Tangle tangle, final int roundIndex) throws Exception { + return new StateDiffViewModel((StateDiff) tangle.load(StateDiff.class, new IntegerIndex(roundIndex)), roundIndex); } - public StateDiffViewModel(final Map state, final Hash hash) { - this.hash = hash; + public StateDiffViewModel(final Map state, final int roundIndex) { + this.roundIndex = new IntegerIndex(roundIndex); this.stateDiff = new StateDiff(); this.stateDiff.state = state; } @@ -32,8 +33,8 @@ public static boolean maybeExists(Tangle tangle, Hash hash) throws Exception { return tangle.maybeHas(StateDiff.class, hash); } - StateDiffViewModel(final StateDiff diff, final Hash hash) { - this.hash = hash; + StateDiffViewModel(final StateDiff diff, final int roundIndex) { + this.roundIndex = new IntegerIndex(roundIndex); this.stateDiff = diff == null || diff.state == null ? new StateDiff(): diff; } @@ -41,8 +42,8 @@ public boolean isEmpty() { return stateDiff == null || stateDiff.state == null || stateDiff.state.size() == 0; } - public Hash getHash() { - return hash; + public int getRoundIndex() { + return this.roundIndex.getValue(); } public Map getDiff() { @@ -50,11 +51,10 @@ public Map getDiff() { } public boolean store(Tangle tangle) throws Exception { - //return Tangle.instance().save(stateDiff, hash).get(); - return tangle.save(stateDiff, hash); + return tangle.save(stateDiff, roundIndex); } public void delete(Tangle tangle) throws Exception { - tangle.delete(StateDiff.class, hash); + tangle.delete(StateDiff.class, roundIndex); } } From f1a81ec55a0933af20901d2ee6a7726894c5d1b6 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 21 May 2019 17:27:50 +0200 Subject: [PATCH 005/223] LatestMilestoneTracker finality update (wip) --- .../net/helix/hlx/conf/BaseHelixConfig.java | 20 +++-- .../net/helix/hlx/conf/MilestoneConfig.java | 10 ++- .../net/helix/hlx/conf/TestnetConfig.java | 2 +- .../milestone/LatestMilestoneTracker.java | 11 +-- .../impl/LatestMilestoneTrackerImpl.java | 73 ++++++++++--------- 5 files changed, 64 insertions(+), 52 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 0b6c310a..ac10629d 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1,5 +1,10 @@ package net.helix.hlx.conf; +import net.helix.hlx.HLX; +import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; +import net.helix.hlx.utils.HelixUtils; + import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; @@ -7,11 +12,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.lang3.ArrayUtils; -import net.helix.hlx.HLX; -import net.helix.hlx.utils.HelixUtils; - import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /* Note: the fields in this class are being deserialized from Jackson so they must follow Java Bean convention. Meaning that every field must have a getter that is prefixed with `get` unless it is a boolean and then it should be @@ -665,8 +671,8 @@ protected void setCacheSizeBytes(int cacheSizeBytes) { } @Override - public String getCoordinator() { - return Defaults.COORDINATOR_ADDRESS; + public Set getValidatorAddresses() { + return Defaults.VALIDATOR_ADDRESSES; } @Override @@ -828,8 +834,8 @@ public interface Defaults { //PoW int POW_THREADS = 0; - //Coo - String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; + //Validator Addresses + Set VALIDATOR_ADDRESSES = Stream.of(HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a")).collect(Collectors.toSet()); //Snapshot boolean LOCAL_SNAPSHOTS_ENABLED = true; diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index 94c1a3c3..99c97956 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -1,19 +1,23 @@ package net.helix.hlx.conf; +import net.helix.hlx.model.Hash; + +import java.util.Set; + /** * Configs that should be used for tracking milestones */ public interface MilestoneConfig extends Config { /** - * @return Descriptions#COORDINATOR + * @return Descriptions#VALIDATOR_ADDRESSES */ - String getCoordinator(); + Set getValidatorAddresses(); /** * @return {@value Descriptions#DONT_VALIDATE_TESTNET_MILESTONE_SIG} */ boolean isDontValidateTestnetMilestoneSig(); interface Descriptions { - String COORDINATOR = "The address of the coordinator"; + String VALIDATOR_ADDRESSES = "The addresses of nodes that are allowed to publish milestones"; String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable coordinator validation on testnet"; } } diff --git a/src/main/java/net/helix/hlx/conf/TestnetConfig.java b/src/main/java/net/helix/hlx/conf/TestnetConfig.java index c77f5d3f..f25cec88 100644 --- a/src/main/java/net/helix/hlx/conf/TestnetConfig.java +++ b/src/main/java/net/helix/hlx/conf/TestnetConfig.java @@ -37,7 +37,7 @@ public String getCoordinator() { } @JsonProperty - @Parameter(names = "--testnet-coordinator", description = MilestoneConfig.Descriptions.COORDINATOR) + @Parameter(names = "--testnet-coordinator", description = MilestoneConfig.Descriptions.VALIDATOR_ADDRESSES) protected void setCoordinator(String coordinator) { this.coordinator = coordinator; } diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index 4eb804cc..e90cd249 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -3,6 +3,8 @@ import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; +import java.util.Set; + /** * The manager that keeps track of the latest milestone by incorporating a background worker that periodically checks if * new milestones have arrived.
@@ -18,7 +20,7 @@ public interface LatestMilestoneTracker { * * @return the index of the latest milestone that was seen by this tracker */ - int getLatestMilestoneIndex(); + int getLatestRoundIndex(); /** * Returns the transaction hash of the latest milestone that was seen by this tracker.
@@ -27,7 +29,7 @@ public interface LatestMilestoneTracker { * * @return the transaction hash of the latest milestone that was seen by this tracker */ - Hash getLatestMilestoneHash(); + Set getLatestRoundHashes(); /** * Sets the latest milestone.
@@ -37,10 +39,9 @@ public interface LatestMilestoneTracker { * milestone but can also be used by tests to mock a certain behaviour or in case we detect a new milestone in other * parts of the code.
* - * @param latestMilestoneHash the transaction hash of the milestone - * @param latestMilestoneIndex the milestone index of the milestone + * @param latestRoundIndex the milestone index of the milestone */ - void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex); + void setLatestMilestone(int latestRoundIndex); /** * Analyzes the given transaction to determine if it is a valid milestone.
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 28f18466..c2a91145 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -70,9 +70,9 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { private MilestoneSolidifier milestoneSolidifier; /** - * Holds the coordinator address which is used to filter possible milestone candidates.
+ * Holds the addresses which are used to filter possible milestone candidates.
*/ - private Hash coordinatorAddress; + private Set validatorAddresses; /** * Holds a reference to the manager of the background worker.
@@ -81,14 +81,14 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { "Latest Milestone Tracker", log.delegate()); /** - * Holds the milestone index of the latest milestone that we have seen / processed.
+ * Holds the round index of the latest round that we have seen / processed.
*/ - private int latestMilestoneIndex; + private int latestRoundIndex; /** - * Holds the transaction hash of the latest milestone that we have seen / processed.
+ * Holds the transaction hashes of the latest round that we have seen / processed.
*/ - private Hash latestMilestoneHash; + private Set latestRoundHashes; /** * A set that allows us to keep track of the candidates that have been seen and added to the {@link @@ -140,7 +140,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; - coordinatorAddress = HashFactory.ADDRESS.create(config.getCoordinator()); + validatorAddresses = config.getValidatorAddresses(); bootstrapLatestMilestoneValue(); @@ -154,23 +154,23 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP * message processor so external receivers get informed about this change.
*/ @Override - public void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex) { - tangle.publish("lmi %d %d", this.latestMilestoneIndex, latestMilestoneIndex); - log.delegate().info("Latest milestone has changed from #" + this.latestMilestoneIndex + " to #" + - latestMilestoneIndex); + public void setLatestMilestone(int latestRoundIndex) { + tangle.publish("lmi %d %d", this.latestRoundIndex, latestRoundIndex); + log.delegate().info("Latest milestone has changed from #" + this.latestRoundIndex + " to #" + + latestRoundIndex); - this.latestMilestoneHash = latestMilestoneHash; - this.latestMilestoneIndex = latestMilestoneIndex; + //this.latestRoundHashes.add(latestMilestoneHash); + this.latestRoundIndex = latestRoundIndex; } @Override - public int getLatestMilestoneIndex() { - return latestMilestoneIndex; + public int getLatestRoundIndex() { + return latestRoundIndex; } @Override - public Hash getLatestMilestoneHash() { - return latestMilestoneHash; + public Set getLatestRoundHashes() { + return latestRoundHashes; } @Override @@ -191,31 +191,30 @@ public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneE @Override public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { try { - if (coordinatorAddress.equals(transaction.getAddressHash()) && - transaction.getCurrentIndex() == 0) { + if (validatorAddresses.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { - int milestoneIndex = milestoneService.getMilestoneIndex(transaction); + int roundIndex = milestoneService.getMilestoneIndex(transaction); // if the milestone is older than our ledger start point: we already processed it in the past - if (milestoneIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { + if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { return true; } - switch (milestoneService.validateMilestone(transaction, milestoneIndex, SpongeFactory.Mode.S256, 1)) { + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { case VALID: - if (milestoneIndex > latestMilestoneIndex) { - setLatestMilestone(transaction.getHash(), milestoneIndex); + if (roundIndex > latestRoundIndex) { + setLatestMilestone(transaction.getHash(), roundIndex); } if (!transaction.isSolid()) { - milestoneSolidifier.add(transaction.getHash(), milestoneIndex); + milestoneSolidifier.add(transaction.getHash(), roundIndex); } transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); break; case INCOMPLETE: - milestoneSolidifier.add(transaction.getHash(), milestoneIndex); + milestoneSolidifier.add(transaction.getHash(), roundIndex); transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); @@ -308,13 +307,15 @@ private void logProgress() { */ private void collectNewMilestoneCandidates() throws MilestoneException { try { - for (Hash hash : AddressViewModel.load(tangle, coordinatorAddress).getHashes()) { - if (Thread.currentThread().isInterrupted()) { - return; - } - - if (seenMilestoneCandidates.add(hash)) { - milestoneCandidatesToAnalyze.addFirst(hash); + for (Hash address : validatorAddresses) { + for (Hash hash : AddressViewModel.load(tangle, address).getHashes()) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + if (seenMilestoneCandidates.add(hash)) { + milestoneCandidatesToAnalyze.addFirst(hash); + } } } } catch (Exception e) { @@ -371,12 +372,12 @@ private void checkIfInitializationComplete() { */ private void bootstrapLatestMilestoneValue() { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - setLatestMilestone(latestSnapshot.getHash(), latestSnapshot.getIndex()); + setLatestMilestone(latestSnapshot.getIndex()); try { RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); - if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getLatestMilestoneIndex()) { - setLatestMilestone(lastMilestoneInDatabase.getHash(), lastMilestoneInDatabase.index()); + if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getLatestRoundIndex()) { + setLatestMilestone(lastMilestoneInDatabase.index()); } } catch (Exception e) { log.error("unexpectedly failed to retrieve the latest milestone from the database", e); From 70a622f5cbe93797a08ebd8874d20b47024ff172 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 21 May 2019 19:26:39 +0200 Subject: [PATCH 006/223] LatestMilestoneTracker update --- .../milestone/LatestMilestoneTracker.java | 11 +++++-- .../impl/LatestMilestoneTrackerImpl.java | 29 +++++++++++-------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index e90cd249..9158a7ac 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -13,6 +13,11 @@ * if our node is "in sync".
*/ public interface LatestMilestoneTracker { + + void addMilestoneToLatestRound(Hash milestoneHash, int latestRoundIndex); + + void clearLatestRoundHashes(); + /** * Returns the index of the latest milestone that was seen by this tracker.
*
@@ -41,7 +46,7 @@ public interface LatestMilestoneTracker { * * @param latestRoundIndex the milestone index of the milestone */ - void setLatestMilestone(int latestRoundIndex); + void setLatestRoundIndex(int latestRoundIndex); /** * Analyzes the given transaction to determine if it is a valid milestone.
@@ -66,10 +71,10 @@ public interface LatestMilestoneTracker { boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException; /** - * Since the {@link LatestMilestoneTracker} scans all milestone candidates whenever IRI restarts, this flag gives us + * Since the {@link LatestMilestoneTracker} scans all milestone candidates whenever the node restarts, this flag gives us * the ability to determine if this initialization process has finished.
*
- * The values returned by {@link #getLatestMilestoneHash()} and {@link #getLatestMilestoneIndex()} will potentially + * The values returned by {@link #getLatestRoundHashes()} ()} and {@link #getLatestRoundIndex()} ()} will potentially * return wrong values until the scan has completed.
* * @return {@code true} if the initial scan of milestones has finished and {@code false} otherwise diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index c2a91145..f9c50d42 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -154,13 +154,17 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP * message processor so external receivers get informed about this change.
*/ @Override - public void setLatestMilestone(int latestRoundIndex) { - tangle.publish("lmi %d %d", this.latestRoundIndex, latestRoundIndex); - log.delegate().info("Latest milestone has changed from #" + this.latestRoundIndex + " to #" + - latestRoundIndex); + public void addMilestoneToLatestRound(Hash milestoneHash, int latestRoundIndex) { + tangle.publish("lmi %d %d", milestoneHash.hexString(), latestRoundIndex); + log.delegate().info("New milestone added to round #{}", latestRoundIndex); - //this.latestRoundHashes.add(latestMilestoneHash); - this.latestRoundIndex = latestRoundIndex; + this.latestRoundHashes.add(milestoneHash); + } + @Override + public void setLatestRoundIndex(int roundIndex) { + tangle.publish("lmi %d %d", this.latestRoundIndex, roundIndex); + log.delegate().info("Latest round has changed from #{} to #{}", this.latestRoundIndex, roundIndex); + this.latestRoundIndex = roundIndex; } @Override @@ -169,9 +173,10 @@ public int getLatestRoundIndex() { } @Override - public Set getLatestRoundHashes() { - return latestRoundHashes; - } + public Set getLatestRoundHashes() { return this.latestRoundHashes; } + + @Override + public void clearLatestRoundHashes() {this.latestRoundHashes.clear();} @Override public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException { @@ -203,7 +208,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { case VALID: if (roundIndex > latestRoundIndex) { - setLatestMilestone(transaction.getHash(), roundIndex); + addMilestoneToLatestRound(transaction.getHash(), roundIndex); } if (!transaction.isSolid()) { @@ -372,12 +377,12 @@ private void checkIfInitializationComplete() { */ private void bootstrapLatestMilestoneValue() { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - setLatestMilestone(latestSnapshot.getIndex()); + setLatestRoundIndex(latestSnapshot.getIndex()); try { RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getLatestRoundIndex()) { - setLatestMilestone(lastMilestoneInDatabase.index()); + setLatestRoundIndex(lastMilestoneInDatabase.index()); } } catch (Exception e) { log.error("unexpectedly failed to retrieve the latest milestone from the database", e); From bb11e0666310d193dbf4f4c8efbcfa18e8640ea2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 22 May 2019 15:07:22 +0200 Subject: [PATCH 007/223] MilestoneService update --- .../service/milestone/MilestoneService.java | 5 +- .../milestone/impl/MilestoneServiceImpl.java | 86 ++++++++++++++----- 2 files changed, 70 insertions(+), 21 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index 1d444952..a54973de 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -6,6 +6,7 @@ import net.helix.hlx.model.Hash; import java.util.Optional; +import java.util.Set; /** * Represents the service that contains all the relevant business logic for interacting with milestones.
@@ -108,6 +109,8 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i */ boolean isTransactionConfirmed(TransactionViewModel transaction); + Set getConfirmedTips(int roundNumber, int quorum) throws Exception; + /** * Retrieves the milestone index of the given transaction by decoding the {@code OBSOLETE_TAG}.
*
@@ -117,5 +120,5 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i * @param milestoneTransaction the transaction that shall have its milestone index retrieved * @return the milestone index of the transaction */ - int getMilestoneIndex(TransactionViewModel milestoneTransaction); + int getRoundIndex(TransactionViewModel milestoneTransaction); } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 0bdee47e..66c07bb7 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -29,6 +29,7 @@ import java.nio.ByteBuffer; import java.util.*; +import java.util.stream.Collectors; import static net.helix.hlx.service.milestone.MilestoneValidity.INCOMPLETE; import static net.helix.hlx.service.milestone.MilestoneValidity.INVALID; @@ -116,7 +117,7 @@ public Optional findLatestProcessedSolidMilestoneInDatabase() th } // trivial case #2: the node was fully synced but the last milestone was not processed, yet - RoundViewModel latestMilestonePredecessor = RoundViewModel.findClosestPrevMilestone(tangle, + RoundViewModel latestMilestonePredecessor = RoundViewModel.findClosestPrevRound(tangle, latestMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex()); if (latestMilestonePredecessor != null && wasMilestoneAppliedToLedger(latestMilestonePredecessor)) { return Optional.of(latestMilestonePredecessor); @@ -153,15 +154,15 @@ public void resetCorruptedMilestone(int index) throws MilestoneException { } @Override - public MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, int milestoneIndex, + public MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, int roundIndex, SpongeFactory.Mode mode, int securityLevel) throws MilestoneException { - if (milestoneIndex < 0 || milestoneIndex >= 0x200000) { + if (roundIndex < 0 || roundIndex >= 0x200000) { return INVALID; } try { - if (RoundViewModel.get(tangle, milestoneIndex) != null) { + if (RoundViewModel.get(tangle, roundIndex) != null) { // Already validated. return VALID; } @@ -199,26 +200,26 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM //validate Merkle path byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, - siblingsTx.getSignature(), 0, milestoneIndex, config.getNumberOfKeysInMilestone()); + siblingsTx.getSignature(), 0, roundIndex, config.getNumberOfKeysInMilestone()); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || - (HashFactory.ADDRESS.create(merkleRoot)).equals( - HashFactory.ADDRESS.create(config.getCoordinator()))) { + (config.getValidatorAddresses().contains(HashFactory.ADDRESS.create(merkleRoot)))) { - RoundViewModel newRoundViewModel = new RoundViewModel(milestoneIndex, - transactionViewModel.getHash()); - newRoundViewModel.store(tangle); + RoundViewModel currentRoundViewModel = RoundViewModel.get(tangle, roundIndex); + currentRoundViewModel.addMilestone(transactionViewModel.getHash()); + currentRoundViewModel.update(tangle); // if we find a NEW milestone that should have been processed before our latest solid // milestone -> reset the ledger state and check the milestones again // // NOTE: this can happen if a new subtangle becomes solid before a previous one while // syncing - if (milestoneIndex < snapshotProvider.getLatestSnapshot().getIndex() && - milestoneIndex > snapshotProvider.getInitialSnapshot().getIndex()) { + //TODO: find a solution for this + /*if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() && + roundIndex > snapshotProvider.getInitialSnapshot().getIndex()) { - resetCorruptedMilestone(milestoneIndex); - } + resetCorruptedMilestone(roundIndex); + }*/ return VALID; } else { @@ -236,12 +237,53 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM } @Override - public int getMilestoneIndex(TransactionViewModel milestoneTransaction) { + public int getRoundIndex(TransactionViewModel milestoneTransaction) { return (int) Serializer.getLong(milestoneTransaction.getBytes(), TransactionViewModel.TAG_OFFSET); //TODO: why is index stored in bundle nonce (obsolete tag) in new version? //return (int) Serializer.getLong(milestoneTransaction.getBytes(), BUNDLE_NONCE_OFFSET); } + @Override + public Set getConfirmedTips(int roundNumber, int quorum) throws Exception { + + RoundViewModel round = RoundViewModel.get(tangle, roundNumber); + Map occurences = new HashMap<>(); + + for (Hash milestoneHash : round.getHashes()) { + final List> bundleTransactions = BundleValidator.validate(tangle, + snapshotProvider.getInitialSnapshot(), milestoneHash); + + for (final List bundleTransactionViewModels : bundleTransactions) { + final TransactionViewModel tipsTx = bundleTransactionViewModels.get(bundleTransactionViewModels.size() - 1); + + for (int i = 0; i < 16; i++) { + Hash tip = HashFactory.TRANSACTION.create(tipsTx.getSignature(), i * Hash.SIZE_IN_BYTES, (i + 1) * Hash.SIZE_IN_BYTES); + if (occurences.containsKey(tip)) { + occurences.put(tip, occurences.get(tip) + 1); + } else { + occurences.put(tip, 1); + } + } + } + } + return occurences.entrySet().stream() + .filter(entry -> entry.getValue() >= quorum) + .map(entry -> entry.getKey()) + .collect(Collectors.toSet()); + } + + /*@Override + public Set getConfirmedTransactions(int roundNumber, int quorum) throws Exception{ + Set confirmedTips = getConfirmedTips(roundNumber, quorum); + ... + }*/ + + /*@Override + public boolean isTransactionConfirmed(Hash transactionHash, int roundNumber, int quorum) { + Set confimedTx = getConfirmedTransactions(roundNumber); + ... + }*/ + @Override public boolean isTransactionConfirmed(TransactionViewModel transaction, int milestoneIndex) { return transaction.snapshotIndex() != 0 && transaction.snapshotIndex() <= milestoneIndex; @@ -318,9 +360,9 @@ private RoundViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) int range = rangeEnd - rangeStart; int middleOfRange = rangeEnd - range / 2; - RoundViewModel milestone = RoundViewModel.findClosestNextMilestone(tangle, middleOfRange - 1, rangeEnd); + RoundViewModel milestone = RoundViewModel.findClosestNextRound(tangle, middleOfRange - 1, rangeEnd); if (milestone == null) { - milestone = RoundViewModel.findClosestPrevMilestone(tangle, middleOfRange, rangeStart); + milestone = RoundViewModel.findClosestPrevRound(tangle, middleOfRange, rangeStart); } return milestone; @@ -338,9 +380,11 @@ private RoundViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) * @throws Exception if anything unexpected happens while checking the milestone */ private boolean wasMilestoneAppliedToLedger(RoundViewModel milestone) throws Exception { - TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash()); + //TODO: find a solution for this + /*TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash()); return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT && - milestoneTransaction.snapshotIndex() != 0; + milestoneTransaction.snapshotIndex() != 0;*/ + return true; } /** @@ -525,9 +569,11 @@ private void resetCorruptedMilestone(int index, Set processedTransactions) if(milestoneToRepair.index() <= snapshotProvider.getLatestSnapshot().getIndex()) { snapshotService.rollBackMilestones(snapshotProvider.getLatestSnapshot(), milestoneToRepair.index()); } + //TODO: find a solution for this + /* updateMilestoneIndexOfMilestoneTransactions(milestoneToRepair.getHash(), milestoneToRepair.index(), 0, processedTransactions); - tangle.delete(StateDiff.class, milestoneToRepair.getHash()); + tangle.delete(StateDiff.class, milestoneToRepair.getHash());*/ } } catch (Exception e) { throw new MilestoneException("failed to repair corrupted milestone with index #" + index, e); From e360e4dfb421765cc202140d04da95e3e95873ba Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 22 May 2019 15:15:41 +0200 Subject: [PATCH 008/223] LatestSolidMilestoneTracker update --- .../impl/LatestSolidMilestoneTrackerImpl.java | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index e1944ad3..a0dd0bf2 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -136,31 +136,27 @@ public void shutdown() { */ @Override public void trackLatestSolidMilestone() throws MilestoneException { - try { - int currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); - if (currentSolidMilestoneIndex < latestMilestoneTracker.getLatestMilestoneIndex()) { - RoundViewModel nextMilestone; - while (!Thread.currentThread().isInterrupted() && - (nextMilestone = RoundViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null && - TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { - - //TODO: graphstream - if (ledgerService.getGraph() != null) { - ledgerService.getGraph().setMilestone(nextMilestone.getHash().hexString()); - } - - syncLatestMilestoneTracker(nextMilestone.getHash(), nextMilestone.index()); - applySolidMilestoneToLedger(nextMilestone); - logChange(currentSolidMilestoneIndex); - - currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); + boolean allSolid = true; + for (Hash milestoneHash : latestMilestoneTracker.getLatestRoundHashes()) { + try { + if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { + allSolid = false; } - } else { - syncLatestMilestoneTracker(snapshotProvider.getLatestSnapshot().getHash(), - currentSolidMilestoneIndex); + } catch (Exception e) { + throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); + } + } + if (allSolid) { + try { + RoundViewModel nextRound = RoundViewModel.get(tangle, latestMilestoneTracker.getLatestRoundIndex() + 1); + syncLatestMilestoneTracker(nextRound.index()); + applySolidMilestoneToLedger(nextRound); + //logChange(currentSolidMilestoneIndex); + //currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); + } + catch (Exception e) { + throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); } - } catch (Exception e) { - throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); } } @@ -192,16 +188,16 @@ private void latestSolidMilestoneTrackerThread() { * our current milestone and consequently try to reapply them in the next iteration of the {@link * #trackLatestSolidMilestone()} method (until the problem is solved).
* - * @param milestone the milestone that shall be applied to the ledger state + * @param round the milestone that shall be applied to the ledger state * @throws Exception if anything unexpected goes wrong while applying the milestone to the ledger */ - private void applySolidMilestoneToLedger(RoundViewModel milestone) throws Exception { - if (ledgerService.applyMilestoneToLedger(milestone)) { - if (isRepairRunning() && isRepairSuccessful(milestone)) { + private void applySolidMilestoneToLedger(RoundViewModel round) throws Exception { + if (ledgerService.applyMilestoneToLedger(round)) { + if (isRepairRunning() && isRepairSuccessful(round)) { stopRepair(); } } else { - repairCorruptedMilestone(milestone); + repairCorruptedMilestone(round); } } @@ -249,12 +245,11 @@ private void stopRepair() { * Note: This method ensures that the latest milestone index is always bigger or equals the latest solid milestone * index. * - * @param milestoneHash transaction hash of the milestone - * @param milestoneIndex milestone index + * @param roundIndex milestone index */ - private void syncLatestMilestoneTracker(Hash milestoneHash, int milestoneIndex) { - if(milestoneIndex > latestMilestoneTracker.getLatestMilestoneIndex()) { - latestMilestoneTracker.setLatestMilestone(milestoneHash, milestoneIndex); + private void syncLatestMilestoneTracker(int roundIndex) { + if(roundIndex > latestMilestoneTracker.getLatestRoundIndex()) { + latestMilestoneTracker.setLatestRoundIndex(roundIndex); } } From 67dc36db40ecca7f37f5599ac3e1d64def437897 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 23 May 2019 15:16:32 +0200 Subject: [PATCH 009/223] MilestoneService 2nd update - renaming and fixing unsolved parts --- .../service/milestone/MilestoneService.java | 8 +- .../milestone/impl/MilestoneServiceImpl.java | 111 +++++++++--------- 2 files changed, 62 insertions(+), 57 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index a54973de..7d8cc927 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -25,7 +25,7 @@ public interface MilestoneService { * processed solid milestone can be found * @throws MilestoneException if anything unexpected happend while performing the search */ - Optional findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException; + Optional findLatestProcessedSolidRoundInDatabase() throws MilestoneException; /** * Analyzes the given transaction to determine if it is a valid milestone.
@@ -34,7 +34,7 @@ public interface MilestoneService { * the signature to analyze if the given milestone was really issued by the coordinator.
* * @param transactionViewModel transaction that shall be analyzed - * @param milestoneIndex milestone index of the transaction (see {@link #getMilestoneIndex(TransactionViewModel)}) + * @param milestoneIndex milestone index of the transaction (see {@link #getRoundIndex(TransactionViewModel)}) * @param mode mode that gets used for the signature verification * @param securityLevel security level that gets used for the signature verification * @return validity status of the transaction regarding its role as a milestone @@ -66,7 +66,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i * @param newIndex the milestone index that shall be set * @throws MilestoneException if anything unexpected happens while updating the milestone index */ - void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int newIndex) throws MilestoneException; + void updateRoundIndexOfMilestoneTransactions(Hash milestoneHash, int newIndex) throws MilestoneException; /** * Resets all milestone related information of the transactions that were "confirmed" by the given milestone and @@ -82,7 +82,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i * @param index milestone index that shall be reverted * @throws MilestoneException if anything goes wrong while resetting the corrupted milestone */ - void resetCorruptedMilestone(int index) throws MilestoneException; + void resetCorruptedRound(int index) throws MilestoneException; /** * Checks if the given transaction was confirmed by the milestone with the given index (or any of its diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 66c07bb7..0cf556e5 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -10,6 +10,7 @@ import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; +import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.StateDiff; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; @@ -103,28 +104,28 @@ public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvide * amount of database requests to a minimum (even with a huge amount of milestones in the database).
*/ @Override - public Optional findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException { + public Optional findLatestProcessedSolidRoundInDatabase() throws MilestoneException { try { // if we have no milestone in our database -> abort - RoundViewModel latestMilestone = RoundViewModel.latest(tangle); - if (latestMilestone == null) { + RoundViewModel latestRound = RoundViewModel.latest(tangle); + if (latestRound == null) { return Optional.empty(); } // trivial case #1: the node was fully synced - if (wasMilestoneAppliedToLedger(latestMilestone)) { - return Optional.of(latestMilestone); + if (wasRoundAppliedToLedger(latestRound)) { + return Optional.of(latestRound); } // trivial case #2: the node was fully synced but the last milestone was not processed, yet - RoundViewModel latestMilestonePredecessor = RoundViewModel.findClosestPrevRound(tangle, - latestMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex()); - if (latestMilestonePredecessor != null && wasMilestoneAppliedToLedger(latestMilestonePredecessor)) { - return Optional.of(latestMilestonePredecessor); + RoundViewModel latestRoundPredecessor = RoundViewModel.findClosestPrevRound(tangle, + latestRound.index(), snapshotProvider.getInitialSnapshot().getIndex()); + if (latestRoundPredecessor != null && wasRoundAppliedToLedger(latestRoundPredecessor)) { + return Optional.of(latestRoundPredecessor); } // non-trivial case: do a binary search in the database - return binarySearchLatestProcessedSolidMilestoneInDatabase(latestMilestone); + return binarySearchLatestProcessedSolidRoundInDatabase(latestRound); } catch (Exception e) { throw new MilestoneException( "unexpected error while trying to find the latest processed solid milestone in the database", e); @@ -132,25 +133,25 @@ public Optional findLatestProcessedSolidMilestoneInDatabase() th } @Override - public void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int index) throws MilestoneException { + public void updateRoundIndexOfMilestoneTransactions(Hash milestoneHash, int index) throws MilestoneException { if (index <= 0) { throw new MilestoneException("the new index needs to be bigger than 0 " + "(use resetCorruptedMilestone to reset the milestone index)"); } - updateMilestoneIndexOfMilestoneTransactions(milestoneHash, index, index, new HashSet<>()); + updateRoundIndexOfMilestoneTransactions(milestoneHash, index, index, new HashSet<>()); } /** * {@inheritDoc} *
- * We redirect the call to {@link #resetCorruptedMilestone(int, Set)} while initiating the set of {@code + * We redirect the call to {@link #resetCorruptedRound(int, Set)} while initiating the set of {@code * processedTransactions} with an empty {@link HashSet} which will ensure that we reset all found * transactions.
*/ @Override - public void resetCorruptedMilestone(int index) throws MilestoneException { - resetCorruptedMilestone(index, new HashSet<>()); + public void resetCorruptedRound(int index) throws MilestoneException { + resetCorruptedRound(index, new HashSet<>()); } @Override @@ -174,7 +175,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM return INCOMPLETE; } else { for (final List bundleTransactionViewModels : bundleTransactions) { - final TransactionViewModel tail = bundleTransactionViewModels.get(0); + final TransactionViewModel tail = bundleTransactionViewModels.get(0); // milestone transaction with signature if (tail.getHash().equals(transactionViewModel.getHash())) { //the signed transaction - which references the confirmed transactions and contains // the Merkle tree siblings. @@ -183,6 +184,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) + //TODO: check if its okay here to use bundle hash instead of tx hash byte[] bundleHash = new byte[Sha3.HASH_LENGTH]; Winternitz.normalizedBundle(siblingsTx.getBundleHash().bytes(), bundleHash); @@ -214,12 +216,11 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM // // NOTE: this can happen if a new subtangle becomes solid before a previous one while // syncing - //TODO: find a solution for this - /*if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() && + if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() && roundIndex > snapshotProvider.getInitialSnapshot().getIndex()) { - resetCorruptedMilestone(roundIndex); - }*/ + resetCorruptedRound(roundIndex); + } return VALID; } else { @@ -247,26 +248,30 @@ public int getRoundIndex(TransactionViewModel milestoneTransaction) { public Set getConfirmedTips(int roundNumber, int quorum) throws Exception { RoundViewModel round = RoundViewModel.get(tangle, roundNumber); - Map occurences = new HashMap<>(); + Map occurrences = new HashMap<>(); for (Hash milestoneHash : round.getHashes()) { final List> bundleTransactions = BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), milestoneHash); for (final List bundleTransactionViewModels : bundleTransactions) { + final TransactionViewModel tipsTx = bundleTransactionViewModels.get(bundleTransactionViewModels.size() - 1); for (int i = 0; i < 16; i++) { Hash tip = HashFactory.TRANSACTION.create(tipsTx.getSignature(), i * Hash.SIZE_IN_BYTES, (i + 1) * Hash.SIZE_IN_BYTES); - if (occurences.containsKey(tip)) { - occurences.put(tip, occurences.get(tip) + 1); + if (tip.equals(Hash.NULL_HASH)) { + break; + } + if (occurrences.containsKey(tip)) { + occurrences.put(tip, occurrences.get(tip) + 1); } else { - occurences.put(tip, 1); + occurrences.put(tip, 1); } } } } - return occurences.entrySet().stream() + return occurrences.entrySet().stream() .filter(entry -> entry.getValue() >= quorum) .map(entry -> entry.getKey()) .collect(Collectors.toSet()); @@ -285,8 +290,8 @@ public boolean isTransactionConfirmed(Hash transactionHash, int roundNumber, int }*/ @Override - public boolean isTransactionConfirmed(TransactionViewModel transaction, int milestoneIndex) { - return transaction.snapshotIndex() != 0 && transaction.snapshotIndex() <= milestoneIndex; + public boolean isTransactionConfirmed(TransactionViewModel transaction, int roundIndex) { + return transaction.snapshotIndex() != 0 && transaction.snapshotIndex() <= roundIndex; } @Override @@ -310,7 +315,7 @@ public boolean isTransactionConfirmed(TransactionViewModel transaction) { * processed solid milestone can be found * @throws Exception if anything unexpected happens while performing the search */ - private Optional binarySearchLatestProcessedSolidMilestoneInDatabase( + private Optional binarySearchLatestProcessedSolidRoundInDatabase( RoundViewModel latestMilestone) throws Exception { Optional lastAppliedCandidate = Optional.empty(); @@ -319,13 +324,13 @@ private Optional binarySearchLatestProcessedSolidMilestoneInData int rangeStart = snapshotProvider.getInitialSnapshot().getIndex() + 1; while (rangeEnd - rangeStart >= 0) { // if no candidate found in range -> return last candidate - RoundViewModel currentCandidate = getMilestoneInMiddleOfRange(rangeStart, rangeEnd); + RoundViewModel currentCandidate = getRoundInMiddleOfRange(rangeStart, rangeEnd); if (currentCandidate == null) { return lastAppliedCandidate; } // if the milestone was applied -> continue to search for "later" ones that might have also been applied - if (wasMilestoneAppliedToLedger(currentCandidate)) { + if (wasRoundAppliedToLedger(currentCandidate)) { rangeStart = currentCandidate.index() + 1; lastAppliedCandidate = Optional.of(currentCandidate); @@ -343,7 +348,7 @@ private Optional binarySearchLatestProcessedSolidMilestoneInData /** * Determines the milestone in the middle of the range defined by {@code rangeStart} and {@code rangeEnd}.
*
- * It is used by the binary search algorithm of {@link #findLatestProcessedSolidMilestoneInDatabase()}. It first + * It is used by the binary search algorithm of {@link #findLatestProcessedSolidRoundInDatabase()}. It first * calculates the index that represents the middle of the range and then tries to find the milestone that is closest * to this index.
*
@@ -356,7 +361,7 @@ private Optional binarySearchLatestProcessedSolidMilestoneInData * found * @throws Exception if anything unexpected happens while trying to get the milestone */ - private RoundViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) throws Exception { + private RoundViewModel getRoundInMiddleOfRange(int rangeStart, int rangeEnd) throws Exception { int range = rangeEnd - rangeStart; int middleOfRange = rangeEnd - range / 2; @@ -375,11 +380,11 @@ private RoundViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) * ledger, we can use it to determine if it was processed by IRI in the past. If this value is set we should also * have a corresponding {@link StateDiff} entry in the database.
* - * @param milestone the milestone that shall be checked + * @param round the milestone that shall be checked * @return {@code true} if the milestone has been processed by IRI before and {@code false} otherwise * @throws Exception if anything unexpected happens while checking the milestone */ - private boolean wasMilestoneAppliedToLedger(RoundViewModel milestone) throws Exception { + private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { //TODO: find a solution for this /*TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash()); return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT && @@ -388,7 +393,7 @@ private boolean wasMilestoneAppliedToLedger(RoundViewModel milestone) throws Exc } /** - * This method implements the logic described by {@link #updateMilestoneIndexOfMilestoneTransactions(Hash, int)} but + * This method implements the logic described by {@link #updateRoundIndexOfMilestoneTransactions(Hash, int)} but * accepts some additional parameters that allow it to be reused by different parts of this service.
* * @param milestoneHash the hash of the transaction @@ -398,7 +403,7 @@ private boolean wasMilestoneAppliedToLedger(RoundViewModel milestone) throws Exc * @throws MilestoneException if anything unexpected happens while updating the milestone index * @param processedTransactions a set of transactions that have been processed already (for the recursive calls) */ - private void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int correctIndex, int newIndex, + private void updateRoundIndexOfMilestoneTransactions(Hash milestoneHash, int correctIndex, int newIndex, Set processedTransactions) throws MilestoneException { processedTransactions.add(milestoneHash); @@ -407,7 +412,7 @@ private void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int Set transactionsToUpdate = new HashSet<>(); try { - prepareMilestoneIndexUpdate(TransactionViewModel.fromHash(tangle, milestoneHash), correctIndex, newIndex, + prepareRoundIndexUpdate(TransactionViewModel.fromHash(tangle, milestoneHash), correctIndex, newIndex, inconsistentMilestones, transactionsToUpdate); DAGHelper.get(tangle).traverseApprovees( @@ -423,7 +428,7 @@ private void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int return true; }, - currentTransaction -> prepareMilestoneIndexUpdate(currentTransaction, correctIndex, newIndex, + currentTransaction -> prepareRoundIndexUpdate(currentTransaction, correctIndex, newIndex, inconsistentMilestones, transactionsToUpdate), processedTransactions ); @@ -432,11 +437,11 @@ private void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int } for(int inconsistentMilestoneIndex : inconsistentMilestones) { - resetCorruptedMilestone(inconsistentMilestoneIndex, processedTransactions); + resetCorruptedRound(inconsistentMilestoneIndex, processedTransactions); } for (TransactionViewModel transactionToUpdate : transactionsToUpdate) { - updateMilestoneIndexOfSingleTransaction(transactionToUpdate, newIndex); + updateRoundIndexOfSingleTransaction(transactionToUpdate, newIndex); } } @@ -450,7 +455,7 @@ private void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int * @param index the milestone index that is set for the given transaction * @throws MilestoneException if anything unexpected happens while updating the transaction */ - private void updateMilestoneIndexOfSingleTransaction(TransactionViewModel transaction, int index) throws + private void updateRoundIndexOfSingleTransaction(TransactionViewModel transaction, int index) throws MilestoneException { try { @@ -489,7 +494,7 @@ private void updateMilestoneIndexOfSingleTransaction(TransactionViewModel transa * @param transactionsToUpdate a set that is used to collect all transactions that need to be updated [output * parameter] */ - private void prepareMilestoneIndexUpdate(TransactionViewModel transaction, int correctMilestoneIndex, + private void prepareRoundIndexUpdate(TransactionViewModel transaction, int correctMilestoneIndex, int newMilestoneIndex, Set corruptMilestones, Set transactionsToUpdate) { if(transaction.snapshotIndex() > correctMilestoneIndex) { @@ -543,7 +548,7 @@ private boolean isMilestoneBundleStructureValid(List bundl } /** - * This method does the same as {@link #resetCorruptedMilestone(int)} but additionally receives a set of {@code + * This method does the same as {@link #resetCorruptedRound(int)} but additionally receives a set of {@code * processedTransactions} that will allow us to not process the same transactions over and over again while * resetting additional milestones in recursive calls.
*
@@ -558,22 +563,22 @@ private boolean isMilestoneBundleStructureValid(List bundl * @param processedTransactions a set of transactions that have been processed already * @throws MilestoneException if anything goes wrong while resetting the corrupted milestone */ - private void resetCorruptedMilestone(int index, Set processedTransactions) throws MilestoneException { + private void resetCorruptedRound(int index, Set processedTransactions) throws MilestoneException { if(index <= snapshotProvider.getInitialSnapshot().getIndex()) { return; } log.info("resetting corrupted milestone #" + index); try { - RoundViewModel milestoneToRepair = RoundViewModel.get(tangle, index); - if(milestoneToRepair != null) { - if(milestoneToRepair.index() <= snapshotProvider.getLatestSnapshot().getIndex()) { - snapshotService.rollBackMilestones(snapshotProvider.getLatestSnapshot(), milestoneToRepair.index()); + RoundViewModel roundToRepair = RoundViewModel.get(tangle, index); + if(roundToRepair != null) { + if(roundToRepair.index() <= snapshotProvider.getLatestSnapshot().getIndex()) { + snapshotService.rollBackMilestones(snapshotProvider.getLatestSnapshot(), roundToRepair.index()); + } + for (Hash milestone : roundToRepair.getHashes()) { + updateRoundIndexOfMilestoneTransactions(milestone, roundToRepair.index(), 0, + processedTransactions); } - //TODO: find a solution for this - /* - updateMilestoneIndexOfMilestoneTransactions(milestoneToRepair.getHash(), milestoneToRepair.index(), 0, - processedTransactions); - tangle.delete(StateDiff.class, milestoneToRepair.getHash());*/ + tangle.delete(StateDiff.class, new IntegerIndex(roundToRepair.index())); } } catch (Exception e) { throw new MilestoneException("failed to repair corrupted milestone with index #" + index, e); From 6811450afb973b2c089cc86d85c9c54d6673b1d4 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 23 May 2019 15:30:31 +0200 Subject: [PATCH 010/223] use new method name --- .../hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index f9c50d42..be3fd298 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -198,7 +198,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw try { if (validatorAddresses.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { - int roundIndex = milestoneService.getMilestoneIndex(transaction); + int roundIndex = milestoneService.getRoundIndex(transaction); // if the milestone is older than our ledger start point: we already processed it in the past if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { From 31a50912a1102acd3ee06432a0c18d58ded3c0de Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 23 May 2019 15:59:45 +0200 Subject: [PATCH 011/223] LatestSolidMilestoneTracker 2nd update - renaming and improve tracking function --- .../impl/LatestSolidMilestoneTrackerImpl.java | 77 ++++++++++--------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index a0dd0bf2..becf2ac1 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -78,10 +78,10 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac /** * Holds the milestone index of the milestone that caused the repair logic to get started.
*/ - private int errorCausingMilestoneIndex = Integer.MAX_VALUE; + private int errorCausingRoundIndex = Integer.MAX_VALUE; /** - * Counter for the backoff repair strategy (see {@link #repairCorruptedMilestone(RoundViewModel)}.
+ * Counter for the backoff repair strategy (see {@link #repairCorruptedRound(RoundViewModel)}.
*/ private int repairBackoffCounter = 0; @@ -136,27 +136,30 @@ public void shutdown() { */ @Override public void trackLatestSolidMilestone() throws MilestoneException { - boolean allSolid = true; - for (Hash milestoneHash : latestMilestoneTracker.getLatestRoundHashes()) { - try { - if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { - allSolid = false; + try { + int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + if (currentSolidRoundIndex < latestMilestoneTracker.getLatestRoundIndex()) { + RoundViewModel nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); + if (nextRound != null) { + // check solidity of milestones + boolean allSolid = true; + for (Hash milestoneHash : nextRound.getHashes()) { + if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { + allSolid = false; + } + } + while (!Thread.currentThread().isInterrupted() && allSolid) { + syncLatestMilestoneTracker(nextRound.index()); + applySolidMilestoneToLedger(nextRound); + logChange(currentSolidRoundIndex); + currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + } } - } catch (Exception e) { - throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); - } - } - if (allSolid) { - try { - RoundViewModel nextRound = RoundViewModel.get(tangle, latestMilestoneTracker.getLatestRoundIndex() + 1); - syncLatestMilestoneTracker(nextRound.index()); - applySolidMilestoneToLedger(nextRound); - //logChange(currentSolidMilestoneIndex); - //currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); - } - catch (Exception e) { - throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); + } else { + syncLatestMilestoneTracker(currentSolidRoundIndex); } + } catch (Exception e) { + throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); } } @@ -197,7 +200,7 @@ private void applySolidMilestoneToLedger(RoundViewModel round) throws Exception stopRepair(); } } else { - repairCorruptedMilestone(round); + repairCorruptedRound(round); } } @@ -218,22 +221,22 @@ private boolean isRepairRunning() { * To determine if the repair routine was successful we check if the processed milestone has a higher index than the * one that initially could not get applied to the ledger.
* - * @param processedMilestone the currently processed milestone + * @param processedRound the currently processed milestone * @return {@code true} if we advanced to a milestone following the corrupted one and {@code false} otherwise */ - private boolean isRepairSuccessful(RoundViewModel processedMilestone) { - return processedMilestone.index() > errorCausingMilestoneIndex; + private boolean isRepairSuccessful(RoundViewModel processedRound) { + return processedRound.index() > errorCausingRoundIndex; } /** * Resets the internal variables that are used to keep track of the repair process.
*
* It gets called whenever we advance to a milestone that has a higher milestone index than the milestone that - * initially caused the repair routine to kick in (see {@link #repairCorruptedMilestone(RoundViewModel)}.
+ * initially caused the repair routine to kick in (see {@link #repairCorruptedRound(RoundViewModel)}.
*/ private void stopRepair() { repairBackoffCounter = 0; - errorCausingMilestoneIndex = Integer.MAX_VALUE; + errorCausingRoundIndex = Integer.MAX_VALUE; } /** @@ -259,17 +262,17 @@ private void syncLatestMilestoneTracker(int roundIndex) { * It simply compares the current latest milestone index against the previous milestone index and emits the log * messages using the {@link #log} and the {@link #messageQ} instances if it differs.
* - * @param prevSolidMilestoneIndex the milestone index before the change + * @param prevSolidRoundIndex the milestone index before the change */ - private void logChange(int prevSolidMilestoneIndex) { + private void logChange(int prevSolidRoundIndex) { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); int latestMilestoneIndex = latestSnapshot.getIndex(); Hash latestMilestoneHash = latestSnapshot.getHash(); - if (prevSolidMilestoneIndex != latestMilestoneIndex) { - log.info("Latest SOLID milestone index changed from #" + prevSolidMilestoneIndex + " to #" + latestMilestoneIndex); + if (prevSolidRoundIndex != latestMilestoneIndex) { + log.info("Latest SOLID milestone index changed from #" + prevSolidRoundIndex + " to #" + latestMilestoneIndex); - tangle.publish("lmsi %d %d", prevSolidMilestoneIndex, latestMilestoneIndex); + tangle.publish("lmsi %d %d", prevSolidRoundIndex, latestMilestoneIndex); tangle.publish("lmhs %s", latestMilestoneHash.hexString()); } } @@ -290,15 +293,15 @@ private void logChange(int prevSolidMilestoneIndex) { * To be able to tell when the problem is fixed and the {@link #repairBackoffCounter} can be reset, we store the * milestone index that caused the problem the first time we call this method.
* - * @param errorCausingMilestone the milestone that failed to be applied + * @param errorCausingRound the milestone that failed to be applied * @throws MilestoneException if we failed to reset the corrupted milestone */ - private void repairCorruptedMilestone(RoundViewModel errorCausingMilestone) throws MilestoneException { + private void repairCorruptedRound(RoundViewModel errorCausingRound) throws MilestoneException { if(repairBackoffCounter++ == 0) { - errorCausingMilestoneIndex = errorCausingMilestone.index(); + errorCausingRoundIndex = errorCausingRound.index(); } - for (int i = errorCausingMilestone.index(); i > errorCausingMilestone.index() - repairBackoffCounter; i--) { - milestoneService.resetCorruptedMilestone(i); + for (int i = errorCausingRound.index(); i > errorCausingRound.index() - repairBackoffCounter; i--) { + milestoneService.resetCorruptedRound(i); } } } From 8f669c2a9ed488cf438c7022c46e7652ef453dbf Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 27 May 2019 17:42:47 +0200 Subject: [PATCH 012/223] update LedgerService and fix updateRoundIndexOfMilestoneTransactions in MilestoneService --- .../hlx/service/ledger/LedgerService.java | 8 +-- .../ledger/impl/LedgerServiceImpl.java | 43 +++++++------ .../service/milestone/MilestoneService.java | 3 +- .../milestone/impl/MilestoneServiceImpl.java | 64 +++++++++---------- 4 files changed, 58 insertions(+), 60 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/ledger/LedgerService.java b/src/main/java/net/helix/hlx/service/ledger/LedgerService.java index 84b37518..74006049 100644 --- a/src/main/java/net/helix/hlx/service/ledger/LedgerService.java +++ b/src/main/java/net/helix/hlx/service/ledger/LedgerService.java @@ -41,11 +41,11 @@ public interface LedgerService { * corresponding {@code snapshotIndex} value. Then it generates the {@link net.helix.hlx.model.StateDiff} that * reflects the accumulated balance changes of all these transactions and applies it to the latest Snapshot.
* - * @param milestone the milestone that shall be applied + * @param round the milestone that shall be applied * @return {@code true} if the milestone could be applied to the ledger and {@code false} otherwise * @throws LedgerException if anything goes wrong while modifying the ledger state */ - boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerException; + boolean applyMilestoneToLedger(RoundViewModel round) throws LedgerException; /** * Checks the consistency of the combined balance changes of the given tips.
@@ -87,13 +87,13 @@ public interface LedgerService { * (by being part of the {@code visitedNonMilestoneSubtangleHashes} set) and collects their balance changes.
* * @param visitedTransactions a set of transaction hashes that shall be considered to be visited already - * @param startTransaction the transaction that marks the start of the dag traversal and that has its approovees + * @param startTransactions the transaction that marks the start of the dag traversal and that has its approovees * examined * @return a map of the balance changes (addresses associated to their balance) or {@code null} if the balance could * not be generated due to inconsistencies * @throws LedgerException if anything unexpected happens while generating the balance changes */ - Map generateBalanceDiff(Set visitedTransactions, Hash startTransaction, int milestoneIndex) + Map generateBalanceDiff(Set visitedTransactions, Set startTransactions, int milestoneIndex) throws LedgerException; boolean updateDiff(Set approvedHashes, final Map diff, Hash tip); // temporary diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index c8b528eb..8d9dd4d8 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -91,7 +91,7 @@ public boolean updateDiff(Set approvedHashes, final Map diff, @Override public void restoreLedgerState() throws LedgerException { try { - Optional milestone = milestoneService.findLatestProcessedSolidMilestoneInDatabase(); + Optional milestone = milestoneService.findLatestProcessedSolidRoundInDatabase(); if (milestone.isPresent()) { snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.get().index()); } @@ -144,7 +144,8 @@ public boolean isBalanceDiffConsistent(Set approvedHashes, Map return true; } Set visitedHashes = new HashSet<>(approvedHashes); - Map currentState = generateBalanceDiff(visitedHashes, tip, + Set startHashes = new HashSet<>(Collections.singleton(tip)); + Map currentState = generateBalanceDiff(visitedHashes, startHashes, snapshotProvider.getLatestSnapshot().getIndex()); if (currentState == null) { return false; @@ -164,7 +165,7 @@ public boolean isBalanceDiffConsistent(Set approvedHashes, Map } @Override - public Map generateBalanceDiff(Set visitedTransactions, Hash startTransaction, int milestoneIndex) + public Map generateBalanceDiff(Set visitedTransactions, Set startTransactions, int milestoneIndex) throws LedgerException { Map state = new HashMap<>(); @@ -175,7 +176,7 @@ public Map generateBalanceDiff(Set visitedTransactions, Hash s countedTx.add(solidEntryPointHash); }); - final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(startTransaction)); + final Queue nonAnalyzedTransactions = new LinkedList<>(startTransactions); Hash transactionPointer; while ((transactionPointer = nonAnalyzedTransactions.poll()) != null) { if (visitedTransactions.add(transactionPointer)) { @@ -231,9 +232,12 @@ public Map generateBalanceDiff(Set visitedTransactions, Hash s return null; } } - - nonAnalyzedTransactions.offer(transactionViewModel.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transactionViewModel.getBranchTransactionHash()); + if (!visitedTransactions.contains(transactionViewModel.getTrunkTransactionHash())) { + nonAnalyzedTransactions.offer(transactionViewModel.getTrunkTransactionHash()); + } + if (!visitedTransactions.contains(transactionViewModel.getBranchTransactionHash())) { + nonAnalyzedTransactions.offer(transactionViewModel.getBranchTransactionHash()); + } } } } catch (Exception e) { @@ -258,53 +262,52 @@ public Map generateBalanceDiff(Set visitedTransactions, Hash s * If inconsistencies in the {@code snapshotIndex} are found it issues a reset of the corresponding milestone to * recover from this problem.
* - * @param milestone the milestone that shall have its {@link net.helix.hlx.model.StateDiff} generated + * @param round the milestone that shall have its {@link net.helix.hlx.model.StateDiff} generated * @return {@code true} if the {@link net.helix.hlx.model.StateDiff} could be generated and {@code false} otherwise * @throws LedgerException if anything unexpected happens while generating the {@link net.helix.hlx.model.StateDiff} */ - private boolean generateStateDiff(RoundViewModel milestone) throws LedgerException { + private boolean generateStateDiff(RoundViewModel round) throws LedgerException { try { - TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, milestone.getHash()); + /*TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, round.getHash()); if (!transactionViewModel.isSolid()) { return false; } final int transactionSnapshotIndex = transactionViewModel.snapshotIndex(); - boolean successfullyProcessed = transactionSnapshotIndex == milestone.index(); + boolean successfullyProcessed = transactionSnapshotIndex == round.index(); if (!successfullyProcessed) { // if the snapshotIndex of our transaction was set already, we have processed our milestones in // the wrong order (i.e. while rescanning the db) if (transactionSnapshotIndex != 0) { - milestoneService.resetCorruptedMilestone(milestone.index()); - } + milestoneService.resetCorruptedRound(round.index()); + }*/ + boolean successfullyProcessed = false; snapshotProvider.getLatestSnapshot().lockRead(); try { - Hash tail = transactionViewModel.getHash(); - Map balanceChanges = generateBalanceDiff(new HashSet<>(), tail, + Set comfirmedTips = milestoneService.getConfirmedTips(round.index(), 2); + Map balanceChanges = generateBalanceDiff(new HashSet<>(), comfirmedTips, snapshotProvider.getLatestSnapshot().getIndex()); successfullyProcessed = balanceChanges != null; if (successfullyProcessed) { successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); if (successfullyProcessed) { - milestoneService.updateMilestoneIndexOfMilestoneTransactions(milestone.getHash(), - milestone.index()); + milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); if (!balanceChanges.isEmpty()) { - new StateDiffViewModel(balanceChanges, milestone.getHash()).store(tangle); + new StateDiffViewModel(balanceChanges, round.index()).store(tangle); } } } } finally { snapshotProvider.getLatestSnapshot().unlockRead(); } - } return successfullyProcessed; } catch (Exception e) { - throw new LedgerException("unexpected error while generating the StateDiff for " + milestone, e); + throw new LedgerException("unexpected error while generating the StateDiff for Round" + round.index(), e); } } } diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index 7d8cc927..4f3d319f 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -62,11 +62,10 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i * if the transaction that got referenced is still "known" to the node by having a sufficiently high pruning * delay).
* - * @param milestoneHash the hash of the transaction * @param newIndex the milestone index that shall be set * @throws MilestoneException if anything unexpected happens while updating the milestone index */ - void updateRoundIndexOfMilestoneTransactions(Hash milestoneHash, int newIndex) throws MilestoneException; + void updateRoundIndexOfMilestoneTransactions(int newIndex) throws MilestoneException; /** * Resets all milestone related information of the transactions that were "confirmed" by the given milestone and diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 0cf556e5..c9829c8c 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -2,6 +2,7 @@ import net.helix.hlx.BundleValidator; import net.helix.hlx.conf.ConsensusConfig; +import net.helix.hlx.controllers.ApproveeViewModel; import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.Sha3; @@ -12,6 +13,7 @@ import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.StateDiff; +import net.helix.hlx.service.ledger.LedgerException; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; import net.helix.hlx.service.milestone.MilestoneValidity; @@ -22,6 +24,7 @@ import net.helix.hlx.utils.Serializer; import net.helix.hlx.utils.dag.DAGHelper; +import net.helix.hlx.utils.dag.TraversalException; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,13 +136,13 @@ public Optional findLatestProcessedSolidRoundInDatabase() throws } @Override - public void updateRoundIndexOfMilestoneTransactions(Hash milestoneHash, int index) throws MilestoneException { + public void updateRoundIndexOfMilestoneTransactions(int index) throws MilestoneException { if (index <= 0) { throw new MilestoneException("the new index needs to be bigger than 0 " + "(use resetCorruptedMilestone to reset the milestone index)"); } - updateRoundIndexOfMilestoneTransactions(milestoneHash, index, index, new HashSet<>()); + updateRoundIndexOfMilestoneTransactions(index, index, new HashSet<>()); } /** @@ -393,45 +396,43 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { } /** - * This method implements the logic described by {@link #updateRoundIndexOfMilestoneTransactions(Hash, int)} but + * This method implements the logic described by {@link #updateRoundIndexOfMilestoneTransactions(int)} but * accepts some additional parameters that allow it to be reused by different parts of this service.
* - * @param milestoneHash the hash of the transaction * @param correctIndex the milestone index of the milestone that would be set if all transactions are marked * correctly * @param newIndex the milestone index that shall be set * @throws MilestoneException if anything unexpected happens while updating the milestone index * @param processedTransactions a set of transactions that have been processed already (for the recursive calls) */ - private void updateRoundIndexOfMilestoneTransactions(Hash milestoneHash, int correctIndex, int newIndex, + private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, Set processedTransactions) throws MilestoneException { - processedTransactions.add(milestoneHash); - Set inconsistentMilestones = new HashSet<>(); - Set transactionsToUpdate = new HashSet<>(); try { - prepareRoundIndexUpdate(TransactionViewModel.fromHash(tangle, milestoneHash), correctIndex, newIndex, - inconsistentMilestones, transactionsToUpdate); - - DAGHelper.get(tangle).traverseApprovees( - milestoneHash, - currentTransaction -> { - // if the transaction was confirmed by a PREVIOUS milestone -> check if we have a back-referencing - // transaction and abort the traversal - if (isTransactionConfirmed(currentTransaction, correctIndex - 1)) { - patchSolidEntryPointsIfNecessary(snapshotProvider.getInitialSnapshot(), currentTransaction); - - return false; + final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex, 2)); + Hash transactionPointer; + while ((transactionPointer = transactionsToUpdate.poll()) != null) { + if (processedTransactions.add(transactionPointer)) { + final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, + transactionPointer); + if (isTransactionConfirmed(transactionViewModel, correctIndex - 1)) { + patchSolidEntryPointsIfNecessary(snapshotProvider.getInitialSnapshot(), transactionViewModel); + } else { + prepareRoundIndexUpdate(transactionViewModel, correctIndex, newIndex, + inconsistentMilestones, transactionsToUpdate); + updateRoundIndexOfSingleTransaction(transactionViewModel, newIndex); + if (!transactionsToUpdate.contains(transactionViewModel.getTrunkTransactionHash())) { + transactionsToUpdate.offer(transactionViewModel.getTrunkTransactionHash()); } + if (!transactionsToUpdate.contains(transactionViewModel.getBranchTransactionHash())) { + transactionsToUpdate.offer(transactionViewModel.getBranchTransactionHash()); + } + } + } + } - return true; - }, - currentTransaction -> prepareRoundIndexUpdate(currentTransaction, correctIndex, newIndex, - inconsistentMilestones, transactionsToUpdate), - processedTransactions - ); } catch (Exception e) { throw new MilestoneException("error while updating the milestone index", e); } @@ -440,9 +441,6 @@ private void updateRoundIndexOfMilestoneTransactions(Hash milestoneHash, int cor resetCorruptedRound(inconsistentMilestoneIndex, processedTransactions); } - for (TransactionViewModel transactionToUpdate : transactionsToUpdate) { - updateRoundIndexOfSingleTransaction(transactionToUpdate, newIndex); - } } /** @@ -495,13 +493,13 @@ private void updateRoundIndexOfSingleTransaction(TransactionViewModel transactio * parameter] */ private void prepareRoundIndexUpdate(TransactionViewModel transaction, int correctMilestoneIndex, - int newMilestoneIndex, Set corruptMilestones, Set transactionsToUpdate) { + int newMilestoneIndex, Set corruptMilestones, Queue transactionsToUpdate) { if(transaction.snapshotIndex() > correctMilestoneIndex) { corruptMilestones.add(transaction.snapshotIndex()); } if (transaction.snapshotIndex() != newMilestoneIndex) { - transactionsToUpdate.add(transaction); + transactionsToUpdate.offer(transaction.getHash()); } } @@ -574,10 +572,8 @@ private void resetCorruptedRound(int index, Set processedTransactions) thr if(roundToRepair.index() <= snapshotProvider.getLatestSnapshot().getIndex()) { snapshotService.rollBackMilestones(snapshotProvider.getLatestSnapshot(), roundToRepair.index()); } - for (Hash milestone : roundToRepair.getHashes()) { - updateRoundIndexOfMilestoneTransactions(milestone, roundToRepair.index(), 0, + updateRoundIndexOfMilestoneTransactions(roundToRepair.index(), 0, processedTransactions); - } tangle.delete(StateDiff.class, new IntegerIndex(roundToRepair.index())); } } catch (Exception e) { From a3b1221badb39eacbe6095ac24c95cc92a2de427 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 27 May 2019 18:00:04 +0200 Subject: [PATCH 013/223] resolve conflicts --- src/main/java/net/helix/hlx/Helix.java | 7 +- .../net/helix/hlx/conf/BaseHelixConfig.java | 43 +- src/main/java/net/helix/hlx/service/API.java | 1086 ++++++++--------- 3 files changed, 535 insertions(+), 601 deletions(-) diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index b3d2c401..0f8ec216 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -4,7 +4,6 @@ import net.helix.hlx.conf.TipSelConfig; import net.helix.hlx.controllers.TipsViewModel; import net.helix.hlx.controllers.TransactionViewModel; -import net.helix.hlx.model.persistables.Round; import net.helix.hlx.network.Node; import net.helix.hlx.network.TransactionRequester; import net.helix.hlx.network.UDPReceiver; @@ -176,7 +175,7 @@ public void init() throws Exception { } if (configuration.isRevalidate()) { - tangle.clearColumn(Round.class); + tangle.clearColumn(net.helix.hlx.model.persistables.Milestone.class); tangle.clearColumn(net.helix.hlx.model.StateDiff.class); tangle.clearMetadata(net.helix.hlx.model.persistables.Transaction.class); } @@ -238,7 +237,7 @@ private void rescanDb() throws Exception { tangle.clearColumn(net.helix.hlx.model.persistables.Approvee.class); tangle.clearColumn(net.helix.hlx.model.persistables.BundleNonce.class); tangle.clearColumn(net.helix.hlx.model.persistables.Tag.class); - tangle.clearColumn(Round.class); + tangle.clearColumn(net.helix.hlx.model.persistables.Milestone.class); tangle.clearColumn(net.helix.hlx.model.StateDiff.class); tangle.clearMetadata(net.helix.hlx.model.persistables.Transaction.class); @@ -313,4 +312,4 @@ private TipSelector createTipSelector(TipSelConfig config) { return new TipSelectorImpl(tangle, snapshotProvider, ledgerService, entryPointSelector, ratingCalculator, walker, config); } -} +} \ No newline at end of file diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index a1d534ba..1e3af764 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1,10 +1,5 @@ package net.helix.hlx.conf; -import net.helix.hlx.HLX; -import net.helix.hlx.model.Hash; -import net.helix.hlx.model.HashFactory; -import net.helix.hlx.utils.HelixUtils; - import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; @@ -12,12 +7,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.lang3.ArrayUtils; +import net.helix.hlx.HLX; +import net.helix.hlx.utils.HelixUtils; + import java.util.ArrayList; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /* Note: the fields in this class are being deserialized from Jackson so they must follow Java Bean convention. Meaning that every field must have a getter that is prefixed with `get` unless it is a boolean and then it should be @@ -103,16 +97,16 @@ public abstract class BaseHelixConfig implements HelixConfig { protected int localSnapshotsDepth = Defaults.LOCAL_SNAPSHOTS_DEPTH; protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; - //Milestone - protected int msDelay = Defaults.MS_DELAY; - protected int minDelay = Defaults.MS_MIN_DELAY; - protected Set validatorAddresses = Defaults.VALIDATOR_ADDRESSES; - //Logging protected boolean saveLogEnabled = Defaults.SAVELOG_ENABLED; protected String saveLogBasePath = Defaults.SAVELOG_BASE_PATH; protected String saveLogXMLFile = Defaults.SAVELOG_XML_FILE; + //Milestone + protected int msDelay = Defaults.MS_DELAY; + protected int minDelay = Defaults.MS_MIN_DELAY; + protected String cooAddress = Defaults.COORDINATOR_ADDRESS; + public BaseHelixConfig() { //empty constructor } @@ -675,8 +669,8 @@ protected void setCacheSizeBytes(int cacheSizeBytes) { } @Override - public Set getValidatorAddresses() { - return Defaults.VALIDATOR_ADDRESSES; + public String getCoordinator() { + return cooAddress; } @Override @@ -727,16 +721,17 @@ public int getPowThreads() { return powThreads; } - @Override - public int getMsDelay() { - return msDelay; - } @JsonProperty @Parameter(names = "--pow-threads", description = PoWConfig.Descriptions.POW_THREADS) protected void setPowThreads(int powThreads) { this.powThreads = powThreads; } + @Override + public int getMsDelay() { + return msDelay; + } + @JsonProperty @Parameter(names = {"--ms-delay", "-m"}, description = MilestoneConfig.Descriptions.MS_DELAY) protected void setMsDelay(int delay) { this.msDelay = delay; } @@ -819,7 +814,7 @@ public interface Defaults { double P_SEND_MILESTONE = 0.02d; double P_PROPAGATE_REQUEST = 0.01d; int MWM = 1; - int PACKET_SIZE = 1200; + int PACKET_SIZE = 800; int REQ_HASH_SIZE = 32; int QUEUE_SIZE = 1_000; double P_DROP_CACHE_ENTRY = 0.02d; @@ -842,10 +837,10 @@ public interface Defaults { boolean TIP_SOLIDIFIER_ENABLED = true; //PoW - int POW_THREADS = 0; + int POW_THREADS = 8; - //Validator Addresses - Set VALIDATOR_ADDRESSES = Stream.of(HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a")).collect(Collectors.toSet()); + //Milestone + String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; int MS_DELAY = 0; int MS_MIN_DELAY = 5; diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index de6c5e2c..8e1254ce 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1,26 +1,27 @@ package net.helix.hlx.service; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; import net.helix.hlx.*; import net.helix.hlx.HLX; import net.helix.hlx.conf.APIConfig; -import net.helix.hlx.conf.ConsensusConfig; -import net.helix.hlx.controllers.AddressViewModel; -import net.helix.hlx.controllers.BundleViewModel; -import net.helix.hlx.controllers.RoundViewModel; -import net.helix.hlx.controllers.TagViewModel; -import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.controllers.*; import net.helix.hlx.crypto.*; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.persistables.Transaction; import net.helix.hlx.network.Neighbor; +import net.helix.hlx.network.Node; +import net.helix.hlx.network.TransactionRequester; import net.helix.hlx.service.dto.*; +import net.helix.hlx.service.ledger.LedgerService; +import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.restserver.RestConnector; +import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.service.spentaddresses.SpentAddressesService; import net.helix.hlx.service.tipselection.TipSelector; import net.helix.hlx.service.tipselection.impl.WalkValidatorImpl; - -import net.helix.hlx.utils.HelixIOUtils; +import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Serializer; import com.google.gson.Gson; @@ -30,33 +31,20 @@ import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.xnio.channels.StreamSinkChannel; -import org.xnio.streams.ChannelInputStream; import java.io.*; -import java.net.InetSocketAddress; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.security.InvalidAlgorithmParameterException; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.regex.Pattern; import java.util.regex.Matcher; import java.util.stream.IntStream; -import io.undertow.Undertow; -import io.undertow.security.api.AuthenticationMode; -import io.undertow.security.impl.BasicAuthenticationMechanism; -import io.undertow.server.HttpHandler; -import io.undertow.server.HttpServerExchange; -import io.undertow.util.*; - -import static io.undertow.Handlers.path; - /** *

* The API makes it possible to interact with the node by requesting information or actions to be taken. @@ -73,228 +61,149 @@ *

*/ @SuppressWarnings("unchecked") - public class API { + private static final Logger log = LoggerFactory.getLogger(API.class); + + //region [CONSTANTS] /////////////////////////////////////////////////////////////////////////////// + public static final String REFERENCE_TRANSACTION_NOT_FOUND = "reference transaction not found"; public static final String REFERENCE_TRANSACTION_TOO_OLD = "reference transaction is too old"; - public static final String INVALID_SUBTANGLE = "This operation cannot be executed: " - + "The subtangle has not been updated yet."; - private static final Logger log = LoggerFactory.getLogger(API.class); - private final HXI hxi; - private final int milestoneStartIndex; + public static final String INVALID_SUBTANGLE = "This operation cannot be executed: " + + "The subtangle has not been updated yet."; - private Undertow server; + private static final String OVER_MAX_ERROR_MESSAGE = "Could not complete request"; + private static final String INVALID_PARAMS = "Invalid parameters"; - private final Gson gson = new GsonBuilder().create(); - private GreedyMiner miner = new GreedyMiner(); - - private final AtomicInteger counter = new AtomicInteger(0); - private Pattern hexPattern = Pattern.compile("[0-9a-f]*"); + private final static char ZERO_LENGTH_ALLOWED = 'Y'; + private final static char ZERO_LENGTH_NOT_ALLOWED = 'N'; private final static int HASH_SIZE = 32; private final static int BYTES_SIZE = 768; private final static long MAX_TIMESTAMP_VALUE = (long) (Math.pow(2, 8) - 1) / 2; // max positive 8 byte value + //endregion //////////////////////////////////////////////////////////////////////////////////////////////////////// + + private static int counterGetTxToApprove = 0; + private static long ellapsedTime_getTxToApprove = 0L; + private static int counter_PoW = 0; + private static long ellapsedTime_PoW = 0L; + + //region [CONSTRUCTOR_FIELDS] /////////////////////////////////////////////////////////////////////////////// + + private final HelixConfig configuration; + private final HXI hxi; + private final TransactionRequester transactionRequester; + private final SpentAddressesService spentAddressesService; + private final Tangle tangle; + private final BundleValidator bundleValidator; + private final SnapshotProvider snapshotProvider; + private final LedgerService ledgerService; + private final Node node; + private final TipSelector tipsSelector; + private final TipsViewModel tipsViewModel; + private final TransactionValidator transactionValidator; + private final LatestMilestoneTracker latestMilestoneTracker; + private final Graphstream graph; + private final int maxFindTxs; private final int maxRequestList; private final int maxGetBytes; - private final int maxBodyLength; - private final boolean testNet; - private final static String overMaxErrorMessage = "Could not complete request"; - private final static String invalidParams = "Invalid parameters"; + private final String[] features; - private ConcurrentHashMap previousEpochsSpentAddresses; + //endregion //////////////////////////////////////////////////////////////////////////////////////////////////////// - //TODO: delete (Iota) - private final static char ZERO_LENGTH_ALLOWED = 'Y'; - private final static char ZERO_LENGTH_NOT_ALLOWED = 'N'; - private Helix instance; + private final Gson gson = new GsonBuilder().create(); + private GreedyMiner miner = new GreedyMiner(); - private final String[] features; + private final AtomicInteger counter = new AtomicInteger(0); + private Pattern hexPattern = Pattern.compile("[0-9a-f]*"); + + private final int milestoneStartIndex; + + final Map, AbstractResponse>> commandRoute; + private RestConnector connector; /** * Starts loading the Helix API, parameters do not have to be initialized. * - * @param instance The data source we interact with during any API call. + * @param configuration configuration * @param hxi If a command is not in the standard API, - * we try to process it as a Nashorn JavaScript module through {@link HXI} + * we try to process it as a Nashorn JavaScript module through {@link HXI} + * @param transactionRequester Service where transactions get requested + * @param spentAddressesService Service to check if addresses are spent + * @param tangle The transaction storage + * @param bundleValidator Validates bundles + * @param snapshotProvider Manager of our currently taken snapshots + * @param ledgerService contains all the relevant business logic for modifying and calculating the ledger state. + * @param node Handles and manages neighbors + * @param tipsSelector Handles logic for selecting tips based on other transactions + * @param tipsViewModel Contains the current tips of this node + * @param transactionValidator Validates transactions + * @param latestMilestoneTracker Service that tracks the latest milestone */ - public API(Helix instance, HXI hxi) { - this.instance = instance; + public API(HelixConfig configuration, HXI hxi, TransactionRequester transactionRequester, + SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, + SnapshotProvider snapshotProvider, LedgerService ledgerService, Node node, TipSelector tipsSelector, + TipsViewModel tipsViewModel, TransactionValidator transactionValidator, + LatestMilestoneTracker latestMilestoneTracker, Graphstream graph) { + this.configuration = configuration; this.hxi = hxi; - APIConfig configuration = instance.configuration; + + this.transactionRequester = transactionRequester; + this.spentAddressesService = spentAddressesService; + this.tangle = tangle; + this.bundleValidator = bundleValidator; + this.snapshotProvider = snapshotProvider; + this.ledgerService = ledgerService; + this.node = node; + this.tipsSelector = tipsSelector; + this.tipsViewModel = tipsViewModel; + this.transactionValidator = transactionValidator; + this.latestMilestoneTracker = latestMilestoneTracker; + this.graph = graph; + maxFindTxs = configuration.getMaxFindTransactions(); maxRequestList = configuration.getMaxRequestsList(); maxGetBytes = configuration.getMaxBytes(); - maxBodyLength = configuration.getMaxBodyLength(); - testNet = configuration.isTestnet(); - milestoneStartIndex = ((ConsensusConfig) configuration).getMilestoneStartIndex(); - - previousEpochsSpentAddresses = new ConcurrentHashMap<>(); - - features = Feature.calculateFeatureNames(instance.configuration); + milestoneStartIndex = configuration.getMilestoneStartIndex(); + + features = Feature.calculateFeatureNames(configuration); + + commandRoute = new HashMap<>(); + commandRoute.put(ApiCommand.ADD_NEIGHBORS, addNeighbors()); + commandRoute.put(ApiCommand.ATTACH_TO_TANGLE, attachToTangle()); + commandRoute.put(ApiCommand.BROADCAST_TRANSACTIONS, broadcastTransactions()); + commandRoute.put(ApiCommand.FIND_TRANSACTIONS, findTransactions()); + commandRoute.put(ApiCommand.GET_BALANCES, getBalances()); + commandRoute.put(ApiCommand.GET_INCLUSION_STATES, getInclusionStates()); + commandRoute.put(ApiCommand.GET_NEIGHBORS, getNeighbors()); + commandRoute.put(ApiCommand.GET_NODE_INFO, getNodeInfo()); + commandRoute.put(ApiCommand.GET_NODE_API_CONFIG, getNodeAPIConfiguration()); + commandRoute.put(ApiCommand.GET_TIPS, getTips()); + commandRoute.put(ApiCommand.GET_TRANSACTIONS_TO_APPROVE, getTransactionsToApprove()); + commandRoute.put(ApiCommand.GET_TRYTES, getHBytes()); + commandRoute.put(ApiCommand.INTERRUPT_ATTACHING_TO_TANGLE, interruptAttachingToTangle()); + commandRoute.put(ApiCommand.REMOVE_NEIGHBORS, removeNeighbors()); + commandRoute.put(ApiCommand.STORE_TRANSACTIONS, storeTransactions()); + commandRoute.put(ApiCommand.GET_MISSING_TRANSACTIONS, getMissingTransactions()); + commandRoute.put(ApiCommand.CHECK_CONSISTENCY, checkConsistency()); + commandRoute.put(ApiCommand.WERE_ADDRESSES_SPENT_FROM, wereAddressesSpentFrom()); } /** - * Prepares the API for usage. Until this method is called, no HTTP requests can be made. - * The order of loading is as follows - *
    - *
  1. - * Read the spend addresses from the previous epoch. Used in wasAddressSpentFrom. - * This only happens if {@link APIConfig#isTestnet()} is false - * If reading from the previous epoch fails, a log is printed. The API will continue to initialize. - *
    2. - *
  2. - * Get the {@link APIConfig} from the {@link Helix} instance, - * and read {@link APIConfig#getPort()} and {@link APIConfig#getApiHost()} - *
    3. - *
  3. - * Builds a secure {@link Undertow} server with the port and host. - * If {@link APIConfig#getRemoteAuth()} is defined, remote authentication is blocked for anyone except - * those defined in {@link APIConfig#getRemoteAuth()} or localhost. - * This is done with {@link BasicAuthenticationMechanism} in a {@link AuthenticationMode#PRO_ACTIVE} mode. - * By default, this authentication is disabled. - *
    4. - *
  4. - * Starts the server, opening it for HTTP API requests - *
    5. - *
+ * Initializes the API for usage. + * Will initialize and start the supplied {@link RestConnector} * - * @throws IOException If we are not on the testnet, and the previousEpochsSpentAddresses files cannot be found. - * Currently this exception is caught + * @param connector THe connector we use to handle API requests */ - public void init() throws IOException { - APIConfig configuration = instance.configuration; - final int apiPort = configuration.getPort(); - final String apiHost = configuration.getApiHost(); - - log.debug("Binding JSON-REST API Undertow server on {}:{}", apiHost, apiPort); - - server = Undertow.builder().addHttpListener(apiPort, apiHost) - .setHandler(path().addPrefixPath("/", addSecurity(new HttpHandler() { - @Override - public void handleRequest(final HttpServerExchange exchange) throws Exception { - HttpString requestMethod = exchange.getRequestMethod(); - if (Methods.OPTIONS.equals(requestMethod)) { - String allowedMethods = "GET,HEAD,POST,PUT,DELETE,TRACE,OPTIONS,CONNECT,PATCH"; - //return list of allowed methods in response headers - exchange.setStatusCode(StatusCodes.OK); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, MimeMappings.DEFAULT_MIME_MAPPINGS.get("txt")); - exchange.getResponseHeaders().put(Headers.CONTENT_LENGTH, 0); - exchange.getResponseHeaders().put(Headers.ALLOW, allowedMethods); - exchange.getResponseHeaders().put(new HttpString("Access-Control-Allow-Origin"), "*"); - exchange.getResponseHeaders().put(new HttpString("Access-Control-Allow-Headers"), "Origin, X-Requested-With, Content-Type, Accept, authorization, X-HELIX-API-Version"); - exchange.getResponseSender().close(); - return; - } - - if (exchange.isInIoThread()) { - exchange.dispatch(this); - return; - } - processRequest(exchange); - } - }))).build(); - server.start(); - } - - /** - * Sends the API response back as JSON to the requester. - * Status code of the HTTP request is also set according to the type of response. - *
    - *
  • {@link ErrorResponse}: 400
    • - *
  • {@link AccessLimitedResponse}: 401
    • - *
  • {@link ExceptionResponse}: 500
    • - *
  • Default: 200
    • - *
- * - * @param exchange Contains information about what the client sent to us - * @param res The response of the API. - * See {@link #processRequest(HttpServerExchange)} - * and {@link #process(String, InetSocketAddress)} for the different responses in each case. - * @param beginningTime The time when we received the request, in milliseconds. - * This will be used to set the response duration in {@link AbstractResponse#setDuration(Integer)} - * @throws IOException When connection to client has been lost - Currently being caught. - */ - private void sendResponse(final HttpServerExchange exchange, final AbstractResponse res, final long beginningTime) - throws IOException { - res.setDuration((int) (System.currentTimeMillis() - beginningTime)); - final String response = gson.toJson(res); - - if (res instanceof ErrorResponse) { - exchange.setStatusCode(400); // bad request - } else if (res instanceof AccessLimitedResponse) { - exchange.setStatusCode(401); // api method not allowed - } else if (res instanceof ExceptionResponse) { - exchange.setStatusCode(500); // internal error - } - - setupResponseHeaders(exchange); - - ByteBuffer responseBuf = ByteBuffer.wrap(response.getBytes(StandardCharsets.UTF_8)); - exchange.setResponseContentLength(responseBuf.array().length); - StreamSinkChannel sinkChannel = exchange.getResponseChannel(); - sinkChannel.getWriteSetter().set( channel -> { - if (responseBuf.remaining() > 0) { - try { - sinkChannel.write(responseBuf); - if (responseBuf.remaining() == 0) { - exchange.endExchange(); - } - } catch (IOException e) { - log.error("Lost connection to client - cannot send response"); - exchange.endExchange(); - sinkChannel.getWriteSetter().set(null); - } - } - else { - exchange.endExchange(); - } - }); - sinkChannel.resumeWrites(); - } - - /** - *

- * Processes an API HTTP request. - * No checks have been done until now, except that it is not an OPTIONS request. - * We can be sure that we are in a thread that allows blocking. - *

- *

- * The request process duration is recorded. - * During this the request gets verified. If it is incorrect, an {@link ErrorResponse} is made. - * Otherwise it is processed in {@link #process(String, InetSocketAddress)}. - * The result is sent back to the requester. - *

- * - * @param exchange Contains the data the client sent to us - * @throws IOException If the body of this HTTP request cannot be read - */ - private void processRequest(final HttpServerExchange exchange) throws IOException { - final ChannelInputStream cis = new ChannelInputStream(exchange.getRequestChannel()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - - final long beginningTime = System.currentTimeMillis(); - final String body = HelixIOUtils.toString(cis, StandardCharsets.UTF_8); - final AbstractResponse response; - String rcvdToken = (exchange.getRequestHeaders().get("Authorization") == null) ? "" : RemoteAuth.getToken(exchange.getRequestHeaders().get("Authorization").get(0)); - String serverToken = instance.configuration.getRemoteAuth(); - - if (!exchange.getRequestHeaders().contains("X-HELIX-API-Version")) { - response = ErrorResponse.create("Invalid API Version"); - } else if (body.length() > maxBodyLength) { - response = ErrorResponse.create("Request too long"); - // TODO: review and improve the authentication mechanism - } else if(!serverToken.equals("") && !rcvdToken.equals(serverToken)) { - response = ErrorResponse.create("Authorization failed"); - } else { - response = process(body, exchange.getSourceAddress()); - } - sendResponse(exchange, response, beginningTime); + public void init(RestConnector connector){ + this.connector = connector; + connector.init(this::process); + connector.start(); } /** @@ -324,19 +233,25 @@ private void processRequest(final HttpServerExchange exchange) throws IOExceptio * * @param requestString The JSON encoded data of the request. * This String is attempted to be converted into a {@code Map}. - * @param sourceAddress The address from the sender of this API request. + * @param netAddress The address from the sender of this API request. * @return The result of this request. - * @throws UnsupportedEncodingException If the requestString cannot be parsed into a Map. - Currently caught and turned into a {@link ExceptionResponse}. */ - private AbstractResponse process(final String requestString, InetSocketAddress sourceAddress) throws UnsupportedEncodingException { + private AbstractResponse process(final String requestString, InetAddress netAddress) { try { // Request JSON data into map - final Map request = gson.fromJson(requestString, Map.class); + Map request; + try { + request = gson.fromJson(requestString, Map.class); + } + catch(JsonSyntaxException jsonSyntaxException) { + return ErrorResponse.create("Invalid JSON syntax: " + jsonSyntaxException.getMessage()); + } + if (request == null) { return ExceptionResponse.create("Invalid request payload: '" + requestString + "'"); } + // Did the requester ask for a command? final String command = (String) request.get("command"); if (command == null) { @@ -345,145 +260,31 @@ private AbstractResponse process(final String requestString, InetSocketAddress s // Is this command allowed to be run from this request address? // We check the remote limit API configuration. - if (instance.configuration.getRemoteLimitApi().contains(command) && - !sourceAddress.getAddress().isLoopbackAddress()) { + if (configuration.getRemoteLimitApi().contains(command) && !configuration.getRemoteTrustedApiHosts().contains(netAddress)) { return AccessLimitedResponse.create("COMMAND " + command + " is not available on this node"); } log.debug("# {} -> Requesting command '{}'", counter.incrementAndGet(), command); - switch (command) { - case "storeMessage": { - if (!testNet) { - return AccessLimitedResponse.create("COMMAND storeMessage is only available on testnet"); - } - - if (!request.containsKey("address") || !request.containsKey("message")) { - return ErrorResponse.create("Invalid params"); - } - - final String address = (String) request.get("address"); - final String message = (String) request.get("message"); - - return storeMessageStatement(address, message); - } - - case "addNeighbors": { - List uris = getParameterAsList(request,"uris",0); - log.debug("Invoking 'addNeighbors' with {}", uris); - return addNeighborsStatement(uris); - } - case "attachToTangle": { - final Hash trunkTransaction = HashFactory.TRANSACTION.create(getParameterAsStringAndValidate(request,"trunkTransaction", HASH_SIZE)); - final Hash branchTransaction = HashFactory.TRANSACTION.create(getParameterAsStringAndValidate(request,"branchTransaction", HASH_SIZE)); - final int minWeightMagnitude = getParameterAsInt(request,"minWeightMagnitude"); - final List hbytes = getParameterAsList(request,"hbytes", BYTES_SIZE); - - List elements = attachToTangleStatement(trunkTransaction, branchTransaction, minWeightMagnitude, hbytes); - return AttachToTangleResponse.create(elements); - } - case "broadcastTransactions": { - final List hbytes = getParameterAsList(request,"hbytes", BYTES_SIZE); - broadcastTransactionsStatement(hbytes); - return AbstractResponse.createEmptyResponse(); - } - case "findTransactions": { - return findTransactionsStatement(request); - } - case "getBalances": { - final List addresses = getParameterAsList(request,"addresses", HASH_SIZE); - final List tips = request.containsKey("tips") ? - getParameterAsList(request,"tips", HASH_SIZE): - null; - final int threshold = getParameterAsInt(request, "threshold"); - return getBalancesStatement(addresses, tips, threshold); - } - case "getInclusionStates": { - if (invalidSubtangleStatus()) { - return ErrorResponse.create(INVALID_SUBTANGLE); - } - final List transactions = getParameterAsList(request,"transactions", HASH_SIZE); - final List tips = getParameterAsList(request,"tips", HASH_SIZE); - - return getInclusionStatesStatement(transactions, tips); - } - case "getNeighbors": { - return getNeighborsStatement(); - } - case "getNodeInfo": { - return getNodeInfoStatement(); - } - case "getTips": { - return getTipsStatement(); - } - case "getTransactionsToApprove": { - final Optional reference = request.containsKey("reference") ? - Optional.of(HashFactory.TRANSACTION.create(getParameterAsStringAndValidate(request,"reference", HASH_SIZE))) - : Optional.empty(); - final int depth = getParameterAsInt(request, "depth"); - return getTransactionsToApproveStatement(depth, reference); - } - case "getHBytes": { - final List hashes = getParameterAsList(request,"hashes", HASH_SIZE); - return getHBytesStatement(hashes); - } - - case "interruptAttachingToTangle": { - return interruptAttachingToTangleStatement(); - } - case "removeNeighbors": { - List uris = getParameterAsList(request,"uris",0); - log.debug("Invoking 'removeNeighbors' with {}", uris); - return removeNeighborsStatement(uris); - } - - case "storeTransactions": { - try { - final List hbytes = getParameterAsList(request, "hbytes", BYTES_SIZE); - storeTransactionsStatement(hbytes); - return AbstractResponse.createEmptyResponse(); - } catch (RuntimeException e) { - // tx not valid - return ErrorResponse.create("Invalid hbytes input!"); - } - } - case "getMissingTransactions": { - //TransactionRequester.instance().rescanTransactionsToRequest(); - synchronized (instance.transactionRequester) { - List missingTx = Arrays.stream(instance.transactionRequester.getRequestedTransactions()) - .map(Hash::hexString) - .collect(Collectors.toList()); - return GetTipsResponse.create(missingTx); - } - } - case "checkConsistency": { - if (invalidSubtangleStatus()) { - return ErrorResponse - .create(INVALID_SUBTANGLE); - } - final List transactions = getParameterAsList(request,"tails", HASH_SIZE); - return checkConsistencyStatement(transactions); - } - case "wereAddressesSpentFrom": { - final List addresses = getParameterAsList(request,"addresses", HASH_SIZE); - return wereAddressesSpentFromStatement(addresses); - } - default: { - AbstractResponse response = hxi.processCommand(command, request); - return response == null ? - ErrorResponse.create("Command [" + command + "] is unknown") : - response; + ApiCommand apiCommand = ApiCommand.findByName(command); + if (apiCommand != null) { + return commandRoute.get(apiCommand).apply(request); + } else { + AbstractResponse response = hxi.processCommand(command, request); + if (response == null) { + return ErrorResponse.create("Command [" + command + "] is unknown"); + } else { + return response; } } - - } catch (final ValidationException e) { - log.info("API Validation failed: " + e.getLocalizedMessage()); - return ErrorResponse.create(e.getLocalizedMessage()); - } catch (final InvalidAlgorithmParameterException e) { - log.info("API InvalidAlgorithmParameter passed: " + e.getLocalizedMessage()); - return ErrorResponse.create(e.getLocalizedMessage()); - } catch (final Exception e) { - log.error("API Exception: ", e); + } catch (ValidationException e) { + log.error("API Validation failed: " + e.getLocalizedMessage()); + return ExceptionResponse.create(e.getLocalizedMessage()); + } catch (IllegalStateException e) { + log.error("API Exception: " + e.getLocalizedMessage()); + return ExceptionResponse.create(e.getLocalizedMessage()); + } catch (RuntimeException e) { + log.error("Unexpected API Exception: " + e.getLocalizedMessage()); return ExceptionResponse.create(e.getLocalizedMessage()); } } @@ -493,6 +294,7 @@ private AbstractResponse process(final String requestString, InetSocketAddress s * If an address has a pending transaction, it is also marked as spend. * * @param addresses List of addresses to check if they were ever spent from. + * @return {@link net.helix.hlx.service.dto.WereAddressesSpentFrom} **/ private AbstractResponse wereAddressesSpentFromStatement(List addresses) throws Exception { final List addressesHash = addresses.stream() @@ -503,7 +305,7 @@ private AbstractResponse wereAddressesSpentFromStatement(List addresses) int index = 0; for (Hash address : addressesHash) { - states[index++] = instance.spentAddressesService.wasAddressSpentFrom(address); + states[index++] = spentAddressesService.wasAddressSpentFrom(address); } return WereAddressesSpentFrom.create(states); } @@ -517,7 +319,7 @@ private AbstractResponse wereAddressesSpentFromStatement(List addresses) * @throws Exception When a model could not be loaded. */ private Hash findTail(Hash hash) throws Exception { - TransactionViewModel tx = TransactionViewModel.fromHash(instance.tangle, hash); + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); final Hash bundleHash = tx.getBundleHash(); long index = tx.getCurrentIndex(); boolean foundApprovee = false; @@ -525,9 +327,9 @@ private Hash findTail(Hash hash) throws Exception { // As long as the index is bigger than 0 and we are still traversing the same bundle // If the hash we asked about is already a tail, this loop never starts while (index-- > 0 && tx.getBundleHash().equals(bundleHash)) { - Set approvees = tx.getApprovers(instance.tangle).getHashes(); + Set approvees = tx.getApprovers(tangle).getHashes(); for (Hash approvee : approvees) { - TransactionViewModel nextTx = TransactionViewModel.fromHash(instance.tangle, approvee); + TransactionViewModel nextTx = TransactionViewModel.fromHash(tangle, approvee); if (nextTx.getBundleHash().equals(bundleHash)) { tx = nextTx; foundApprovee = true; @@ -564,9 +366,9 @@ private AbstractResponse checkConsistencyStatement(List transactionsList boolean state = true; String info = ""; - //check transactions themselves are valid + // Check if the transactions themselves are valid for (Hash transaction : transactions) { - TransactionViewModel txVM = TransactionViewModel.fromHash(instance.tangle, transaction); + TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); if (txVM.getType() == TransactionViewModel.PREFILLED_SLOT) { return ErrorResponse.create("Invalid transaction, missing: " + transaction); } @@ -575,22 +377,23 @@ private AbstractResponse checkConsistencyStatement(List transactionsList } - if (!instance.transactionValidator.checkSolidity(txVM.getHash(), false)) { + if (!txVM.isSolid()) { state = false; info = "tails are not solid (missing a referenced tx): " + transaction; break; - } else if (BundleValidator.validate(instance.tangle, instance.snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() == 0) { + // TODO: When validate isn't static anymore, change to bundleValidator.validate(). + } else if (BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() == 0) { state = false; info = "tails are not consistent (bundle is invalid): " + transaction; break; } } + // Transactions are valid, lets check ledger consistency if (state) { - instance.snapshotProvider.getLatestSnapshot().lockRead(); + snapshotProvider.getLatestSnapshot().lockRead(); try { - WalkValidatorImpl walkValidator = new WalkValidatorImpl(instance.tangle, instance.snapshotProvider, instance.ledgerService, - instance.configuration); + WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); for (Hash transaction : transactions) { if (!walkValidator.isValid(transaction)) { state = false; @@ -599,11 +402,11 @@ private AbstractResponse checkConsistencyStatement(List transactionsList } } } finally { - instance.snapshotProvider.getLatestSnapshot().unlockRead(); + snapshotProvider.getLatestSnapshot().unlockRead(); } } - return CheckConsistency.create(state,info); + return CheckConsistency.create(state, info); } /** @@ -613,7 +416,7 @@ private AbstractResponse checkConsistencyStatement(List transactionsList * @return false if we received at least a solid milestone, otherwise true */ public boolean invalidSubtangleStatus() { - return (instance.snapshotProvider.getLatestSnapshot().getIndex() == instance.snapshotProvider.getInitialSnapshot().getIndex()); + return (snapshotProvider.getLatestSnapshot().getIndex() == snapshotProvider.getInitialSnapshot().getIndex()); } /** * Returns the set of neighbors you are connected with, as well as their activity statistics (or counters). @@ -622,7 +425,7 @@ public boolean invalidSubtangleStatus() { * @return {@link net.helix.hlx.service.dto.GetNeighborsResponse} **/ private AbstractResponse getNeighborsStatement() { - return GetNeighborsResponse.create(instance.node.getNeighbors()); + return GetNeighborsResponse.create(node.getNeighbors()); } /** @@ -642,9 +445,9 @@ private AbstractResponse addNeighborsStatement(final List uris) { try { for (final String uriString : uris) { log.info("Adding neighbor: " + uriString); - final Neighbor neighbor = instance.node.newNeighbor(new URI(uriString), true); - if (!instance.node.getNeighbors().contains(neighbor)) { - instance.node.getNeighbors().add(neighbor); + final Neighbor neighbor = node.newNeighbor(new URI(uriString), true); + if (!node.getNeighbors().contains(neighbor)) { + node.getNeighbors().add(neighbor); numberOfAddedNeighbors++; } } @@ -672,7 +475,7 @@ private AbstractResponse removeNeighborsStatement(List uris) { try { for (final String uriString : uris) { log.info("Removing neighbor: " + uriString); - if (instance.node.removeNeighbor(new URI(uriString),true)) { + if (node.removeNeighbor(new URI(uriString),true)) { numberOfRemovedNeighbors++; } } @@ -693,56 +496,17 @@ private AbstractResponse removeNeighborsStatement(List uris) { private synchronized AbstractResponse getHBytesStatement(List hashes) throws Exception { final List elements = new LinkedList<>(); for (final String hash : hashes) { - final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(instance.tangle, HashFactory.TRANSACTION.create(hash)); + final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, HashFactory.TRANSACTION.create(hash)); if (transactionViewModel != null) { elements.add(Hex.toHexString(transactionViewModel.getBytes())); } } if (elements.size() > maxGetBytes){ - return ErrorResponse.create(overMaxErrorMessage); + return ErrorResponse.create(OVER_MAX_ERROR_MESSAGE); } return GetHBytesResponse.create(elements); } - private static int counterGetTxToApprove = 0; - - /** - * Can be 0 or more, and is set to 0 every 100 requests. - * Each increase indicates another 2 tips send. - * - * @return The current amount of times this node has returned transactions to approve - */ - private static int getCounterGetTxToApprove() { - return counterGetTxToApprove; - } - - /** - * Increases the amount of tips send for transactions to approve by one - */ - private static void incCounterGetTxToApprove() { - counterGetTxToApprove++; - } - - private static long ellapsedTime_getTxToApprove = 0L; - - /** - * Can be 0 or more, and is set to 0 every 100 requests. - * - * @return The current amount of time spent on sending transactions to approve in milliseconds - */ - private static long getEllapsedTimeGetTxToApprove() { - return ellapsedTime_getTxToApprove; - } - - /** - * Increases the current amount of time spent on sending transactions to approve - * - * @param ellapsedTime the time to add, in milliseconds - */ - private static void incEllapsedTimeGetTxToApprove(long ellapsedTime) { - ellapsedTime_getTxToApprove += ellapsedTime; - } - /** * Tip selection which returns trunkTransaction and branchTransaction. * The input value depth determines how many milestones to go back for finding the transactions to approve. @@ -756,8 +520,8 @@ private static void incEllapsedTimeGetTxToApprove(long ellapsedTime) { * @return {@link net.helix.hlx.service.dto.GetTransactionsToApproveResponse} * @throws Exception When tip selection has failed. Currently caught and returned as an {@link ErrorResponse}. **/ - private synchronized AbstractResponse getTransactionsToApproveStatement(int depth, Optional reference) throws Exception { - if (depth < 0 || depth > instance.configuration.getMaxDepth()) { + private synchronized AbstractResponse getTransactionsToApproveStatement(int depth, Optional reference) { + if (depth < 0 || depth > configuration.getMaxDepth()) { return ErrorResponse.create("Invalid depth input"); } @@ -786,7 +550,7 @@ List getTransactionToApproveTips(int depth, Optional reference) thro throw new IllegalStateException(INVALID_SUBTANGLE); } - List tips = instance.tipsSelector.getTransactionsToApprove(depth, reference); + List tips = tipsSelector.getTransactionsToApprove(depth, reference); if (log.isDebugEnabled()) { gatherStatisticsOnTipSelection(); @@ -822,7 +586,7 @@ private void gatherStatisticsOnTipSelection() { * @return {@link net.helix.hlx.service.dto.GetTipsResponse} **/ private synchronized AbstractResponse getTipsStatement() throws Exception { - return GetTipsResponse.create(instance.tipsViewModel.getTips() + return GetTipsResponse.create(tipsViewModel.getTips() .stream() .map(Hash::hexString) .collect(Collectors.toList())); @@ -843,24 +607,22 @@ public void storeTransactionsStatement(final List txHex) throws Exceptio for (final String hex : txHex) { //validate all tx txBytes = Hex.decode(hex); - final TransactionViewModel transactionViewModel = instance.transactionValidator.validateBytes(txBytes, - instance.transactionValidator.getMinWeightMagnitude()); + final TransactionViewModel transactionViewModel = transactionValidator.validateBytes(txBytes, + transactionValidator.getMinWeightMagnitude()); elements.add(transactionViewModel); } - - for (final TransactionViewModel transactionViewModel : elements) { //store transactions - if(transactionViewModel.store(instance.tangle, instance.snapshotProvider.getInitialSnapshot())) { // v + if(transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot())) { // v transactionViewModel.setArrivalTime(System.currentTimeMillis() / 1000L); - instance.transactionValidator.updateStatus(transactionViewModel); + transactionValidator.updateStatus(transactionViewModel); transactionViewModel.updateSender("local"); - transactionViewModel.update(instance.tangle, instance.snapshotProvider.getInitialSnapshot(), "sender"); + transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); } - if (instance.graph != null) { - instance.graph.addNode(transactionViewModel.getHash().hexString(), transactionViewModel.getTrunkTransactionHash().hexString(), transactionViewModel.getBranchTransactionHash().hexString()); + if (graph != null) { + graph.addNode(transactionViewModel.getHash().hexString(), transactionViewModel.getTrunkTransactionHash().hexString(), transactionViewModel.getBranchTransactionHash().hexString()); } } } @@ -881,30 +643,39 @@ private AbstractResponse interruptAttachingToTangleStatement(){ * @return {@link net.helix.hlx.service.dto.GetNodeInfoResponse} **/ private AbstractResponse getNodeInfoStatement() throws Exception { - String name = instance.configuration.isTestnet() ? HLX.TESTNET_NAME : HLX.MAINNET_NAME; - RoundViewModel milestone = RoundViewModel.first(instance.tangle); + String name = configuration.isTestnet() ? HLX.TESTNET_NAME : HLX.MAINNET_NAME; + MilestoneViewModel milestone = MilestoneViewModel.first(tangle); return GetNodeInfoResponse.create(name, HLX.VERSION, Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().freeMemory(), System.getProperty("java.version"), Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), - instance.latestMilestoneTracker.getLatestMilestoneHash(), - instance.latestMilestoneTracker.getLatestMilestoneIndex(), + latestMilestoneTracker.getLatestMilestoneHash(), + latestMilestoneTracker.getLatestMilestoneIndex(), - instance.snapshotProvider.getLatestSnapshot().getHash(), - instance.snapshotProvider.getLatestSnapshot().getIndex(), + snapshotProvider.getLatestSnapshot().getHash(), + snapshotProvider.getLatestSnapshot().getIndex(), milestone != null ? milestone.index() : -1, - instance.snapshotProvider.getLatestSnapshot().getInitialIndex(), + snapshotProvider.getLatestSnapshot().getInitialIndex(), - instance.node.howManyNeighbors(), - instance.node.queuedTransactionsSize(), + node.howManyNeighbors(), + node.queuedTransactionsSize(), System.currentTimeMillis(), - instance.tipsViewModel.size(), - instance.transactionRequester.numberOfTransactionsToRequest(), + tipsViewModel.size(), + transactionRequester.numberOfTransactionsToRequest(), features, - instance.configuration.getCoordinator()); + configuration.getCoordinator()); + } + + /** + * Returns information about this node configuration. + * + * @return {@link GetNodeAPIConfigurationResponse} + */ + private AbstractResponse getNodeAPIConfigurationStatement() { + return GetNodeAPIConfigurationResponse.create(configuration); } /** @@ -940,7 +711,7 @@ private AbstractResponse getInclusionStatesStatement(final List transact List tipsIndex = new LinkedList<>(); { for(Hash tip: tps) { - TransactionViewModel tx = TransactionViewModel.fromHash(instance.tangle, tip); + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, tip); if (tx.getType() != TransactionViewModel.PREFILLED_SLOT) { tipsIndex.add(tx.snapshotIndex()); } @@ -964,7 +735,7 @@ private AbstractResponse getInclusionStatesStatement(final List transact // Sets to 1 if the transaction index is below the max index of tips (included). for(Hash hash: trans) { - TransactionViewModel transaction = TransactionViewModel.fromHash(instance.tangle, hash); + TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, hash); if(transaction.getType() == TransactionViewModel.PREFILLED_SLOT || transaction.snapshotIndex() == 0) { inclusionStates[count] = -1; } else if(transaction.snapshotIndex() > maxTipsIndex) { @@ -975,55 +746,55 @@ private AbstractResponse getInclusionStatesStatement(final List transact count++; } - Set analyzedTips = new HashSet<>(); - Map sameIndexTransactionCount = new HashMap<>(); - Map> sameIndexTips = new HashMap<>(); + Set analyzedTips = new HashSet<>(); + Map sameIndexTransactionCount = new HashMap<>(); + Map> sameIndexTips = new HashMap<>(); - // Sorts all tips per snapshot index. Stops if a tip is not in our database, or just as a hash. - for (final Hash tip : tps) { - TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(instance.tangle, tip); - if (transactionViewModel.getType() == TransactionViewModel.PREFILLED_SLOT){ - return ErrorResponse.create("One of the tips is absent"); + // Sorts all tips per snapshot index. Stops if a tip is not in our database, or just as a hash. + for (final Hash tip : tps) { + TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, tip); + if (transactionViewModel.getType() == TransactionViewModel.PREFILLED_SLOT){ + return ErrorResponse.create("One of the tips is absent"); + } + int snapshotIndex = transactionViewModel.snapshotIndex(); + sameIndexTips.putIfAbsent(snapshotIndex, new LinkedList<>()); + sameIndexTips.get(snapshotIndex).add(tip); } - int snapshotIndex = transactionViewModel.snapshotIndex(); - sameIndexTips.putIfAbsent(snapshotIndex, new LinkedList<>()); - sameIndexTips.get(snapshotIndex).add(tip); - } - // Loop over all transactions without a state, and counts the amount per snapshot index - for(int i = 0; i < inclusionStates.length; i++) { - if(inclusionStates[i] == 0) { - TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(instance.tangle, trans.get(i)); - int snapshotIndex = transactionViewModel.snapshotIndex(); - sameIndexTransactionCount.putIfAbsent(snapshotIndex, 0); - sameIndexTransactionCount.put(snapshotIndex, sameIndexTransactionCount.get(snapshotIndex) + 1); + // Loop over all transactions without a state, and counts the amount per snapshot index + for(int i = 0; i < inclusionStates.length; i++) { + if(inclusionStates[i] == 0) { + TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, trans.get(i)); + int snapshotIndex = transactionViewModel.snapshotIndex(); + sameIndexTransactionCount.putIfAbsent(snapshotIndex, 0); + sameIndexTransactionCount.put(snapshotIndex, sameIndexTransactionCount.get(snapshotIndex) + 1); + } } - } - // Loop over all snapshot indexes of transactions that were not confirmed. - // If we encounter an invalid tangle, stop this function completely. - for(Integer index : sameIndexTransactionCount.keySet()) { - // Get the tips from the snapshot indexes we are missing - Queue sameIndexTip = sameIndexTips.get(index); + // Loop over all snapshot indexes of transactions that were not confirmed. + // If we encounter an invalid tangle, stop this function completely. + for(Integer index : sameIndexTransactionCount.keySet()) { + // Get the tips from the snapshot indexes we are missing + Queue sameIndexTip = sameIndexTips.get(index); - // We have tips on the same level as transactions, do a manual search. - if (sameIndexTip != null && !exhaustiveSearchWithinIndex( - sameIndexTip, analyzedTips, trans, - inclusionStates, sameIndexTransactionCount.get(index), index)) { + // We have tips on the same level as transactions, do a manual search. + if (sameIndexTip != null && !exhaustiveSearchWithinIndex( + sameIndexTip, analyzedTips, trans, + inclusionStates, sameIndexTransactionCount.get(index), index)) { - return ErrorResponse.create(INVALID_SUBTANGLE); + return ErrorResponse.create(INVALID_SUBTANGLE); + } + } + final boolean[] inclusionStatesBoolean = new boolean[inclusionStates.length]; + for(int i = 0; i < inclusionStates.length; i++) { + // If a state is 0 by now, we know nothing so assume not included + inclusionStatesBoolean[i] = inclusionStates[i] == 1; } - } - final boolean[] inclusionStatesBoolean = new boolean[inclusionStates.length]; - for(int i = 0; i < inclusionStates.length; i++) { - // If a state is 0 by now, we know nothing so assume not included - inclusionStatesBoolean[i] = inclusionStates[i] == 1; - } - { - return GetInclusionStatesResponse.create(inclusionStatesBoolean); + { + return GetInclusionStatesResponse.create(inclusionStatesBoolean); + } } - } final boolean[] inclusionStatesBoolean = new boolean[inclusionStates.length]; for(int i = 0; i < inclusionStates.length; i++) { inclusionStatesBoolean[i] = inclusionStates[i] == 1; @@ -1065,7 +836,7 @@ private boolean exhaustiveSearchWithinIndex( // Check if the transactions have indeed this index. Otherwise ignore. // Starts off with the tips in nonAnalyzedTransactions, but transaction trunk & branch gets added. - final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(instance.tangle, pointer); + final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, pointer); if (transactionViewModel.snapshotIndex() == index) { // Do we have the complete transaction? if (transactionViewModel.getType() == TransactionViewModel.PREFILLED_SLOT) { @@ -1122,7 +893,7 @@ private synchronized AbstractResponse findTransactionsStatement(final Map bundles = getParameterAsSet(request,"bundles",HASH_SIZE); for (final String bundle : bundles) { bundlesTransactions.addAll( - BundleViewModel.load(instance.tangle, HashFactory.BUNDLE.create(bundle)) + BundleViewModel.load(tangle, HashFactory.BUNDLE.create(bundle)) .getHashes()); } foundTransactions.addAll(bundlesTransactions); @@ -1134,7 +905,7 @@ private synchronized AbstractResponse findTransactionsStatement(final Map addresses = getParameterAsSet(request,"addresses",HASH_SIZE); for (final String address : addresses) { addressesTransactions.addAll( - AddressViewModel.load(instance.tangle, HashFactory.ADDRESS.create(address)) + AddressViewModel.load(tangle, HashFactory.ADDRESS.create(address)) .getHashes()); } foundTransactions.addAll(addressesTransactions); @@ -1147,14 +918,14 @@ private synchronized AbstractResponse findTransactionsStatement(final Map approvees = getParameterAsSet(request,"approvees",HASH_SIZE); for (final String approvee : approvees) { approveeTransactions.addAll( - TransactionViewModel.fromHash(instance.tangle, HashFactory.TRANSACTION.create(approvee)) - .getApprovers(instance.tangle) + TransactionViewModel.fromHash(tangle, HashFactory.TRANSACTION.create(approvee)) + .getApprovers(tangle) .getHashes()); } foundTransactions.addAll(approveeTransactions); @@ -1177,7 +948,7 @@ private synchronized AbstractResponse findTransactionsStatement(final Map maxFindTxs){ - return ErrorResponse.create(overMaxErrorMessage); + return ErrorResponse.create(OVER_MAX_ERROR_MESSAGE); } final List elements = foundTransactions.stream() @@ -1255,13 +1026,13 @@ public void broadcastTransactionsStatement(final List txHex) { //validate all tx txBytes = Hex.decode(hex); // TODO something possibly going wrong here. - final TransactionViewModel transactionViewModel = instance.transactionValidator.validateBytes(txBytes, instance.transactionValidator.getMinWeightMagnitude()); + final TransactionViewModel transactionViewModel = transactionValidator.validateBytes(txBytes, transactionValidator.getMinWeightMagnitude()); elements.add(transactionViewModel); } for (final TransactionViewModel transactionViewModel : elements) { //push first in line to broadcast transactionViewModel.weightMagnitude = Sha3.HASH_LENGTH; - instance.node.broadcast(transactionViewModel); + node.broadcast(transactionViewModel); } } @@ -1297,11 +1068,11 @@ private AbstractResponse getBalancesStatement(List addresses, final List hashes; final Map balances = new HashMap<>(); - instance.snapshotProvider.getLatestSnapshot().lockRead(); - final int index = instance.snapshotProvider.getLatestSnapshot().getIndex(); + snapshotProvider.getLatestSnapshot().lockRead(); + final int index = snapshotProvider.getLatestSnapshot().getIndex(); if (tips == null || tips.size() == 0) { - hashes = Collections.singletonList(instance.snapshotProvider.getLatestSnapshot().getHash()); + hashes = Collections.singletonList(snapshotProvider.getLatestSnapshot().getHash()); } else { hashes = tips.stream() .map(tip -> (HashFactory.TRANSACTION.create(tip))) @@ -1311,7 +1082,7 @@ private AbstractResponse getBalancesStatement(List addresses, try { // Get the balance for each address at the last snapshot for (final Hash address : addressList) { - Long value = instance.snapshotProvider.getLatestSnapshot().getBalance(address); + Long value = snapshotProvider.getLatestSnapshot().getBalance(address); if (value == null) { value = 0L; } @@ -1324,10 +1095,10 @@ private AbstractResponse getBalancesStatement(List addresses, // Calculate the difference created by the non-verified transactions which tips approve. // This difference is put in a map with address -> value changed for (Hash tip : hashes) { - if (!TransactionViewModel.exists(instance.tangle, tip)) { + if (!TransactionViewModel.exists(tangle, tip)) { return ErrorResponse.create("Tip not found: " + Hex.toHexString(tip.bytes())); } - if (!instance.ledgerService.isBalanceDiffConsistent(visitedHashes, diff, tip)) { + if (!ledgerService.isBalanceDiffConsistent(visitedHashes, diff, tip)) { return ErrorResponse.create("Tips are not consistent"); } } @@ -1335,7 +1106,7 @@ private AbstractResponse getBalancesStatement(List addresses, // Update the found balance according to 'diffs' balance changes diff.forEach((key, value) -> balances.computeIfPresent(key, (hash, aLong) -> value + aLong)); } finally { - instance.snapshotProvider.getLatestSnapshot().unlockRead(); + snapshotProvider.getLatestSnapshot().unlockRead(); } final List elements = addressList.stream() @@ -1347,8 +1118,6 @@ private AbstractResponse getBalancesStatement(List addresses, .collect(Collectors.toList()), index); } - private static int counter_PoW = 0; - /** * Can be 0 or more, and is set to 0 every 100 requests. * Each increase indicates another 2 tips sent. @@ -1367,8 +1136,6 @@ public static void incCounterPoW() { API.counter_PoW++; } - private static long ellapsedTime_PoW = 0L; - /** * Can be 0 or more, and is set to 0 every 100 requests. * @@ -1458,15 +1225,15 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac System.arraycopy(Serializer.serialize(MAX_TIMESTAMP_VALUE),0,txBytes,TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_OFFSET, TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_SIZE); - if (!instance.configuration.isPoWDisabled()) { - if (!miner.mine(txBytes, minWeightMagnitude, 4)) { - transactionViewModels.clear(); - break; - } - } + if (!configuration.isPoWDisabled()) { + if (!miner.mine(txBytes, minWeightMagnitude, 4)) { + transactionViewModels.clear(); + break; + } + } //validate PoW - throws exception if invalid - final TransactionViewModel transactionViewModel = instance.transactionValidator.validateBytes(txBytes, instance.transactionValidator.getMinWeightMagnitude()); + final TransactionViewModel transactionViewModel = transactionValidator.validateBytes(txBytes, transactionValidator.getMinWeightMagnitude()); transactionViewModels.add(transactionViewModel); prevTransaction = transactionViewModel.getHash(); @@ -1491,6 +1258,41 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac return elements; } + /** + * Can be 0 or more, and is set to 0 every 100 requests. + * Each increase indicates another 2 tips send. + * + * @return The current amount of times this node has returned transactions to approve + */ + private static int getCounterGetTxToApprove() { + return counterGetTxToApprove; + } + + /** + * Increases the amount of tips send for transactions to approve by one + */ + private static void incCounterGetTxToApprove() { + counterGetTxToApprove++; + } + + /** + * Can be 0 or more, and is set to 0 every 100 requests. + * + * @return The current amount of time spent on sending transactions to approve in milliseconds + */ + private static long getEllapsedTimeGetTxToApprove() { + return ellapsedTime_getTxToApprove; + } + + /** + * Increases the current amount of time spent on sending transactions to approve + * + * @param ellapsedTime the time to add, in milliseconds + */ + private static void incEllapsedTimeGetTxToApprove(long ellapsedTime) { + ellapsedTime_getTxToApprove += ellapsedTime; + } + /** * Transforms an object parameter into an int. * @@ -1501,7 +1303,7 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac */ private int getParameterAsInt(Map request, String paramName) throws ValidationException { validateParamExists(request, paramName); - final int result; + int result; try { result = ((Double) request.get(paramName)).intValue(); } catch (ClassCastException e) { @@ -1523,7 +1325,7 @@ private int getParameterAsInt(Map request, String paramName) thr private String getParameterAsStringAndValidate(Map request, String paramName, int size) throws ValidationException { validateParamExists(request, paramName); String result = (String) request.get(paramName); - //validateBytes(paramName, size, result); + validateHex(paramName, size, result); return result; } @@ -1535,14 +1337,14 @@ private String getParameterAsStringAndValidate(Map request, Stri */ private void validateParamExists(Map request, String paramName) throws ValidationException { if (!request.containsKey(paramName)) { - throw new ValidationException(invalidParams); + throw new ValidationException(INVALID_PARAMS); } } /** * Translates the parameter into a {@link List}. * We then validate if the amount of elements does not exceed the maximum allowed. - * Afterwards we verify if each element is valid according to {@link #validateBytes(String, int, String)}. + * Afterwards we verify if each element is valid according to {@link #validateHex(String, int, String)}. * * @param request All request parameters * @param paramName The name of the parameter we want to turn into a list of Strings @@ -1556,16 +1358,14 @@ private List getParameterAsList(Map request, String para validateParamExists(request, paramName); final List paramList = (List) request.get(paramName); if (paramList.size() > maxRequestList) { - throw new ValidationException(overMaxErrorMessage); + throw new ValidationException(OVER_MAX_ERROR_MESSAGE); } - /* if (size > 0) { //validate for (final String param : paramList) { - validateBytes(paramName, size, param); + validateHex(paramName, size, param); } - }*/ - + } return paramList; } @@ -1578,79 +1378,36 @@ private List getParameterAsList(Map request, String para * @param result The String we validate. * @throws ValidationException If the string is not exactly bytes of size length */ - private void validateBytes(String paramName, int size, String result) throws ValidationException { - if (!validBytes(result,size,ZERO_LENGTH_NOT_ALLOWED)) { + private void validateHex(String paramName, int size, String result) throws ValidationException { + if (!validHex(result,size,ZERO_LENGTH_NOT_ALLOWED)) { throw new ValidationException("Invalid " + paramName + " input"); } } /** - * Checks if a string is of a certain length, and contains exactly size amount of bytes. - * Our byte strings only contain a-f and 0-9. + * Checks if a string is of a certain length, and contains exactly size amount of hex. + * Input string may only contain a-f and 0-9. * - * @param bytes The bytes we validate. - * @param length The amount of bytes it should contain. + * @param input The input we validate. + * @param length The amount of hex it should contain. * @param zeroAllowed If set to '{@value #ZERO_LENGTH_ALLOWED}', an empty byte string is also valid. - * @throws ValidationException If the string is not exactly bytes of size length + * @throws ValidationException If the string is not exactly hex of size length * @return true if the string is valid, otherwise false */ - private boolean validBytes(String bytes, int length, char zeroAllowed) { - if (bytes.length() == 0 && zeroAllowed == ZERO_LENGTH_ALLOWED) { + private boolean validHex(String input, int length, char zeroAllowed) { + if (input.length() == 0 && zeroAllowed == ZERO_LENGTH_ALLOWED) { return true; } - if (bytes.length() != length) { + if (input.length() != length*2) { return false; } - Matcher matcher = hexPattern.matcher(bytes); + Matcher matcher = hexPattern.matcher(input); return matcher.matches(); } - /** - * Updates the {@link HttpServerExchange} {@link HeaderMap} with the proper response settings. - * @param exchange Contains information about what the client has send to us - */ - private static void setupResponseHeaders(final HttpServerExchange exchange) { - final HeaderMap headerMap = exchange.getResponseHeaders(); - headerMap.add(new HttpString("Access-Control-Allow-Origin"),"*"); - headerMap.add(new HttpString("Keep-Alive"), "timeout=500, max=100"); - } - - /** - * Sets up the {@link HttpHandler} to have correct security settings. - * Remote authentication is blocked for anyone except - * those defined in {@link APIConfig#getRemoteAuth()} or localhost. - * This is done with {@link BasicAuthenticationMechanism} in a {@link AuthenticationMode#PRO_ACTIVE} mode. - * - * @param toWrap the path handler used in creating the server. - * @return The updated handler - * - * TODO: Will become deprecated soon! - */ - private HttpHandler addSecurity(final HttpHandler toWrap) { - return toWrap; - /* - String credentials = instance.configuration.getRemoteAuth(); - if (credentials == null || credentials.isEmpty()) { - return toWrap; - } - - final Map users = new HashMap<>(2); - users.put(credentials.split(":")[0], credentials.split(":")[1].toCharArray()); - - IdentityManager identityManager = new MapIdentityManager(users); - HttpHandler handler = toWrap; - handler = new AuthenticationCallHandler(handler); - handler = new AuthenticationConstraintHandler(handler); - final List mechanisms = Collections.singletonList(new BasicAuthenticationMechanism("Helix Realm")); - handler = new AuthenticationMechanismsHandler(handler, mechanisms); - handler = new SecurityInitialHandler(AuthenticationMode.PRO_ACTIVE, identityManager, handler); - return handler; - */ - } - public void shutDown() { - if (server != null) { - server.stop(); + if (connector != null) { + connector.stop(); } } @@ -1730,10 +1487,10 @@ private void attachStoreAndBroadcast(final String address, final String message, public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign) throws Exception { // get tips - int latestMilestoneIndex = instance.latestMilestoneTracker.getLatestMilestoneIndex(); + int latestMilestoneIndex = latestMilestoneTracker.getLatestMilestoneIndex(); long nextIndex = latestMilestoneIndex+1; List txToApprove = new ArrayList<>(); - if(Hash.NULL_HASH.equals(instance.latestMilestoneTracker.getLatestMilestoneHash())) { + if(Hash.NULL_HASH.equals(latestMilestoneTracker.getLatestMilestoneHash())) { txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); } else { @@ -1796,4 +1553,187 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri broadcastTransactionsStatement(powResult); } -} + // + // FUNCTIONAL COMMAND ROUTES + // + private Function, AbstractResponse> addNeighbors() { + return request -> { + List uris = getParameterAsList(request,"uris",0); + log.debug("Invoking 'addNeighbors' with {}", uris); + return addNeighborsStatement(uris); + }; + } + + private Function, AbstractResponse> attachToTangle() { + return request -> { + final Hash trunkTransaction = HashFactory.TRANSACTION.create(getParameterAsStringAndValidate(request,"trunkTransaction", HASH_SIZE)); + final Hash branchTransaction = HashFactory.TRANSACTION.create(getParameterAsStringAndValidate(request,"branchTransaction", HASH_SIZE)); + final int minWeightMagnitude = getParameterAsInt(request,"minWeightMagnitude"); + + final List hbytes = getParameterAsList(request,"hbytes", BYTES_SIZE); + + List elements = attachToTangleStatement(trunkTransaction, branchTransaction, minWeightMagnitude, hbytes); + return AttachToTangleResponse.create(elements); + }; + } + + private Function, AbstractResponse> broadcastTransactions() { + return request -> { + final List hbytes = getParameterAsList(request,"hbytes", BYTES_SIZE); + broadcastTransactionsStatement(hbytes); + return AbstractResponse.createEmptyResponse(); + }; + } + + private Function, AbstractResponse> findTransactions() { + return request -> { + try { + return findTransactionsStatement(request); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> getBalances() { + return request -> { + final List addresses = getParameterAsList(request,"addresses", HASH_SIZE); + final List tips = request.containsKey("tips") ? + getParameterAsList(request,"tips", HASH_SIZE): + null; + final int threshold = getParameterAsInt(request, "threshold"); + + try { + return getBalancesStatement(addresses, tips, threshold); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> getInclusionStates() { + return request -> { + if (invalidSubtangleStatus()) { + return ErrorResponse.create(INVALID_SUBTANGLE); + } + final List transactions = getParameterAsList(request, "transactions", HASH_SIZE); + final List tips = getParameterAsList(request, "tips", HASH_SIZE); + + try { + return getInclusionStatesStatement(transactions, tips); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> getNeighbors() { + return request -> getNeighborsStatement(); + } + + private Function, AbstractResponse> getNodeInfo() { + return request -> { + try { + return getNodeInfoStatement(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> getNodeAPIConfiguration() { + return request -> getNodeAPIConfigurationStatement(); + } + + private Function, AbstractResponse> getTips() { + return request -> { + try { + return getTipsStatement(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> getTransactionsToApprove() { + return request -> { + Optional reference = request.containsKey("reference") ? + Optional.of(HashFactory.TRANSACTION.create(getParameterAsStringAndValidate(request,"reference", HASH_SIZE))) + : Optional.empty(); + int depth = getParameterAsInt(request, "depth"); + + return getTransactionsToApproveStatement(depth, reference); + }; + } + + private Function, AbstractResponse> getHBytes() { + return request -> { + final List hashes = getParameterAsList(request,"hashes", HASH_SIZE); + try { + return getHBytesStatement(hashes); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> interruptAttachingToTangle() { + return request -> interruptAttachingToTangleStatement(); + } + + private Function, AbstractResponse> removeNeighbors() { + return request -> { + List uris = getParameterAsList(request,"uris",0); + log.debug("Invoking 'removeNeighbors' with {}", uris); + return removeNeighborsStatement(uris); + }; + } + + private Function, AbstractResponse> storeTransactions() { + return request -> { + try { + final List hbytes = getParameterAsList(request,"hbytes", BYTES_SIZE); + storeTransactionsStatement(hbytes); + } catch (Exception e) { + //transaction not valid + return ErrorResponse.create("Invalid bytes input"); + } + return AbstractResponse.createEmptyResponse(); + }; + } + + private Function, AbstractResponse> getMissingTransactions() { + return request -> { + synchronized (transactionRequester) { + List missingTx = Arrays.stream(transactionRequester.getRequestedTransactions()) + .map(Hash::toString) + .collect(Collectors.toList()); + return GetTipsResponse.create(missingTx); + } + }; + } + + private Function, AbstractResponse> checkConsistency() { + return request -> { + if (invalidSubtangleStatus()) { + return ErrorResponse.create(INVALID_SUBTANGLE); + } + final List transactions = getParameterAsList(request,"tails", HASH_SIZE); + try { + return checkConsistencyStatement(transactions); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + private Function, AbstractResponse> wereAddressesSpentFrom() { + return request -> { + final List addresses = getParameterAsList(request,"addresses", HASH_SIZE); + try { + return wereAddressesSpentFromStatement(addresses); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } +} \ No newline at end of file From 0ee07006ace34d232b34cd69e61e467ebdfe91da Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 27 May 2019 18:06:12 +0200 Subject: [PATCH 014/223] resolve2 --- src/main/java/net/helix/hlx/service/API.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 8e1254ce..f934c786 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -644,15 +644,15 @@ private AbstractResponse interruptAttachingToTangleStatement(){ **/ private AbstractResponse getNodeInfoStatement() throws Exception { String name = configuration.isTestnet() ? HLX.TESTNET_NAME : HLX.MAINNET_NAME; - MilestoneViewModel milestone = MilestoneViewModel.first(tangle); + RoundViewModel round = RoundViewModel.first(tangle); return GetNodeInfoResponse.create(name, HLX.VERSION, Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().freeMemory(), System.getProperty("java.version"), Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), - latestMilestoneTracker.getLatestMilestoneHash(), - latestMilestoneTracker.getLatestMilestoneIndex(), + latestMilestoneTracker.getLatestRoundHashes(), + latestMilestoneTracker.getLatestRoundIndex(), snapshotProvider.getLatestSnapshot().getHash(), snapshotProvider.getLatestSnapshot().getIndex(), @@ -1487,10 +1487,10 @@ private void attachStoreAndBroadcast(final String address, final String message, public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign) throws Exception { // get tips - int latestMilestoneIndex = latestMilestoneTracker.getLatestMilestoneIndex(); + int latestMilestoneIndex = latestMilestoneTracker.getLatestRoundIndex(); long nextIndex = latestMilestoneIndex+1; List txToApprove = new ArrayList<>(); - if(Hash.NULL_HASH.equals(latestMilestoneTracker.getLatestMilestoneHash())) { + if(Hash.NULL_HASH.equals(latestMilestoneTracker.getLatestRoundHashes())) { txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); } else { From 94c9807c4f3e9e182c7209a2aeee41940d55285e Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 27 May 2019 19:43:03 +0200 Subject: [PATCH 015/223] rename coordinatorAddress to trusteeAddress --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 6 ++++-- src/main/java/net/helix/hlx/conf/MilestoneConfig.java | 2 +- .../service/milestone/impl/LatestMilestoneTrackerImpl.java | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index f86a7282..1f141407 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -5,10 +5,12 @@ import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import net.helix.hlx.model.HashFactory; import org.apache.commons.lang3.ArrayUtils; import net.helix.hlx.HLX; import net.helix.hlx.utils.HelixUtils; +import net.helix.hlx.model.Hash; import java.net.InetAddress; import java.net.UnknownHostException; @@ -110,7 +112,7 @@ public abstract class BaseHelixConfig implements HelixConfig { //Milestone protected int msDelay = Defaults.MS_DELAY; protected int minDelay = Defaults.MS_MIN_DELAY; - protected String cooAddress = Defaults.COORDINATOR_ADDRESS; + protected Hash cooAddress = HashFactory.ADDRESS.create(Defaults.COORDINATOR_ADDRESS); public BaseHelixConfig() { //empty constructor @@ -698,7 +700,7 @@ protected void setCacheSizeBytes(int cacheSizeBytes) { } @Override - public String getCoordinator() { + public Hash getTrusteeAddresses() { return cooAddress; } diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index a518835d..ac9ef941 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -11,7 +11,7 @@ public interface MilestoneConfig extends Config { /** * @return Descriptions#VALIDATOR_ADDRESSES */ - Set getValidatorAddresses(); + Hash getTrusteeAddresses(); /** * @return {@value Descriptions#DONT_VALIDATE_TESTNET_MILESTONE_SIG} */ diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index be3fd298..7ab330be 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -140,7 +140,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; - validatorAddresses = config.getValidatorAddresses(); + //validatorAddresses = config.getValidatorAddresses(); bootstrapLatestMilestoneValue(); From ff9ac39937c765ca5cb67d46d024fd4d39bed2d0 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 27 May 2019 20:04:39 +0200 Subject: [PATCH 016/223] update SnapshotService --- .../snapshot/impl/SnapshotServiceImpl.java | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 36241dd1..9af5232b 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -123,27 +123,27 @@ public SnapshotServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider * anything unexpected happens (creating a backup of the state requires a lot of memory).
*/ @Override - public void replayMilestones(Snapshot snapshot, int targetMilestoneIndex) throws SnapshotException { + public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws SnapshotException { Map balanceChanges = new HashMap<>(); Set skippedMilestones = new HashSet<>(); RoundViewModel lastAppliedMilestone = null; try { - for (int currentMilestoneIndex = snapshot.getIndex() + 1; currentMilestoneIndex <= targetMilestoneIndex; - currentMilestoneIndex++) { + for (int currentRoundIndex = snapshot.getIndex() + 1; currentRoundIndex <= targetRoundIndex; + currentRoundIndex++) { - RoundViewModel currentMilestone = RoundViewModel.get(tangle, currentMilestoneIndex); - if (currentMilestone != null) { - StateDiffViewModel stateDiffViewModel = StateDiffViewModel.load(tangle, currentMilestone.getHash()); + RoundViewModel currentRound = RoundViewModel.get(tangle, currentRoundIndex); + if (currentRound != null) { + StateDiffViewModel stateDiffViewModel = StateDiffViewModel.load(tangle, currentRoundIndex); if(!stateDiffViewModel.isEmpty()) { stateDiffViewModel.getDiff().forEach((address, change) -> { balanceChanges.compute(address, (k, balance) -> (balance == null ? 0 : balance) + change); }); } - lastAppliedMilestone = currentMilestone; + lastAppliedMilestone = currentRound; } else { - skippedMilestones.add(currentMilestoneIndex); + skippedMilestones.add(currentRoundIndex); } } @@ -154,13 +154,16 @@ public void replayMilestones(Snapshot snapshot, int targetMilestoneIndex) throws snapshot.applyStateDiff(new SnapshotStateDiffImpl(balanceChanges)); snapshot.setIndex(lastAppliedMilestone.index()); - snapshot.setHash(lastAppliedMilestone.getHash()); - TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, + //TODO: Store all milestone hashes in snapshot or merkle root? + //snapshot.setHash(lastAppliedMilestone.getHash()); + + //TODO: From where we should get the timestamp of the round/snapshot? + /*TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, lastAppliedMilestone.getHash()); if(milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT) { snapshot.setTimestamp(milestoneTransaction.getTimestamp()); - } + }*/ for (int skippedMilestoneIndex : skippedMilestones) { snapshot.addSkippedMilestone(skippedMilestoneIndex); @@ -299,8 +302,8 @@ public Map generateSeenMilestones(LatestMilestoneTracker latestMi Map seenMilestones = new HashMap<>(); try { RoundViewModel seenMilestone = targetMilestone; - while ((seenMilestone = RoundViewModel.findClosestNextMilestone(tangle, seenMilestone.index(), - latestMilestoneTracker.getLatestMilestoneIndex())) != null) { + while ((seenMilestone = RoundViewModel.findClosestNextRound(tangle, seenMilestone.index(), + latestMilestoneTracker.getLatestRoundIndex())) != null) { seenMilestones.put(seenMilestone.getHash(), seenMilestone.index()); @@ -337,7 +340,7 @@ private boolean rollbackLastMilestone(Tangle tangle, Snapshot snapshot) throws S try { // revert the last balance changes - StateDiffViewModel stateDiffViewModel = StateDiffViewModel.load(tangle, snapshot.getHash()); + StateDiffViewModel stateDiffViewModel = StateDiffViewModel.load(tangle, snapshot.getIndex()); if (!stateDiffViewModel.isEmpty()) { SnapshotStateDiffImpl snapshotStateDiff = new SnapshotStateDiffImpl( stateDiffViewModel.getDiff().entrySet().stream().map( @@ -378,8 +381,10 @@ private boolean rollbackLastMilestone(Tangle tangle, Snapshot snapshot) throws S // otherwise set metadata of the previous milestone RoundViewModel currentMilestone = RoundViewModel.get(tangle, currentIndex); snapshot.setIndex(currentMilestone.index()); - snapshot.setHash(currentMilestone.getHash()); - snapshot.setTimestamp(TransactionViewModel.fromHash(tangle, currentMilestone.getHash()).getTimestamp()); + + //TODO: see above + //snapshot.setHash(currentMilestone.getHash()); + //snapshot.setTimestamp(TransactionViewModel.fromHash(tangle, currentMilestone.getHash()).getTimestamp()); return true; } catch (Exception e) { @@ -408,7 +413,7 @@ private RoundViewModel determineMilestoneForLocalSnapshot(Tangle tangle, Snapsho RoundViewModel targetMilestone; try { - targetMilestone = RoundViewModel.findClosestPrevMilestone(tangle, targetMilestoneIndex, + targetMilestone = RoundViewModel.findClosestPrevRound(tangle, targetMilestoneIndex, snapshotProvider.getInitialSnapshot().getIndex()); } catch (Exception e) { throw new SnapshotException("could not load the target milestone", e); @@ -680,7 +685,7 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot solidEntryPoints.put(currentMilestone.getHash(), targetMilestone.index()); - nextMilestone = RoundViewModel.findClosestPrevMilestone(tangle, currentMilestone.index(), + nextMilestone = RoundViewModel.findClosestPrevRound(tangle, currentMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex()); progressLogger.progress(); From 2fc1f42ff2a99030fd696ab7f933c2cde59b3f58 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 29 May 2019 12:23:19 +0200 Subject: [PATCH 017/223] update SnapshotService and fix related problems with seenMilestones (seenRounds) --- .../service/snapshot/SnapshotMetaData.java | 7 +-- .../hlx/service/snapshot/SnapshotService.java | 7 +-- .../service/snapshot/impl/SnapshotImpl.java | 9 ++-- .../snapshot/impl/SnapshotMetaDataImpl.java | 28 +++++----- .../snapshot/impl/SnapshotServiceImpl.java | 53 +++++++++---------- 5 files changed, 54 insertions(+), 50 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java b/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java index 9d833292..6a9228b8 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java +++ b/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java @@ -3,6 +3,7 @@ import net.helix.hlx.model.Hash; import java.util.Map; +import java.util.List; /** * Represents the meta data of a snapshot. @@ -189,7 +190,7 @@ public interface SnapshotMetaData { * * @return map of milestone transaction hashes associated to their milestone index */ - Map getSeenMilestones(); + List getSeenRounds(); /** * Setter for the seen milestones. @@ -199,9 +200,9 @@ public interface SnapshotMetaData { * start requesting the missing milestones without having to discover them one by one, which massively * increases the sync speed of new nodes. * - * @param seenMilestones map of milestone transaction hashes associated to their milestone index + * @param seenRounds map of milestone transaction hashes associated to their milestone index */ - void setSeenMilestones(Map seenMilestones); + void setSeenRounds(List seenRounds); /** * Replaces the meta data values of this instance with the values of another meta data object. diff --git a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java index 6158fea5..3f8e1476 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java +++ b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java @@ -6,6 +6,7 @@ import net.helix.hlx.service.transactionpruning.TransactionPruner; import java.util.Map; +import java.util.List; /** * Represents the service for snapshots that contains the relevant business logic for modifying {@link Snapshot}s and @@ -102,10 +103,10 @@ Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundVi * very first time. * * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones - * @param targetMilestone milestone that is used as a reference point for the snapshot + * @param targetRound milestone that is used as a reference point for the snapshot * @return a map of solid entry points associating their hash to the milestone index that confirmed them * @throws SnapshotException if anything goes wrong while generating the solid entry points */ - Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - RoundViewModel targetMilestone) throws SnapshotException; + List generateSeenRounds(LatestMilestoneTracker latestMilestoneTracker, + RoundViewModel targetRound) throws SnapshotException; } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java index 9f4120a2..16988b65 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java @@ -13,6 +13,7 @@ import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.List; /** * Implements the basic contract of the {@link Snapshot} interface. @@ -421,11 +422,11 @@ public int getSolidEntryPointIndex(Hash solidEntrypoint) { * This is a thread-safe wrapper for the underlying {@link SnapshotMetaData} method. */ @Override - public Map getSeenMilestones() { + public List getSeenRounds() { lockRead(); try { - return metaData.getSeenMilestones(); + return metaData.getSeenRounds(); } finally { unlockRead(); } @@ -437,11 +438,11 @@ public Map getSeenMilestones() { * This is a thread-safe wrapper for the underlying {@link SnapshotMetaData} method. */ @Override - public void setSeenMilestones(Map seenMilestones) { + public void setSeenRounds(List seenRounds) { lockWrite(); try { - metaData.setSeenMilestones(seenMilestones); + metaData.setSeenRounds(seenRounds); } finally { unlockWrite(); } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java index a33b6eab..209e343d 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java @@ -6,6 +6,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.LinkedList; +import java.util.List; /** * Implements the basic contract of the {@link SnapshotMetaData} interface. @@ -47,9 +49,9 @@ public class SnapshotMetaDataImpl implements SnapshotMetaData { private Map solidEntryPoints; /** - * Internal property for the value returned by {@link SnapshotMetaData#getSeenMilestones()}. + * Internal property for the value returned by {@link SnapshotMetaData#getSeenRounds()}. */ - private Map seenMilestones; + private List seenRounds; /** * Creates a meta data object with the given information. @@ -61,10 +63,10 @@ public class SnapshotMetaDataImpl implements SnapshotMetaData { * @param timestamp timestamp of the transaction that the snapshot belongs to * @param solidEntryPoints map with the transaction hashes of the solid entry points associated to their milestone * index - * @param seenMilestones map of milestone transaction hashes associated to their milestone index + * @param seenRounds map of milestone transaction hashes associated to their milestone index */ public SnapshotMetaDataImpl(Hash hash, int index, Long timestamp, Map solidEntryPoints, - Map seenMilestones) { + List seenRounds) { this.initialHash = hash; this.initialIndex = index; @@ -74,7 +76,7 @@ public SnapshotMetaDataImpl(Hash hash, int index, Long timestamp, Map(solidEntryPoints)); - setSeenMilestones(new HashMap<>(seenMilestones)); + setSeenRounds(new LinkedList<>(seenRounds)); } /** @@ -85,7 +87,7 @@ public SnapshotMetaDataImpl(Hash hash, int index, Long timestamp, Map getSeenMilestones() { - return seenMilestones; + public List getSeenRounds() { + return seenRounds; } /** * {@inheritDoc} */ @Override - public void setSeenMilestones(Map seenMilestones) { - this.seenMilestones = seenMilestones; + public void setSeenRounds(List seenRounds) { + this.seenRounds = seenRounds; } /** @@ -249,13 +251,13 @@ public void update(SnapshotMetaData newMetaData) { setHash(newMetaData.getHash()); setTimestamp(newMetaData.getTimestamp()); setSolidEntryPoints(new HashMap<>(newMetaData.getSolidEntryPoints())); - setSeenMilestones(new HashMap<>(newMetaData.getSeenMilestones())); + setSeenRounds(new LinkedList<>(newMetaData.getSeenRounds())); } @Override public int hashCode() { return Objects.hash(getClass(), initialHash, initialIndex, initialTimestamp, hash, index, timestamp, - solidEntryPoints, seenMilestones); + solidEntryPoints, seenRounds); } @Override @@ -275,7 +277,7 @@ public boolean equals(Object obj) { Objects.equals(index, ((SnapshotMetaDataImpl) obj).index) && Objects.equals(timestamp, ((SnapshotMetaDataImpl) obj).timestamp) && Objects.equals(solidEntryPoints, ((SnapshotMetaDataImpl) obj).solidEntryPoints) && - Objects.equals(seenMilestones, ((SnapshotMetaDataImpl) obj).seenMilestones); + Objects.equals(seenRounds, ((SnapshotMetaDataImpl) obj).seenRounds); } } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 9af5232b..98968b86 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -20,10 +20,7 @@ import net.helix.hlx.utils.log.ProgressLogger; import net.helix.hlx.utils.log.interval.IntervalProgressLogger; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -233,15 +230,15 @@ public void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, Tra * {@inheritDoc} */ @Override - public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundViewModel targetMilestone) + public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundViewModel targetRound) throws SnapshotException { - if (targetMilestone == null) { + if (targetRound == null) { throw new SnapshotException("the target milestone must not be null"); - } else if (targetMilestone.index() > snapshotProvider.getLatestSnapshot().getIndex()) { - throw new SnapshotException("the snapshot target " + targetMilestone + " was not solidified yet"); - } else if (targetMilestone.index() < snapshotProvider.getInitialSnapshot().getIndex()) { - throw new SnapshotException("the snapshot target " + targetMilestone + " is too old"); + } else if (targetRound.index() > snapshotProvider.getLatestSnapshot().getIndex()) { + throw new SnapshotException("the snapshot target " + targetRound + " was not solidified yet"); + } else if (targetRound.index() < snapshotProvider.getInitialSnapshot().getIndex()) { + throw new SnapshotException("the snapshot target " + targetRound + " is too old"); } snapshotProvider.getInitialSnapshot().lockRead(); @@ -250,26 +247,26 @@ public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, Snapshot snapshot; try { int distanceFromInitialSnapshot = Math.abs(snapshotProvider.getInitialSnapshot().getIndex() - - targetMilestone.index()); + targetRound.index()); int distanceFromLatestSnapshot = Math.abs(snapshotProvider.getLatestSnapshot().getIndex() - - targetMilestone.index()); + targetRound.index()); if (distanceFromInitialSnapshot <= distanceFromLatestSnapshot) { snapshot = snapshotProvider.getInitialSnapshot().clone(); - replayMilestones(snapshot, targetMilestone.index()); + replayMilestones(snapshot, targetRound.index()); } else { snapshot = snapshotProvider.getLatestSnapshot().clone(); - rollBackMilestones(snapshot, targetMilestone.index() + 1); + rollBackMilestones(snapshot, targetRound.index() + 1); } } finally { snapshotProvider.getInitialSnapshot().unlockRead(); snapshotProvider.getLatestSnapshot().unlockRead(); } - snapshot.setSolidEntryPoints(generateSolidEntryPoints(targetMilestone)); - snapshot.setSeenMilestones(generateSeenMilestones(latestMilestoneTracker, targetMilestone)); + snapshot.setSolidEntryPoints(generateSolidEntryPoints(targetRound)); + snapshot.setSeenRounds(generateSeenRounds(latestMilestoneTracker, targetRound)); return snapshot; } @@ -292,20 +289,20 @@ public Map generateSolidEntryPoints(RoundViewModel targetMileston * {@inheritDoc} */ @Override - public Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - RoundViewModel targetMilestone) throws SnapshotException { + public List generateSeenRounds(LatestMilestoneTracker latestMilestoneTracker, + RoundViewModel targetRound) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [processing seen milestones]", log) .start(config.getLocalSnapshotsDepth()); - Map seenMilestones = new HashMap<>(); + List seenRounds = new LinkedList<>(); try { - RoundViewModel seenMilestone = targetMilestone; - while ((seenMilestone = RoundViewModel.findClosestNextRound(tangle, seenMilestone.index(), + RoundViewModel seenRound = targetRound; + while ((seenRound = RoundViewModel.findClosestNextRound(tangle, seenRound.index(), latestMilestoneTracker.getLatestRoundIndex())) != null) { - seenMilestones.put(seenMilestone.getHash(), seenMilestone.index()); + seenRounds.add(seenRound.index()); progressLogger.progress(); } @@ -317,7 +314,7 @@ public Map generateSeenMilestones(LatestMilestoneTracker latestMi progressLogger.finish(); - return seenMilestones; + return seenRounds; } /** @@ -580,17 +577,17 @@ private boolean isOrphaned(Tangle tangle, TransactionViewModel transaction, * * @param tangle Tangle object which acts as a database interface * @param transactionHash hash of the transaction that shall be checked - * @param targetMilestone milestone that is used as an anchor for our checks + * @param targetRound milestone that is used as an anchor for our checks * @return true if the transaction is a solid entry point and false otherwise */ - private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundViewModel targetMilestone) { + private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundViewModel targetRound) { Set unconfirmedApprovers = new HashSet<>(); try { for (Hash approverHash : ApproveeViewModel.load(tangle, transactionHash).getHashes()) { TransactionViewModel approver = TransactionViewModel.fromHash(tangle, approverHash); - if (approver.snapshotIndex() > targetMilestone.index()) { + if (approver.snapshotIndex() > targetRound.index()) { return true; } else if (approver.snapshotIndex() == 0) { unconfirmedApprovers.add(approver); @@ -598,7 +595,8 @@ private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundView } Set processedTransactions = new HashSet<>(); - TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, targetMilestone.getHash()); + //TODO: which milestone should we take as reference transaction for isOrphaned() + TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, targetRound.getHash()); for (TransactionViewModel unconfirmedApprover : unconfirmedApprovers) { if (!isOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) { return true; @@ -673,6 +671,7 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot progressLogger.getCurrentStep() < progressLogger.getStepCount()) { RoundViewModel currentMilestone = nextMilestone; + //TODO: reimplement traverseApprovees(), does it make sense to use several entry points when traversing? DAGHelper.get(tangle).traverseApprovees( currentMilestone.getHash(), currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(), From 07e7c2751210d583b4e3d5d5c8fe76fbb7c9b57e Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 29 May 2019 12:25:45 +0200 Subject: [PATCH 018/223] rename getLatestRoundIndex() in LocalSnapshotManager --- .../hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java index 00ff9b60..e865e4be 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java @@ -118,7 +118,7 @@ public void shutdown() { private void monitorThread(LatestMilestoneTracker latestMilestoneTracker) { while (!Thread.currentThread().isInterrupted()) { int localSnapshotInterval = latestMilestoneTracker.isInitialScanComplete() && - snapshotProvider.getLatestSnapshot().getIndex() == latestMilestoneTracker.getLatestMilestoneIndex() + snapshotProvider.getLatestSnapshot().getIndex() == latestMilestoneTracker.getLatestRoundIndex() ? config.getLocalSnapshotsIntervalSynced() : config.getLocalSnapshotsIntervalUnsynced(); From f44881304deedbcc511654645439c9e2d484e6bd Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 29 May 2019 13:27:37 +0200 Subject: [PATCH 019/223] raw design and impl of ValidatorTracker --- .../net/helix/hlx/conf/BaseHelixConfig.java | 2 +- .../net/helix/hlx/conf/MilestoneConfig.java | 2 +- .../service/milestone/ValidatorTracker.java | 19 +++ .../milestone/impl/ValidatorTrackerImpl.java | 110 ++++++++++++++++++ 4 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java create mode 100644 src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 1f141407..57026670 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -700,7 +700,7 @@ protected void setCacheSizeBytes(int cacheSizeBytes) { } @Override - public Hash getTrusteeAddresses() { + public Hash getTrusteeAddress() { return cooAddress; } diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index ac9ef941..30e1b39a 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -11,7 +11,7 @@ public interface MilestoneConfig extends Config { /** * @return Descriptions#VALIDATOR_ADDRESSES */ - Hash getTrusteeAddresses(); + Hash getTrusteeAddress(); /** * @return {@value Descriptions#DONT_VALIDATE_TESTNET_MILESTONE_SIG} */ diff --git a/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java b/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java new file mode 100644 index 00000000..e1872f56 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java @@ -0,0 +1,19 @@ +package net.helix.hlx.service.milestone; + +import net.helix.hlx.model.Hash; + +public interface ValidatorTracker { + + boolean processTrusteeTransaction(Hash transactionHash) throws Exception; + + void updateValidatorAddresses() throws Exception; + + void analyzeTrusteeTransactions() throws Exception; + + void collectNewTrusteeTransactions() throws Exception; + + void start(); + + void shutdown(); + +} diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java new file mode 100644 index 00000000..27a997e6 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java @@ -0,0 +1,110 @@ +package net.helix.hlx.service.milestone.impl; + +import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.controllers.AddressViewModel; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.Hash; +import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.ValidatorTracker; +import net.helix.hlx.storage.Tangle; +import net.helix.hlx.utils.log.interval.IntervalLogger; +import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; +import net.helix.hlx.utils.thread.SilentScheduledExecutorService; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class ValidatorTrackerImpl implements ValidatorTracker { + + private static final int MAX_CANDIDATES_TO_ANALYZE = 5000; + private static final int RESCAN_INTERVAL = 1000; + private Hash Trustee_Address; + + private static final IntervalLogger log = new IntervalLogger(LatestMilestoneTrackerImpl.class); + private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Validator Tracker", log.delegate()); + + private Tangle tangle; + private LatestMilestoneTracker latestMilestoneTracker; + private Set validatorAddresses; + private final Set seenTrusteeTransactions = new HashSet<>(); + private final Deque trusteeTransactionsToAnalyze = new ArrayDeque<>(); + + private boolean firstRun = true; + + public ValidatorTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, HelixConfig config) { + + this.tangle = tangle; + this.latestMilestoneTracker = latestMilestoneTracker; + this.Trustee_Address = config.getTrusteeAddress(); + + return this; + } + + public boolean processTrusteeTransaction(Hash transactionHash) throws Exception { + TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); + try { + if (Trustee_Address.equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { + return true; + }else { + return false; + } + } catch (Exception e) { + throw new Exception("unexpected error while processing trustee transaction " + transaction, e); + } + } + + public void updateValidatorAddresses() throws Exception{ + + } + + public void analyzeTrusteeTransactions() throws Exception { + int transactionsToAnalyze = Math.min(trusteeTransactionsToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); + for (int i = 0; i < transactionsToAnalyze; i++) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Hash trusteeTransactionHash = trusteeTransactionsToAnalyze.pollFirst(); + if(!processTrusteeTransaction(trusteeTransactionHash)) { + seenTrusteeTransactions.remove(trusteeTransactionHash); + } + } + } + + public void collectNewTrusteeTransactions() throws Exception { + try { + for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { + if (Thread.currentThread().isInterrupted()) { + return; + } + if (seenTrusteeTransactions.add(hash)) { + trusteeTransactionsToAnalyze.addFirst(hash); + } + } + } catch (Exception e) { + throw new Exception("failed to collect the new trustee transactions", e); + } + } + + private void validatorTrackerThread() { + try { + collectNewTrusteeTransactions(); + analyzeTrusteeTransactions(); + } catch (Exception e) { + log.error("error while scaning for trustee transactions", e); + } + } + + public void start(){ + executorService.silentScheduleWithFixedDelay(this::validatorTrackerThread, 0, RESCAN_INTERVAL, + TimeUnit.MILLISECONDS); + } + + public void shutdown(){ + executorService.shutdownNow(); + } +} From f8f55b94eadf70976e545c6af3cb6f8a340ed8ab Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 5 Jun 2019 11:15:10 +0200 Subject: [PATCH 020/223] implement processTrusteeTransaction --- .../service/milestone/ValidatorTracker.java | 2 +- .../milestone/impl/ValidatorTrackerImpl.java | 44 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java b/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java index e1872f56..488d960a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java @@ -6,7 +6,7 @@ public interface ValidatorTracker { boolean processTrusteeTransaction(Hash transactionHash) throws Exception; - void updateValidatorAddresses() throws Exception; + void updateValidatorAddresses(Hash transaction, int roundIndex) throws Exception; void analyzeTrusteeTransactions() throws Exception; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java index 27a997e6..7c6d8724 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java @@ -3,9 +3,13 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.AddressViewModel; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneService; +import net.helix.hlx.service.milestone.MilestoneSolidifier; import net.helix.hlx.service.milestone.ValidatorTracker; +import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; @@ -29,6 +33,9 @@ public class ValidatorTrackerImpl implements ValidatorTracker { private Tangle tangle; private LatestMilestoneTracker latestMilestoneTracker; + private SnapshotProvider snapshotProvider; + private MilestoneService milestoneService; + private MilestoneSolidifier milestoneSolidifier; private Set validatorAddresses; private final Set seenTrusteeTransactions = new HashSet<>(); private final Deque trusteeTransactionsToAnalyze = new ArrayDeque<>(); @@ -44,11 +51,39 @@ public ValidatorTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMil return this; } + @Override public boolean processTrusteeTransaction(Hash transactionHash) throws Exception { TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); try { - if (Trustee_Address.equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { - return true; + if (Trustee_Address.equals(transaction.getAddressHash())) { + + } + int roundIndex = milestoneService.getRoundIndex(transaction); + + // if the trustee transaction is older than our ledger start point: we already processed it in the past + if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { + return true; + } + + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { + case VALID: + if (roundIndex > latestMilestoneTracker.getLatestRoundIndex()) { + updateValidatorAddresses(transaction.getHash(), roundIndex); + } + + if (!transaction.isSolid()) { + milestoneSolidifier.add(transaction.getHash(), roundIndex); + } + + transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); + break; + + case INCOMPLETE: + milestoneSolidifier.add(transaction.getHash(), roundIndex); + + transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); + + return false; }else { return false; } @@ -57,10 +92,12 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception } } - public void updateValidatorAddresses() throws Exception{ + @Override + public void updateValidatorAddresses(Hash transaction, int roundIndex) throws Exception{ } + @Override public void analyzeTrusteeTransactions() throws Exception { int transactionsToAnalyze = Math.min(trusteeTransactionsToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); for (int i = 0; i < transactionsToAnalyze; i++) { @@ -75,6 +112,7 @@ public void analyzeTrusteeTransactions() throws Exception { } } + @Override public void collectNewTrusteeTransactions() throws Exception { try { for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { From 936efaca4328d20f0162315e0ae6cfe468cec115 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 5 Jun 2019 11:18:33 +0200 Subject: [PATCH 021/223] proof if tx is orphaned from the perspective of one of the confirmed tips --- .../service/snapshot/impl/SnapshotServiceImpl.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 98968b86..21fc9191 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -595,11 +595,14 @@ private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundView } Set processedTransactions = new HashSet<>(); - //TODO: which milestone should we take as reference transaction for isOrphaned() - TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, targetRound.getHash()); for (TransactionViewModel unconfirmedApprover : unconfirmedApprovers) { - if (!isOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) { - return true; + // TODO: need access from milestoneService + // if one of the unconfirmed approvers isn't orphaned from the perspective of one of the confirmed tips, the transaction is a solid entry point + for (Hash milestoneHash : milestoneService.getConfirmedTips(targetRound.index(), quorum)) { + TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); + if (!isOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) { + return true; + } } } } catch (Exception e) { From 063d0e0977a74233435b59290878e616195f73c2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 13 Jun 2019 16:40:38 +0200 Subject: [PATCH 022/223] implement processTrusteeTransaction and updateValidatorAddresses --- .../milestone/impl/ValidatorTrackerImpl.java | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java index 7c6d8724..b3d67278 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java @@ -2,23 +2,23 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.AddressViewModel; +import net.helix.hlx.controllers.BundleViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.milestone.MilestoneService; import net.helix.hlx.service.milestone.MilestoneSolidifier; import net.helix.hlx.service.milestone.ValidatorTracker; +import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; import net.helix.hlx.utils.thread.SilentScheduledExecutorService; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; public class ValidatorTrackerImpl implements ValidatorTracker { @@ -40,13 +40,15 @@ public class ValidatorTrackerImpl implements ValidatorTracker { private final Set seenTrusteeTransactions = new HashSet<>(); private final Deque trusteeTransactionsToAnalyze = new ArrayDeque<>(); - private boolean firstRun = true; - - public ValidatorTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, HelixConfig config) { + public ValidatorTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, HelixConfig config) { this.tangle = tangle; this.latestMilestoneTracker = latestMilestoneTracker; this.Trustee_Address = config.getTrusteeAddress(); + this.snapshotProvider = snapshotProvider; + this.milestoneService = milestoneService; + this.milestoneSolidifier = milestoneSolidifier; + //bootstrapLatestValidators(); return this; } @@ -57,7 +59,6 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception try { if (Trustee_Address.equals(transaction.getAddressHash())) { - } int roundIndex = milestoneService.getRoundIndex(transaction); // if the trustee transaction is older than our ledger start point: we already processed it in the past @@ -65,6 +66,7 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception return true; } + // we use milestone validation because its the same process (TODO: Rename function and implement more general) switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { case VALID: if (roundIndex > latestMilestoneTracker.getLatestRoundIndex()) { @@ -75,18 +77,19 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception milestoneSolidifier.add(transaction.getHash(), roundIndex); } - transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); break; case INCOMPLETE: milestoneSolidifier.add(transaction.getHash(), roundIndex); - transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); - return false; + + default: + } }else { return false; } + return true; } catch (Exception e) { throw new Exception("unexpected error while processing trustee transaction " + transaction, e); } @@ -94,7 +97,25 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception @Override public void updateValidatorAddresses(Hash transaction, int roundIndex) throws Exception{ - + TransactionViewModel tail = TransactionViewModel.fromHash(tangle, transaction); + BundleViewModel bundle = BundleViewModel.load(tangle, tail.getBundleHash()); + + for (Hash txHash : bundle.getHashes()) { + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, txHash); + // get first tx with validator addresses in signatureMessageFragment + // TODO: If we need more than one tx to send the validator addresses, how do we know how many tx containing validators + // -> we have to count from last index + // n to (n-security) : tx with signature + // n-security-1 : tx with merkle path + // 0 to (n-security-1) : tx with validators + if (tx.getCurrentIndex() == 0) { + for (int i = 0; i < TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE / Hash.SIZE_IN_BYTES; i++) { + Hash address = HashFactory.ADDRESS.create(Arrays.copyOfRange(tx.getSignature(), i * Hash.SIZE_IN_BYTES, (i+1) * Hash.SIZE_IN_BYTES)); + // TODO: Do we send all validators or only adding and removing ones + validatorAddresses.add(address); + } + } + } } @Override From f803d3158d429518191c9bb14b28cdcc20cf4be2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 13 Jun 2019 17:37:51 +0200 Subject: [PATCH 023/223] implement validation function for merkle signatures --- .../java/net/helix/hlx/crypto/Merkle.java | 32 +++++++++++++++++++ .../service/milestone/MilestoneService.java | 2 +- .../impl/LatestMilestoneTrackerImpl.java | 2 +- .../milestone/impl/MilestoneServiceImpl.java | 29 +++-------------- .../milestone/impl/ValidatorTrackerImpl.java | 2 +- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index 0b349fff..19e3b67c 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -1,6 +1,8 @@ package net.helix.hlx.crypto; +import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; import org.bouncycastle.util.encoders.Hex; import java.io.BufferedWriter; @@ -9,7 +11,9 @@ import java.io.FileWriter; import java.io.IOException; import java.io.File; +import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.List; public class Merkle { @@ -79,6 +83,34 @@ public static byte[][][] buildMerkleTree(String seed, int pubkeyDepth, int first return merkleTree; } + public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int index, int securityLevel, int depth) { + + final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); + + //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) + //TODO: check if its okay here to use bundle hash instead of tx hash + byte[] bundleHash = new byte[Sha3.HASH_LENGTH]; + Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes(), bundleHash); + + //validate leaf signature + ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * securityLevel); + + for (int i = 0; i < securityLevel; i++) { + byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i+1)); + byte[] digest = Winternitz.digest(mode, bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); + bb.put(digest); + } + + byte[] digests = bb.array(); + byte[] address = Winternitz.address(mode, digests); + + //validate Merkle path + byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, + merkleTx.getSignature(), 0, index, depth); + + return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); + } + public static byte[][][] readKeyfile(File keyfile, StringBuilder seedBuilder) throws IOException { try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { String[] fields = br.readLine().split(" "); diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index 4f3d319f..f96e7c13 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -41,7 +41,7 @@ public interface MilestoneService { * @throws MilestoneException if anything unexpected goes wrong while validating the milestone transaction */ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, int milestoneIndex, - SpongeFactory.Mode mode, int securityLevel) throws MilestoneException; + SpongeFactory.Mode mode, int securityLevel, Set validatorAddresses) throws MilestoneException; /** * Updates the milestone index of all transactions that belong to a milestone.
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 7ab330be..deb5ad03 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -205,7 +205,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw return true; } - switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validatorAddresses)) { case VALID: if (roundIndex > latestRoundIndex) { addMilestoneToLatestRound(transaction.getHash(), roundIndex); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index c9829c8c..166ad51e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -159,7 +159,7 @@ public void resetCorruptedRound(int index) throws MilestoneException { @Override public MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, int roundIndex, - SpongeFactory.Mode mode, int securityLevel) throws MilestoneException { + SpongeFactory.Mode mode, int securityLevel, Set validatorAddresses) throws MilestoneException { if (roundIndex < 0 || roundIndex >= 0x200000) { return INVALID; @@ -180,35 +180,14 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM for (final List bundleTransactionViewModels : bundleTransactions) { final TransactionViewModel tail = bundleTransactionViewModels.get(0); // milestone transaction with signature if (tail.getHash().equals(transactionViewModel.getHash())) { - //the signed transaction - which references the confirmed transactions and contains - // the Merkle tree siblings. - final TransactionViewModel siblingsTx = bundleTransactionViewModels.get(securityLevel); if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { - //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) - //TODO: check if its okay here to use bundle hash instead of tx hash - byte[] bundleHash = new byte[Sha3.HASH_LENGTH]; - Winternitz.normalizedBundle(siblingsTx.getBundleHash().bytes(), bundleHash); - - //validate leaf signature - ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * securityLevel); - - for (int i = 0; i < securityLevel; i++) { - byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i+1)); - byte[] digest = Winternitz.digest(mode, bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); - bb.put(digest); - } - - byte[] digests = bb.array(); - byte[] address = Winternitz.address(mode, digests); - - //validate Merkle path - byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, - siblingsTx.getSignature(), 0, roundIndex, config.getNumberOfKeysInMilestone()); + Hash senderAddress = tail.getAddressHash(); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || - (config.getValidatorAddresses().contains(HashFactory.ADDRESS.create(merkleRoot)))) { + (validatorAddresses.contains(senderAddress)) && validSignature) { RoundViewModel currentRoundViewModel = RoundViewModel.get(tangle, roundIndex); currentRoundViewModel.addMilestone(transactionViewModel.getHash()); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java index b3d67278..36627dca 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java @@ -67,7 +67,7 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception } // we use milestone validation because its the same process (TODO: Rename function and implement more general) - switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validatorAddresses)) { case VALID: if (roundIndex > latestMilestoneTracker.getLatestRoundIndex()) { updateValidatorAddresses(transaction.getHash(), roundIndex); From 0cb821ba1b9af57767d3f209900286018c466ed6 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 16 Jun 2019 14:53:22 +0200 Subject: [PATCH 024/223] sync new validator when latest round ends --- .../milestone/LatestMilestoneTracker.java | 2 + .../service/milestone/ValidatorTracker.java | 8 ++- .../impl/LatestMilestoneTrackerImpl.java | 7 +++ .../impl/LatestSolidMilestoneTrackerImpl.java | 14 ++++- .../milestone/impl/MilestoneServiceImpl.java | 1 + .../milestone/impl/ValidatorTrackerImpl.java | 54 +++++++++++++++---- 6 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index 9158a7ac..a739853e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -48,6 +48,8 @@ public interface LatestMilestoneTracker { */ void setLatestRoundIndex(int latestRoundIndex); + void setLatestValidators(Set validatorAddresses); + /** * Analyzes the given transaction to determine if it is a valid milestone.
*
diff --git a/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java b/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java index 488d960a..c8c7a472 100644 --- a/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java @@ -2,11 +2,17 @@ import net.helix.hlx.model.Hash; +import java.util.Set; + public interface ValidatorTracker { + public Set getLatestValidators(); + + Set getValidatorsOfRound(int roundIndex) throws Exception; + boolean processTrusteeTransaction(Hash transactionHash) throws Exception; - void updateValidatorAddresses(Hash transaction, int roundIndex) throws Exception; + Set getValidatorAddresses(Hash transaction) throws Exception; void analyzeTrusteeTransactions() throws Exception; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index deb5ad03..e3798fb1 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -167,6 +167,13 @@ public void setLatestRoundIndex(int roundIndex) { this.latestRoundIndex = roundIndex; } + @Override + public void setLatestValidators(Set validatorAddresses) { + tangle.publish("lv %d %d", this.latestRoundIndex, validatorAddresses); + log.delegate().info("Validators for round #{}: {}", this.latestRoundIndex, validatorAddresses); + this.validatorAddresses = validatorAddresses; + } + @Override public int getLatestRoundIndex() { return latestRoundIndex; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index becf2ac1..98b4a7fd 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -8,6 +8,7 @@ import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; import net.helix.hlx.service.milestone.LatestSolidMilestoneTracker; +import net.helix.hlx.service.milestone.ValidatorTracker; import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; @@ -16,6 +17,7 @@ import net.helix.hlx.utils.thread.SilentScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.Set; /** * Creates a manager that keeps track of the latest solid milestones and that triggers the application of these @@ -64,6 +66,8 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac */ private LedgerService ledgerService; + private ValidatorTracker validatorTracker; + /** * Holds a reference to the manager of the background worker.
*/ @@ -106,13 +110,14 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac */ public LatestSolidMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, LedgerService ledgerService, - LatestMilestoneTracker latestMilestoneTracker) { + LatestMilestoneTracker latestMilestoneTracker, ValidatorTracker validatorTracker) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.milestoneService = milestoneService; this.ledgerService = ledgerService; this.latestMilestoneTracker = latestMilestoneTracker; + this.validatorTracker = validatorTracker; return this; } @@ -142,6 +147,8 @@ public void trackLatestSolidMilestone() throws MilestoneException { RoundViewModel nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); if (nextRound != null) { // check solidity of milestones + // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? + // TODO: This solution is definitly wrong, we should continue even there are non solid milestones boolean allSolid = true; for (Hash milestoneHash : nextRound.getHashes()) { if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { @@ -149,6 +156,7 @@ public void trackLatestSolidMilestone() throws MilestoneException { } } while (!Thread.currentThread().isInterrupted() && allSolid) { + syncValidatorTracker(); syncLatestMilestoneTracker(nextRound.index()); applySolidMilestoneToLedger(nextRound); logChange(currentSolidRoundIndex); @@ -256,6 +264,10 @@ private void syncLatestMilestoneTracker(int roundIndex) { } } + private void syncValidatorTracker() { + latestMilestoneTracker.setLatestValidators(validatorTracker.getLatestValidators()); + } + /** * Emits a log message whenever the latest solid milestone changes.
*
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 166ad51e..e7171aea 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -186,6 +186,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); + // TODO: we should also check here if there is already a milestone with the same address if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java index 36627dca..5752b1c9 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java @@ -3,10 +3,13 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.AddressViewModel; import net.helix.hlx.controllers.BundleViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; +import net.helix.hlx.model.IntegerIndex; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.milestone.MilestoneService; import net.helix.hlx.service.milestone.MilestoneSolidifier; @@ -36,7 +39,7 @@ public class ValidatorTrackerImpl implements ValidatorTracker { private SnapshotProvider snapshotProvider; private MilestoneService milestoneService; private MilestoneSolidifier milestoneSolidifier; - private Set validatorAddresses; + private Set latestValidators = new HashSet<>(); private final Set seenTrusteeTransactions = new HashSet<>(); private final Deque trusteeTransactionsToAnalyze = new ArrayDeque<>(); @@ -53,6 +56,28 @@ public ValidatorTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMil return this; } + @Override + public Set getLatestValidators() { + return latestValidators; + } + + + @Override + public Set getValidatorsOfRound(int roundIndex) throws Exception{ + try { + Set validators = new HashSet<>(); + for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { + TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, hash); + if (milestoneService.getRoundIndex(transaction) == roundIndex) { + validators = getValidatorAddresses(hash); + } + } + return validators; + }catch (Exception e) { + throw new Exception("unexpected error while getting Validators of round #{}" + roundIndex, e); + } + } + @Override public boolean processTrusteeTransaction(Hash transactionHash) throws Exception { TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); @@ -67,10 +92,12 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception } // we use milestone validation because its the same process (TODO: Rename function and implement more general) - switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validatorAddresses)) { + Set validation = new HashSet<>(); + validation.add(Trustee_Address); + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validation)) { case VALID: if (roundIndex > latestMilestoneTracker.getLatestRoundIndex()) { - updateValidatorAddresses(transaction.getHash(), roundIndex); + latestValidators = getValidatorAddresses(transaction.getHash()); } if (!transaction.isSolid()) { @@ -96,26 +123,31 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception } @Override - public void updateValidatorAddresses(Hash transaction, int roundIndex) throws Exception{ + public Set getValidatorAddresses(Hash transaction) throws Exception{ TransactionViewModel tail = TransactionViewModel.fromHash(tangle, transaction); BundleViewModel bundle = BundleViewModel.load(tangle, tail.getBundleHash()); + int security = 1; + Set validators = new HashSet<>(); for (Hash txHash : bundle.getHashes()) { TransactionViewModel tx = TransactionViewModel.fromHash(tangle, txHash); - // get first tx with validator addresses in signatureMessageFragment - // TODO: If we need more than one tx to send the validator addresses, how do we know how many tx containing validators + // get transactions with validator addresses in signatureMessageFragment // -> we have to count from last index - // n to (n-security) : tx with signature - // n-security-1 : tx with merkle path - // 0 to (n-security-1) : tx with validators - if (tx.getCurrentIndex() == 0) { + // n-security-1 to n: tx with signature + // n-security-1: tx with merkle path + // 0 to n-security: tx with validator addresses + if ((tx.getCurrentIndex() <= 0) && (tx.getCurrentIndex() < bundle.size() - security - 1)) { for (int i = 0; i < TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE / Hash.SIZE_IN_BYTES; i++) { Hash address = HashFactory.ADDRESS.create(Arrays.copyOfRange(tx.getSignature(), i * Hash.SIZE_IN_BYTES, (i+1) * Hash.SIZE_IN_BYTES)); // TODO: Do we send all validators or only adding and removing ones - validatorAddresses.add(address); + if (address.equals(Hash.NULL_HASH)){ + break; + } + validators.add(address); } } } + return validators; } @Override From aa29320d322e2baba3d2c375bbb900334507903d Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 16 Jun 2019 20:51:10 +0200 Subject: [PATCH 025/223] implement getConfirmedTips in RoundViewModel for better accessibility --- .../helix/hlx/controllers/RoundViewModel.java | 42 +++++++++++++++++++ .../service/milestone/MilestoneService.java | 2 +- .../milestone/impl/MilestoneServiceImpl.java | 32 ++------------ 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 01d7c71d..f14aa846 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -1,16 +1,22 @@ package net.helix.hlx.controllers; +import net.helix.hlx.BundleValidator; import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.persistables.Round; +import net.helix.hlx.model.persistables.Transaction; import net.helix.hlx.storage.Indexable; import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Pair; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.Set; +import java.util.stream.Collectors; /** * Acts as a controller interface for a {@link Round} hash object. This controller is used by the @@ -208,6 +214,42 @@ public static RoundViewModel findClosestNextRound(Tangle tangle, int index, int return nextRoundViewModel; } + public Set getConfirmedTips(Tangle tangle) throws Exception { + + Map occurrences = new HashMap<>(); + int security = 1; + int quorum = 2 * size() / 3; + + for (Hash milestoneHash : getHashes()) { + + TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); + BundleViewModel bundle = BundleViewModel.load(tangle, milestoneTx.getBundleHash()); + + for (Hash bundleTxHash : bundle.getHashes()) { + + TransactionViewModel bundleTx = TransactionViewModel.fromHash(tangle, bundleTxHash); + if ((bundleTx.getCurrentIndex() >= 0) && (bundleTx.getCurrentIndex() < bundle.size() - security - 1)) { + + for (int i = 0; i < 16; i++) { + Hash tip = HashFactory.TRANSACTION.create(bundleTx.getSignature(), i * Hash.SIZE_IN_BYTES, (i + 1) * Hash.SIZE_IN_BYTES); + if (tip.equals(Hash.NULL_HASH)) { + break; + } + if (occurrences.containsKey(tip)) { + occurrences.put(tip, occurrences.get(tip) + 1); + } else { + occurrences.put(tip, 1); + } + } + } + } + } + return occurrences.entrySet().stream() + .filter(entry -> entry.getValue() >= quorum) + .map(entry -> entry.getKey()) + .collect(Collectors.toSet()); + } + /** * Save the {@link Round} object, indexed by its integer index, to the database. * diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index f96e7c13..277703f6 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -108,7 +108,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i */ boolean isTransactionConfirmed(TransactionViewModel transaction); - Set getConfirmedTips(int roundNumber, int quorum) throws Exception; + Set getConfirmedTips(int roundNumber) throws Exception; /** * Retrieves the milestone index of the given transaction by decoding the {@code OBSOLETE_TAG}.
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index e7171aea..0368f355 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -228,36 +228,10 @@ public int getRoundIndex(TransactionViewModel milestoneTransaction) { } @Override - public Set getConfirmedTips(int roundNumber, int quorum) throws Exception { + public Set getConfirmedTips(int roundNumber) throws Exception { RoundViewModel round = RoundViewModel.get(tangle, roundNumber); - Map occurrences = new HashMap<>(); - - for (Hash milestoneHash : round.getHashes()) { - final List> bundleTransactions = BundleValidator.validate(tangle, - snapshotProvider.getInitialSnapshot(), milestoneHash); - - for (final List bundleTransactionViewModels : bundleTransactions) { - - final TransactionViewModel tipsTx = bundleTransactionViewModels.get(bundleTransactionViewModels.size() - 1); - - for (int i = 0; i < 16; i++) { - Hash tip = HashFactory.TRANSACTION.create(tipsTx.getSignature(), i * Hash.SIZE_IN_BYTES, (i + 1) * Hash.SIZE_IN_BYTES); - if (tip.equals(Hash.NULL_HASH)) { - break; - } - if (occurrences.containsKey(tip)) { - occurrences.put(tip, occurrences.get(tip) + 1); - } else { - occurrences.put(tip, 1); - } - } - } - } - return occurrences.entrySet().stream() - .filter(entry -> entry.getValue() >= quorum) - .map(entry -> entry.getKey()) - .collect(Collectors.toSet()); + return round.getConfirmedTips(tangle); } /*@Override @@ -391,7 +365,7 @@ private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIn Set inconsistentMilestones = new HashSet<>(); try { - final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex, 2)); + final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex)); Hash transactionPointer; while ((transactionPointer = transactionsToUpdate.poll()) != null) { if (processedTransactions.add(transactionPointer)) { From 7ed2cdf2dbdc7f7f126fd08bf7b43f71de6c528f Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 16 Jun 2019 20:53:26 +0200 Subject: [PATCH 026/223] start from confirmed tips to get/check solid entry points --- .../snapshot/impl/SnapshotServiceImpl.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 21fc9191..4713fa9d 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -598,7 +598,7 @@ private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundView for (TransactionViewModel unconfirmedApprover : unconfirmedApprovers) { // TODO: need access from milestoneService // if one of the unconfirmed approvers isn't orphaned from the perspective of one of the confirmed tips, the transaction is a solid entry point - for (Hash milestoneHash : milestoneService.getConfirmedTips(targetRound.index(), quorum)) { + for (Hash milestoneHash : targetRound.getConfirmedTips(tangle)) { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); if (!isOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) { return true; @@ -674,18 +674,18 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot progressLogger.getCurrentStep() < progressLogger.getStepCount()) { RoundViewModel currentMilestone = nextMilestone; - //TODO: reimplement traverseApprovees(), does it make sense to use several entry points when traversing? - DAGHelper.get(tangle).traverseApprovees( - currentMilestone.getHash(), - currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(), - currentTransaction -> { - if (isSolidEntryPoint(tangle, currentTransaction.getHash(), targetMilestone)) { - solidEntryPoints.put(currentTransaction.getHash(), targetMilestone.index()); + for (Hash confirmedTip : currentMilestone.getConfirmedTips(tangle)) { + DAGHelper.get(tangle).traverseApprovees( + confirmedTip, + currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(), + currentTransaction -> { + if (isSolidEntryPoint(tangle, currentTransaction.getHash(), targetMilestone)) { + solidEntryPoints.put(currentTransaction.getHash(), targetMilestone.index()); + } } - } - ); - - solidEntryPoints.put(currentMilestone.getHash(), targetMilestone.index()); + ); + solidEntryPoints.put(confirmedTip, targetMilestone.index()); + } nextMilestone = RoundViewModel.findClosestPrevRound(tangle, currentMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex()); From faf76386f7f25c45dfd2bd933ce1c83f87702d0b Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 16 Jun 2019 20:54:34 +0200 Subject: [PATCH 027/223] remove latest milestone --- src/main/java/net/helix/hlx/service/API.java | 5 ++--- .../hlx/service/dto/GetNodeInfoResponse.java | 17 ++--------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 63e074fa..ec79227a 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -654,13 +654,12 @@ private AbstractResponse getNodeInfoStatement() throws Exception { System.getProperty("java.version"), Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), - latestMilestoneTracker.getLatestRoundHashes(), latestMilestoneTracker.getLatestRoundIndex(), snapshotProvider.getLatestSnapshot().getHash(), snapshotProvider.getLatestSnapshot().getIndex(), - milestone != null ? milestone.index() : -1, + round != null ? round.index() : -1, snapshotProvider.getLatestSnapshot().getInitialIndex(), node.howManyNeighbors(), @@ -669,7 +668,7 @@ private AbstractResponse getNodeInfoStatement() throws Exception { tipsViewModel.size(), transactionRequester.numberOfTransactionsToRequest(), features, - configuration.getCoordinator()); + configuration.getTrusteeAddress().hexString()); } /** diff --git a/src/main/java/net/helix/hlx/service/dto/GetNodeInfoResponse.java b/src/main/java/net/helix/hlx/service/dto/GetNodeInfoResponse.java index 9e207b29..e2b596e6 100644 --- a/src/main/java/net/helix/hlx/service/dto/GetNodeInfoResponse.java +++ b/src/main/java/net/helix/hlx/service/dto/GetNodeInfoResponse.java @@ -49,13 +49,9 @@ public class GetNodeInfoResponse extends AbstractResponse { */ private long jreTotalMemory; - /** - * The hash of the latest transaction that was signed off by the coordinator. - */ - private String latestMilestone; /** - * Index of the {@link #latestMilestone} + * Index of the latest round */ private int latestMilestoneIndex; @@ -135,7 +131,6 @@ public class GetNodeInfoResponse extends AbstractResponse { * @param jreVersion {@link #jreVersion} * @param maxMemory {@link #jreMaxMemory} * @param totalMemory {@link #jreTotalMemory} - * @param latestMilestone {@link #latestMilestone} * @param latestMilestoneIndex {@link #latestMilestoneIndex} * @param latestSolidSubtangleMilestone {@link #latestSolidSubtangleMilestone} * @param latestSolidSubtangleMilestoneIndex {@link #latestSolidSubtangleMilestoneIndex} @@ -151,7 +146,7 @@ public class GetNodeInfoResponse extends AbstractResponse { * @return a {@link GetNodeInfoResponse} filled with all the provided parameters */ public static AbstractResponse create(String appName, String appVersion, int jreAvailableProcessors, long jreFreeMemory, - String jreVersion, long maxMemory, long totalMemory, Hash latestMilestone, int latestMilestoneIndex, + String jreVersion, long maxMemory, long totalMemory, int latestMilestoneIndex, Hash latestSolidSubtangleMilestone, int latestSolidSubtangleMilestoneIndex, int milestoneStartIndex, int lastSnapshottedMilestoneIndex, int neighbors, int packetsQueueSize, long currentTimeMillis, int tips, int numberOfTransactionsToRequest, String[] features, String coordinatorAddress) { @@ -164,7 +159,6 @@ public static AbstractResponse create(String appName, String appVersion, int jre res.jreMaxMemory = maxMemory; res.jreTotalMemory = totalMemory; - res.latestMilestone = Hex.toHexString(latestMilestone.bytes()); res.latestMilestoneIndex = latestMilestoneIndex; res.latestSolidSubtangleMilestone = Hex.toHexString(latestSolidSubtangleMilestone.bytes()); @@ -241,13 +235,6 @@ public String getJreVersion() { return jreVersion; } - /** - * - * @return {@link #latestMilestone} - */ - public String getLatestMilestone() { - return latestMilestone; - } /** * From d2436725c35dff3afc3d63492d31c74a81153262 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 11:26:30 +0200 Subject: [PATCH 028/223] integrate validatorTracker --- src/main/java/net/helix/hlx/Helix.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index 8eed9a36..6b094624 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -78,6 +78,7 @@ public class Helix { public final LocalSnapshotManagerImpl localSnapshotManager; public final MilestoneServiceImpl milestoneService; public final LatestMilestoneTrackerImpl latestMilestoneTracker; + public final ValidatorTrackerImpl validatorTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; public final SeenMilestonesRetrieverImpl seenMilestonesRetriever; public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); @@ -123,6 +124,7 @@ public Helix(HelixConfig configuration) throws TransactionPruningException, Snap : null; milestoneService = new MilestoneServiceImpl(); latestMilestoneTracker = new LatestMilestoneTrackerImpl(); + validatorTracker = new ValidatorTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); @@ -165,7 +167,7 @@ public void init() throws Exception { } if (configuration.isRevalidate()) { - tangle.clearColumn(net.helix.hlx.model.persistables.Milestone.class); + tangle.clearColumn(net.helix.hlx.model.persistables.Round.class); tangle.clearColumn(net.helix.hlx.model.StateDiff.class); tangle.clearMetadata(net.helix.hlx.model.persistables.Transaction.class); } @@ -209,8 +211,9 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx } milestoneService.init(tangle, snapshotProvider, snapshotService, configuration); latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); + validatorTracker.init(tangle, latestMilestoneTracker, snapshotProvider, milestoneService, milestoneSolidifier, configuration); latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService, - latestMilestoneTracker); + latestMilestoneTracker, validatorTracker); seenMilestonesRetriever.init(tangle, snapshotProvider, transactionRequester); milestoneSolidifier.init(snapshotProvider, transactionValidator); ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, graph); @@ -228,7 +231,7 @@ private void rescanDb() throws Exception { tangle.clearColumn(net.helix.hlx.model.persistables.Approvee.class); tangle.clearColumn(net.helix.hlx.model.persistables.BundleNonce.class); tangle.clearColumn(net.helix.hlx.model.persistables.Tag.class); - tangle.clearColumn(net.helix.hlx.model.persistables.Milestone.class); + tangle.clearColumn(net.helix.hlx.model.persistables.Round.class); tangle.clearColumn(net.helix.hlx.model.StateDiff.class); tangle.clearMetadata(net.helix.hlx.model.persistables.Transaction.class); From 4eb4ffd71cb8eefc9fce3a290bd91fe3fd565633 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 11:27:24 +0200 Subject: [PATCH 029/223] hardcode own coordinator address --- src/main/java/net/helix/hlx/service/milestone/MSS.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MSS.java b/src/main/java/net/helix/hlx/service/milestone/MSS.java index 39f0bcd6..baa2ac1d 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MSS.java +++ b/src/main/java/net/helix/hlx/service/milestone/MSS.java @@ -31,7 +31,7 @@ public MSS(HelixConfig configuration, API api) { int minDelay = this.config.getMinDelay(); this.mwm = this.config.getMwm(); this.message = StringUtils.repeat('0', 1024); - this.address = this.config.getCoordinator(); + this.address = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; this.sign = !this.config.isDontValidateTestnetMilestoneSig(); if(this.delay < minDelay) { From b7e3986ffa2ca0eaa9f01d3c7463768f780faea2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 11:30:18 +0200 Subject: [PATCH 030/223] store seenRounds without hash --- .../snapshot/impl/SnapshotProviderImpl.java | 28 ++++++++++--------- .../snapshot/impl/SnapshotServiceImpl.java | 1 - 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java index 45d9a774..d9df84b7 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java @@ -14,6 +14,8 @@ import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; +import java.util.List; +import java.util.LinkedList; import java.util.stream.Stream; import org.slf4j.Logger; @@ -297,7 +299,7 @@ private Snapshot loadBuiltInSnapshot() throws SnapshotException { config.getMilestoneStartIndex(), config.getSnapshotTime(), solidEntryPoints, - new HashMap<>() + new LinkedList<>() ) ); } @@ -421,9 +423,9 @@ private SnapshotMetaData readSnapshotMetaDatafromFile(File snapshotMetaDataFile) int amountOfSeenMilestones = readAmountOfSeenMilestonesFromMetaDataFile(reader); Map solidEntryPoints = readSolidEntryPointsFromMetaDataFile(reader, amountOfSolidEntryPoints); - Map seenMilestones = readSeenMilestonesFromMetaDataFile(reader, amountOfSeenMilestones); + List seenRounds = readSeenRoundsFromMetaDataFile(reader, amountOfSeenMilestones); - return new SnapshotMetaDataImpl(hash, index, timestamp, solidEntryPoints, seenMilestones); + return new SnapshotMetaDataImpl(hash, index, timestamp, solidEntryPoints, seenRounds); } catch (IOException e) { throw new SnapshotException("failed to read from the snapshot metadata file at " + snapshotMetaDataFile.getAbsolutePath(), e); @@ -576,17 +578,17 @@ private Map readSolidEntryPointsFromMetaDataFile(BufferedReader r * This method reads the seen milestones of the {@link Snapshot} from the metadata file. * * @param reader reader that is used to read the file - * @param amountOfSeenMilestones the amount of seen milestones we expect + * @param amountOfSeenRounds the amount of seen milestones we expect * @return the seen milestones of the {@link Snapshot} * @throws SnapshotException if anything goes wrong while reading the seen milestones from the file * @throws IOException if we could not read from the file */ - private Map readSeenMilestonesFromMetaDataFile(BufferedReader reader, int amountOfSeenMilestones) + private List readSeenRoundsFromMetaDataFile(BufferedReader reader, int amountOfSeenRounds) throws SnapshotException, IOException { - Map seenMilestones = new HashMap<>(); + List seenRounds = new LinkedList<>(); - for(int i = 0; i < amountOfSeenMilestones; i++) { + for(int i = 0; i < amountOfSeenRounds; i++) { String line; if ((line = reader.readLine()) == null) { throw new SnapshotException("could not read a seen milestone from the metadata file"); @@ -595,7 +597,7 @@ private Map readSeenMilestonesFromMetaDataFile(BufferedReader rea String[] parts = line.split(";", 2); if(parts.length == 2) { try { - seenMilestones.put(HashFactory.TRANSACTION.create(parts[0]), Integer.parseInt(parts[1])); + seenRounds.add(Integer.parseInt(parts[1])); } catch (NumberFormatException e) { throw new SnapshotException("could not parse a seen milestone from the metadata file", e); } @@ -604,7 +606,7 @@ private Map readSeenMilestonesFromMetaDataFile(BufferedReader rea } } - return seenMilestones; + return seenRounds; } /** @@ -622,7 +624,7 @@ private void writeSnapshotMetaDataToDisk(SnapshotMetaData snapshotMetaData, Stri try { Map solidEntryPoints = snapshotMetaData.getSolidEntryPoints(); - Map seenMilestones = snapshotMetaData.getSeenMilestones(); + List seenMilestones = snapshotMetaData.getSeenRounds(); Files.write( Paths.get(filePath), @@ -639,10 +641,10 @@ private void writeSnapshotMetaDataToDisk(SnapshotMetaData snapshotMetaData, Stri .stream() .sorted(Map.Entry.comparingByValue()) .map(entry -> entry.getKey().hexString() + ";" + entry.getValue()), - seenMilestones.entrySet() + seenMilestones .stream() - .sorted(Map.Entry.comparingByValue()) - .map(entry -> entry.getKey().hexString() + ";" + entry.getValue()) + .sorted() + .map(entry -> entry.toString()) ) ).iterator() ); diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 4713fa9d..30f85b8b 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -596,7 +596,6 @@ private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundView Set processedTransactions = new HashSet<>(); for (TransactionViewModel unconfirmedApprover : unconfirmedApprovers) { - // TODO: need access from milestoneService // if one of the unconfirmed approvers isn't orphaned from the perspective of one of the confirmed tips, the transaction is a solid entry point for (Hash milestoneHash : targetRound.getConfirmedTips(tangle)) { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); From a9813a532f030a4f3d4e05465804050b1a5b4b31 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 11:32:29 +0200 Subject: [PATCH 031/223] add all milestones of round to elementsToDelete --- .../jobs/MilestonePrunerJob.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java index 9d191451..13a85af6 100644 --- a/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/net/helix/hlx/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -263,21 +263,24 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { RoundViewModel roundViewModel = RoundViewModel.get(getTangle(), getCurrentIndex()); if (roundViewModel != null) { - elementsToDelete.add(new Pair<>(roundViewModel.getHash(), Transaction.class)); elementsToDelete.add(new Pair<>(new IntegerIndex(roundViewModel.index()), Round.class)); - DAGHelper.get(getTangle()).traverseApprovees(roundViewModel.getHash(), - approvedTransaction -> approvedTransaction.snapshotIndex() >= roundViewModel.index(), - approvedTransaction -> { - /*if (approvedTransaction.value() < 0 && - !spentAddressesService.wasAddressSpentFrom(approvedTransaction.getAddressHash())) { - log.warn("Pruned spend transaction " + approvedTransaction.getHash() + - " did not have its spent address recorded. Persisting it now"); - spentAddressesService - .persistSpentAddresses(Collections.singletonList(approvedTransaction)); - }*/ //todo patchfixes - elementsToDelete.add(new Pair<>(approvedTransaction.getHash(), Transaction.class)); - }); + for (Hash milestoneHash : roundViewModel.getHashes()) { + elementsToDelete.add(new Pair<>(milestoneHash, Transaction.class)); + + DAGHelper.get(getTangle()).traverseApprovees(milestoneHash, + approvedTransaction -> approvedTransaction.snapshotIndex() >= roundViewModel.index(), + approvedTransaction -> { + /*if (approvedTransaction.value() < 0 && + !spentAddressesService.wasAddressSpentFrom(approvedTransaction.getAddressHash())) { + log.warn("Pruned spend transaction " + approvedTransaction.getHash() + + " did not have its spent address recorded. Persisting it now"); + spentAddressesService + .persistSpentAddresses(Collections.singletonList(approvedTransaction)); + }*/ //todo patchfixes + elementsToDelete.add(new Pair<>(approvedTransaction.getHash(), Transaction.class)); + }); + } } return elementsToDelete; From 656c47f7a4a8dc512def47364d56333a4f19dd66 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 11:36:14 +0200 Subject: [PATCH 032/223] get a random milestone of round as solid entry point for tip selection (just temporary solution, TODO: which milestone should be selected when there are multiple milestones in one round?) --- .../impl/EntryPointSelectorImpl.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index da5da4ac..01ec088d 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -7,6 +7,9 @@ import net.helix.hlx.service.tipselection.EntryPointSelector; import net.helix.hlx.storage.Tangle; +import java.util.Random; + + /** * Implementation of {@link EntryPointSelector} that given a depth {@code N}, returns a N-deep milestone. * Meaning milestone(latestSolid - depth) @@ -35,10 +38,19 @@ public EntryPointSelectorImpl(Tangle tangle, SnapshotProvider snapshotProvider, public Hash getEntryPoint(int depth) throws Exception { int milestoneIndex = Math.max(snapshotProvider.getLatestSnapshot().getIndex() - depth - 1, snapshotProvider.getInitialSnapshot().getIndex()); - RoundViewModel roundViewModel = RoundViewModel.findClosestNextMilestone(tangle, milestoneIndex, - latestMilestoneTracker.getLatestMilestoneIndex()); - if (roundViewModel != null && roundViewModel.getHash() != null) { - return roundViewModel.getHash(); + RoundViewModel roundViewModel = RoundViewModel.findClosestNextRound(tangle, milestoneIndex, + latestMilestoneTracker.getLatestRoundIndex()); + //todo which transaction using as solid entry point when there are multiple milestones / confirmed tips? + //temporary solution: select random + if (roundViewModel != null && roundViewModel.getHashes() != null) { + int size = roundViewModel.getHashes().size(); + int item = new Random().nextInt(size); + int i = 0; + for(Hash obj : roundViewModel.getHashes()) { + if (i == item) + return obj; + i++; + } } return snapshotProvider.getLatestSnapshot().getHash(); From c9bfb393c7db03f6ecc6467aceceb2ceac1b0c69 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 11:59:11 +0200 Subject: [PATCH 033/223] request all milestones of a round --- .../impl/SeenMilestonesRetrieverImpl.java | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java index 3cf8a9f6..d116a4a0 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java @@ -1,5 +1,6 @@ package net.helix.hlx.service.milestone.impl; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.network.TransactionRequester; @@ -13,6 +14,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.LinkedList; +import java.util.List; /** * Creates a manager that proactively requests the missing "seen milestones" (defined in the local snapshot file).
@@ -65,7 +68,7 @@ public class SeenMilestonesRetrieverImpl implements SeenMilestonesRetriever { /** * The list of seen milestones that need to be requested.
*/ - private Map seenMilestones; + private List seenMilestones; /** * This method initializes the instance and registers its dependencies.
@@ -92,7 +95,7 @@ public SeenMilestonesRetrieverImpl init(Tangle tangle, SnapshotProvider snapshot this.snapshotProvider = snapshotProvider; this.transactionRequester = transactionRequester; - seenMilestones = new ConcurrentHashMap<>(snapshotProvider.getInitialSnapshot().getSeenMilestones()); + seenMilestones = new LinkedList<>(snapshotProvider.getInitialSnapshot().getSeenRounds()); return this; } @@ -112,21 +115,25 @@ public SeenMilestonesRetrieverImpl init(Tangle tangle, SnapshotProvider snapshot */ @Override public void retrieveSeenMilestones() { - seenMilestones.forEach((milestoneHash, milestoneIndex) -> { + seenMilestones.forEach((roundIndex) -> { try { - if (milestoneIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { - seenMilestones.remove(milestoneHash); - } else if (milestoneIndex < snapshotProvider.getLatestSnapshot().getIndex() + RETRIEVE_RANGE) { - TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); - if (milestoneTransaction.getType() == TransactionViewModel.PREFILLED_SLOT && - !transactionRequester.isTransactionRequested(milestoneHash, true)) { - - transactionRequester.requestTransaction(milestoneHash, true); + if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { + seenMilestones.remove(roundIndex); + } else if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() + RETRIEVE_RANGE) { + RoundViewModel round = RoundViewModel.get(tangle, roundIndex); + for (Hash milestoneHash : round.getHashes()) { + TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); + if (milestoneTransaction.getType() == TransactionViewModel.PREFILLED_SLOT && + !transactionRequester.isTransactionRequested(milestoneHash, true)) { + + transactionRequester.requestTransaction(milestoneHash, true); + } } - // the transactionRequester will never drop milestone requests - we can therefore remove it from the // list of milestones to request - seenMilestones.remove(milestoneHash); + //todo here the roundIndex will probably interpreted as index and not as the Object to delete, which is wrong + //todo anyway it doesn't make sense to store Integers in a List (or does it?) + seenMilestones.remove(roundIndex); } log.info("Requesting seen milestones (" + seenMilestones.size() + " left) ..."); From d6b440a9ac4fc980d6a0646898b57c181b7ce2df Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 12:06:24 +0200 Subject: [PATCH 034/223] traverse tangle starting at confirmed tips --- .../hlx/service/ledger/impl/LedgerServiceImpl.java | 2 +- .../impl/SpentAddressesServiceImpl.java | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index 8d9dd4d8..637dcc95 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -286,7 +286,7 @@ private boolean generateStateDiff(RoundViewModel round) throws LedgerException { boolean successfullyProcessed = false; snapshotProvider.getLatestSnapshot().lockRead(); try { - Set comfirmedTips = milestoneService.getConfirmedTips(round.index(), 2); + Set comfirmedTips = milestoneService.getConfirmedTips(round.index()); Map balanceChanges = generateBalanceDiff(new HashSet<>(), comfirmedTips, snapshotProvider.getLatestSnapshot().getIndex()); successfullyProcessed = balanceChanges != null; diff --git a/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java index 4e790f9f..cb644709 100644 --- a/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -81,11 +81,13 @@ public void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) for (int i = fromMilestoneIndex; i < toMilestoneIndex; i++) { RoundViewModel currentMilestone = RoundViewModel.get(tangle, i); if (currentMilestone != null) { - DAGHelper.get(tangle).traverseApprovees( - currentMilestone.getHash(), - transactionViewModel -> transactionViewModel.snapshotIndex() >= currentMilestone.index(), - transactionViewModel -> addressesToCheck.add(transactionViewModel.getAddressHash()) - ); + for (Hash confirmedTip : currentMilestone.getConfirmedTips(tangle)) { + DAGHelper.get(tangle).traverseApprovees( + confirmedTip, + transactionViewModel -> transactionViewModel.snapshotIndex() >= currentMilestone.index(), + transactionViewModel -> addressesToCheck.add(transactionViewModel.getAddressHash()) + ); + } } } } catch (Exception e) { From d66144e07966b107232c46088ebe10607a106375 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 12:50:39 +0200 Subject: [PATCH 035/223] implementation of getTipSet, getConfirmedTips, getConfirmingMilestones and getRandomConfirmingMilestone --- .../helix/hlx/controllers/RoundViewModel.java | 83 +++++++++++++------ src/main/java/net/helix/hlx/network/Node.java | 4 +- .../impl/EntryPointSelectorImpl.java | 9 +- 3 files changed, 63 insertions(+), 33 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index f14aa846..761fecf1 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -11,11 +11,8 @@ import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Pair; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import java.util.Set; import java.util.stream.Collectors; /** @@ -24,6 +21,9 @@ */ public class RoundViewModel { private final Round round; + //todo might be nice to have direct access of confirmedTips and confirming Milestones + //private final Set confirmedTips = new HashSet<>(); + //private final Set confirmingMilestones = new HashSet<>(); private static final Map rounds = new ConcurrentHashMap<>(); private RoundViewModel(final Round round) { @@ -214,33 +214,43 @@ public static RoundViewModel findClosestNextRound(Tangle tangle, int index, int return nextRoundViewModel; } - public Set getConfirmedTips(Tangle tangle) throws Exception { + public Set getTipSet(Tangle tangle, Hash milestone) throws Exception { - Map occurrences = new HashMap<>(); int security = 1; - int quorum = 2 * size() / 3; + TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestone); + BundleViewModel bundle = BundleViewModel.load(tangle, milestoneTx.getBundleHash()); + Set tips = new HashSet<>(); - for (Hash milestoneHash : getHashes()) { + for (Hash bundleTxHash : bundle.getHashes()) { - TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); - BundleViewModel bundle = BundleViewModel.load(tangle, milestoneTx.getBundleHash()); + TransactionViewModel bundleTx = TransactionViewModel.fromHash(tangle, bundleTxHash); + if ((bundleTx.getCurrentIndex() >= 0) && (bundleTx.getCurrentIndex() < bundle.size() - security - 1)) { - for (Hash bundleTxHash : bundle.getHashes()) { + for (int i = 0; i < 16; i++) { + Hash tip = HashFactory.TRANSACTION.create(bundleTx.getSignature(), i * Hash.SIZE_IN_BYTES, (i + 1) * Hash.SIZE_IN_BYTES); + if (tip.equals(Hash.NULL_HASH)) { + break; + } + tips.add(tip); + } + } + } + return tips; + } - TransactionViewModel bundleTx = TransactionViewModel.fromHash(tangle, bundleTxHash); - if ((bundleTx.getCurrentIndex() >= 0) && (bundleTx.getCurrentIndex() < bundle.size() - security - 1)) { + public Set getConfirmedTips(Tangle tangle) throws Exception { - for (int i = 0; i < 16; i++) { - Hash tip = HashFactory.TRANSACTION.create(bundleTx.getSignature(), i * Hash.SIZE_IN_BYTES, (i + 1) * Hash.SIZE_IN_BYTES); - if (tip.equals(Hash.NULL_HASH)) { - break; - } - if (occurrences.containsKey(tip)) { - occurrences.put(tip, occurrences.get(tip) + 1); - } else { - occurrences.put(tip, 1); - } - } + Map occurrences = new HashMap<>(); + int quorum = 2 * size() / 3; + + for (Hash milestoneHash : getHashes()) { + Set tips = getTipSet(tangle, milestoneHash); + + for (Hash tip : tips) { + if (occurrences.containsKey(tip)) { + occurrences.put(tip, occurrences.get(tip) + 1); + } else { + occurrences.put(tip, 1); } } } @@ -250,6 +260,31 @@ public Set getConfirmedTips(Tangle tangle) throws Exception { .collect(Collectors.toSet()); } + public Set getConfirmingMilestones(Tangle tangle) throws Exception { + Set confirmedTips = getConfirmedTips(tangle); + Set confirmingMilestones = new HashSet<>(); + + for (Hash milestoneHash : getHashes()) { + Set tips = getTipSet(tangle, milestoneHash); + if (confirmedTips.equals(tips)) { + confirmingMilestones.add(milestoneHash); + } + } + return confirmingMilestones; + } + + public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { + Set confirmingMilestones = getConfirmingMilestones(tangle); + int item = new Random().nextInt(confirmingMilestones.size()); + int i = 0; + for(Hash obj : confirmingMilestones) { + if (i == item) + return obj; + i++; + } + return (Hash) confirmingMilestones.toArray()[0]; + } + /** * Save the {@link Round} object, indexed by its integer index, to the database. * diff --git a/src/main/java/net/helix/hlx/network/Node.java b/src/main/java/net/helix/hlx/network/Node.java index 0243dc5a..81edd02d 100644 --- a/src/main/java/net/helix/hlx/network/Node.java +++ b/src/main/java/net/helix/hlx/network/Node.java @@ -5,6 +5,7 @@ import net.helix.hlx.TransactionValidator; import net.helix.hlx.conf.NodeConfig; import net.helix.hlx.controllers.BundleViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TipsViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; @@ -577,7 +578,8 @@ public void replyToRequest(Hash requestedHash, Neighbor neighbor) { } private Hash getRandomTipPointer() throws Exception { - Hash tip = rnd.nextDouble() < configuration.getpSendMilestone() ? latestMilestoneTracker.getLatestMilestoneHash() : tipsViewModel.getRandomSolidTipHash(); + RoundViewModel latestRound = RoundViewModel.latest(tangle); + Hash tip = rnd.nextDouble() < configuration.getpSendMilestone() ? latestRound.getRandomConfirmingMilestone(tangle) : tipsViewModel.getRandomSolidTipHash(); return tip == null ? Hash.NULL_HASH : tip; } diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index 01ec088d..8b8ba90b 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -43,14 +43,7 @@ public Hash getEntryPoint(int depth) throws Exception { //todo which transaction using as solid entry point when there are multiple milestones / confirmed tips? //temporary solution: select random if (roundViewModel != null && roundViewModel.getHashes() != null) { - int size = roundViewModel.getHashes().size(); - int item = new Random().nextInt(size); - int i = 0; - for(Hash obj : roundViewModel.getHashes()) { - if (i == item) - return obj; - i++; - } + return roundViewModel.getRandomConfirmingMilestone(tangle); } return snapshotProvider.getLatestSnapshot().getHash(); From 95e0702508ae9c4396b873dee1f56aebefb9fb5d Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 12:52:04 +0200 Subject: [PATCH 036/223] send a tip request packet of transactions corresponding to all milestones of latest round --- src/main/java/net/helix/hlx/network/Node.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/main/java/net/helix/hlx/network/Node.java b/src/main/java/net/helix/hlx/network/Node.java index 81edd02d..0dcc364b 100644 --- a/src/main/java/net/helix/hlx/network/Node.java +++ b/src/main/java/net/helix/hlx/network/Node.java @@ -679,27 +679,28 @@ private Runnable spawnTipRequesterThread() { while (!shuttingDown.get()) { try { - final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, latestMilestoneTracker.getLatestMilestoneHash()); - System.arraycopy(transactionViewModel.getBytes(), 0, tipRequestingPacket.getData(), 0, TransactionViewModel.SIZE); - System.arraycopy(transactionViewModel.getHash().bytes(), 0, tipRequestingPacket.getData(), TransactionViewModel.SIZE, - reqHashSize); - //Hash.SIZE_IN_BYTES); - - neighbors.forEach(n -> n.send(tipRequestingPacket)); - - long now = System.currentTimeMillis(); - if ((now - lastTime) > 10000L) { - lastTime = now; - tangle.publish("rstat %d %d %d %d %d", - getReceiveQueueSize(), getBroadcastQueueSize(), - transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), - TransactionViewModel.getNumberOfStoredTransactions(tangle)); - log.info("toProcess = {} , toBroadcast = {} , toRequest = {} , toReply = {} / totalTransactions = {}", - getReceiveQueueSize(), getBroadcastQueueSize(), - transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), - TransactionViewModel.getNumberOfStoredTransactions(tangle)); + for (Hash milestoneHash : latestMilestoneTracker.getLatestRoundHashes()) { + final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, milestoneHash); + System.arraycopy(transactionViewModel.getBytes(), 0, tipRequestingPacket.getData(), 0, TransactionViewModel.SIZE); + System.arraycopy(transactionViewModel.getHash().bytes(), 0, tipRequestingPacket.getData(), TransactionViewModel.SIZE, + reqHashSize); + //Hash.SIZE_IN_BYTES); + + neighbors.forEach(n -> n.send(tipRequestingPacket)); + + long now = System.currentTimeMillis(); + if ((now - lastTime) > 10000L) { + lastTime = now; + tangle.publish("rstat %d %d %d %d %d", + getReceiveQueueSize(), getBroadcastQueueSize(), + transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), + TransactionViewModel.getNumberOfStoredTransactions(tangle)); + log.info("toProcess = {} , toBroadcast = {} , toRequest = {} , toReply = {} / totalTransactions = {}", + getReceiveQueueSize(), getBroadcastQueueSize(), + transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), + TransactionViewModel.getNumberOfStoredTransactions(tangle)); + } } - Thread.sleep(5000); } catch (final Exception e) { log.error("Tips Requester Thread Exception:", e); From 018a7bcdccb97f28642297fa20e80a448df21c4d Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 17 Jun 2019 12:57:33 +0200 Subject: [PATCH 037/223] implement getTrusteeAddress TestnetConfig --- .../java/net/helix/hlx/conf/TestnetConfig.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/TestnetConfig.java b/src/main/java/net/helix/hlx/conf/TestnetConfig.java index f25cec88..a1c16380 100644 --- a/src/main/java/net/helix/hlx/conf/TestnetConfig.java +++ b/src/main/java/net/helix/hlx/conf/TestnetConfig.java @@ -1,14 +1,17 @@ package net.helix.hlx.conf; +import net.helix.hlx.model.Hash; + import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonProperty; +import net.helix.hlx.model.HashFactory; import java.util.Objects; public class TestnetConfig extends BaseHelixConfig { - protected String coordinator = Defaults.COORDINATOR_ADDRESS; + protected Hash cooAddress = Defaults.COORDINATOR_ADDRESS; protected boolean dontValidateTestnetMilestoneSig = Defaults.DONT_VALIDATE_MILESTONE_SIG; protected String snapshotFile = Defaults.SNAPSHOT_FILE; protected String snapshotSignatureFile = Defaults.SNAPSHOT_SIG; @@ -32,14 +35,14 @@ public boolean isTestnet() { } @Override - public String getCoordinator() { - return coordinator; + public Hash getTrusteeAddress() { + return cooAddress; } @JsonProperty @Parameter(names = "--testnet-coordinator", description = MilestoneConfig.Descriptions.VALIDATOR_ADDRESSES) - protected void setCoordinator(String coordinator) { - this.coordinator = coordinator; + protected void setCoordinator(Hash coordinator) { + this.cooAddress = coordinator; } @Override @@ -160,7 +163,7 @@ public void setDbLogPath(String dbLogPath) { } public interface Defaults { - String COORDINATOR_ADDRESS = "6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"; + Hash COORDINATOR_ADDRESS = HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"); boolean DONT_VALIDATE_MILESTONE_SIG = false; String LOCAL_SNAPSHOTS_BASE_PATH = "testnet"; String SNAPSHOT_FILE = "/snapshotTestnet.txt"; From 7973dd082a211b941975eec3172eee8dfd1e26de Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 18 Jun 2019 12:43:48 +0200 Subject: [PATCH 038/223] spawnTipRequesterThread temporary solution with NULL_HASH --- src/main/java/net/helix/hlx/network/Node.java | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/main/java/net/helix/hlx/network/Node.java b/src/main/java/net/helix/hlx/network/Node.java index 0dcc364b..539a33e4 100644 --- a/src/main/java/net/helix/hlx/network/Node.java +++ b/src/main/java/net/helix/hlx/network/Node.java @@ -679,28 +679,29 @@ private Runnable spawnTipRequesterThread() { while (!shuttingDown.get()) { try { - for (Hash milestoneHash : latestMilestoneTracker.getLatestRoundHashes()) { - final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, milestoneHash); - System.arraycopy(transactionViewModel.getBytes(), 0, tipRequestingPacket.getData(), 0, TransactionViewModel.SIZE); - System.arraycopy(transactionViewModel.getHash().bytes(), 0, tipRequestingPacket.getData(), TransactionViewModel.SIZE, - reqHashSize); - //Hash.SIZE_IN_BYTES); - - neighbors.forEach(n -> n.send(tipRequestingPacket)); - - long now = System.currentTimeMillis(); - if ((now - lastTime) > 10000L) { - lastTime = now; - tangle.publish("rstat %d %d %d %d %d", - getReceiveQueueSize(), getBroadcastQueueSize(), - transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), - TransactionViewModel.getNumberOfStoredTransactions(tangle)); - log.info("toProcess = {} , toBroadcast = {} , toRequest = {} , toReply = {} / totalTransactions = {}", - getReceiveQueueSize(), getBroadcastQueueSize(), - transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), - TransactionViewModel.getNumberOfStoredTransactions(tangle)); - } + //todo don't know whats going on here ?? + + final TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, Hash.NULL_HASH); + System.arraycopy(transactionViewModel.getBytes(), 0, tipRequestingPacket.getData(), 0, TransactionViewModel.SIZE); + System.arraycopy(transactionViewModel.getHash().bytes(), 0, tipRequestingPacket.getData(), TransactionViewModel.SIZE, + reqHashSize); + //Hash.SIZE_IN_BYTES); + + neighbors.forEach(n -> n.send(tipRequestingPacket)); + + long now = System.currentTimeMillis(); + if ((now - lastTime) > 10000L) { + lastTime = now; + tangle.publish("rstat %d %d %d %d %d", + getReceiveQueueSize(), getBroadcastQueueSize(), + transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), + TransactionViewModel.getNumberOfStoredTransactions(tangle)); + log.info("toProcess = {} , toBroadcast = {} , toRequest = {} , toReply = {} / totalTransactions = {}", + getReceiveQueueSize(), getBroadcastQueueSize(), + transactionRequester.numberOfTransactionsToRequest(), getReplyQueueSize(), + TransactionViewModel.getNumberOfStoredTransactions(tangle)); } + Thread.sleep(5000); } catch (final Exception e) { log.error("Tips Requester Thread Exception:", e); From 6ef842f596514bbb8b6a375dd4c4718a74005631 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 18 Jun 2019 12:44:52 +0200 Subject: [PATCH 039/223] no tip selection for milestones --- src/main/java/net/helix/hlx/service/API.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index ec79227a..76d42e3a 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1493,12 +1493,8 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri int latestMilestoneIndex = latestMilestoneTracker.getLatestRoundIndex(); long nextIndex = latestMilestoneIndex+1; List txToApprove = new ArrayList<>(); - if(Hash.NULL_HASH.equals(latestMilestoneTracker.getLatestRoundHashes())) { - txToApprove.add(Hash.NULL_HASH); - txToApprove.add(Hash.NULL_HASH); - } else { - txToApprove = getTransactionToApproveTips(3, Optional.empty()); - } + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); // A milestone consists of two transactions. // The last transaction (currentIndex == lastIndex) contains the siblings for the merkle tree. From a9c4f1f860839538bfc798c41dd205d7356aca68 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 18 Jun 2019 12:45:30 +0200 Subject: [PATCH 040/223] rename getTrusteeAddress --- src/test/java/net/helix/hlx/conf/ConfigTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/net/helix/hlx/conf/ConfigTest.java b/src/test/java/net/helix/hlx/conf/ConfigTest.java index e629eece..89b84f30 100644 --- a/src/test/java/net/helix/hlx/conf/ConfigTest.java +++ b/src/test/java/net/helix/hlx/conf/ConfigTest.java @@ -104,7 +104,7 @@ public void testArgsParsingMainnet() { Assert.assertEquals("db path", "/db", helixConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, helixConfig.isZmqEnabled()); Assert.assertNotEquals("mwm", 4, helixConfig.getMwm()); - Assert.assertNotEquals("coo", helixConfig.getCoordinator(), "TTTTTTTTT"); + Assert.assertNotEquals("coo", helixConfig.getTrusteeAddress(), "TTTTTTTTT"); Assert.assertEquals("--testnet-no-coo-validation", false, helixConfig.isDontValidateTestnetMilestoneSig()); } @@ -172,7 +172,7 @@ public void testArgsParsingTestnet() { Assert.assertEquals("db path", "/db", helixConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, helixConfig.isZmqEnabled()); Assert.assertEquals("mwm", 4, helixConfig.getMwm()); - Assert.assertEquals("coo", "TTTTTTTTT", helixConfig.getCoordinator()); + Assert.assertEquals("coo", "TTTTTTTTT", helixConfig.getTrusteeAddress()); Assert.assertEquals("--testnet-no-coo-validation", true, helixConfig.isDontValidateTestnetMilestoneSig()); } From f30800d3514124ac01acdb8cf271a9116cbea190 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 18 Jun 2019 12:48:10 +0200 Subject: [PATCH 041/223] implement round counter thread --- .../impl/LatestMilestoneTrackerImpl.java | 57 ++++++++++++++++--- .../impl/LatestSolidMilestoneTrackerImpl.java | 8 +-- .../milestone/impl/MilestoneServiceImpl.java | 5 +- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index e3798fb1..38bdff56 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -7,6 +7,7 @@ import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; +import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; @@ -43,6 +44,9 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { */ private static final int RESCAN_INTERVAL = 1000; + private static final int ROUND_DURATION = 5000; + + /** * Holds the logger of this class (a rate limited logger that doesn't spam the CLI output).
*/ @@ -77,9 +81,14 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { /** * Holds a reference to the manager of the background worker.
*/ - private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + private final SilentScheduledExecutorService executorService1 = new DedicatedScheduledExecutorService( "Latest Milestone Tracker", log.delegate()); + private final SilentScheduledExecutorService executorService2 = new DedicatedScheduledExecutorService( + "Round Counter", log.delegate()); + + private long roundStart; + /** * Holds the round index of the latest round that we have seen / processed.
*/ @@ -140,9 +149,14 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; - //validatorAddresses = config.getValidatorAddresses(); + validatorAddresses = new HashSet<>(); + validatorAddresses.add(HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a")); + + latestRoundHashes = new HashSet<>(); - bootstrapLatestMilestoneValue(); + roundStart = System.currentTimeMillis(); + + bootstrapLatestRoundIndex(); return this; } @@ -156,7 +170,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP @Override public void addMilestoneToLatestRound(Hash milestoneHash, int latestRoundIndex) { tangle.publish("lmi %d %d", milestoneHash.hexString(), latestRoundIndex); - log.delegate().info("New milestone added to round #{}", latestRoundIndex); + log.delegate().info("New milestone ({}/{}) added to round #{}", latestRoundHashes.size() + 1, validatorAddresses.size(), latestRoundIndex); this.latestRoundHashes.add(milestoneHash); } @@ -258,15 +272,44 @@ public boolean isInitialScanComplete() { */ @Override public void start() { - executorService.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, + executorService1.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, + TimeUnit.MILLISECONDS); + executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION * 2, ROUND_DURATION, TimeUnit.MILLISECONDS); } + @Override public void shutdown() { - executorService.shutdownNow(); + executorService1.shutdownNow(); + executorService2.shutdownNow(); } + + public void roundCounter(){ + try { + incrementRound(); + }catch (Exception e) { + log.error("error while incrementing round", e); + } + } + + public void incrementRound() throws MilestoneException{ + try { + // init new round + roundStart = System.currentTimeMillis(); + RoundViewModel currentRoundViewModel = new RoundViewModel(latestRoundIndex + 1, new HashSet<>()); + currentRoundViewModel.store(tangle); + System.out.println("Stored round #" + currentRoundViewModel.index()); + // clear and increment latest round + clearLatestRoundHashes(); + setLatestRoundIndex(latestRoundIndex + 1); + } catch (Exception e) { + throw new MilestoneException("unexpected error while incrementing round #{}" + (latestRoundIndex + 1), e); + } + } + + /** * This method contains the logic for scanning for new latest milestones that gets executed in a background * worker.
@@ -382,7 +425,7 @@ private void checkIfInitializationComplete() { * milestone at the end of our database. While this last entry in the database doesn't necessarily have to be the * latest one we know it at least gives a reasonable value most of the times.
*/ - private void bootstrapLatestMilestoneValue() { + private void bootstrapLatestRoundIndex() { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); setLatestRoundIndex(latestSnapshot.getIndex()); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 98b4a7fd..1ebf7f08 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -17,7 +17,6 @@ import net.helix.hlx.utils.thread.SilentScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.Set; /** * Creates a manager that keeps track of the latest solid milestones and that triggers the application of these @@ -33,7 +32,7 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac * Holds the interval (in milliseconds) in which the {@link #trackLatestSolidMilestone()} method gets * called by the background worker.
*/ - private static final int RESCAN_INTERVAL = 5000; + private static final int RESCAN_INTERVAL = 1000; /** * Holds the logger of this class (a rate limited logger than doesn't spam the CLI output).
@@ -145,6 +144,7 @@ public void trackLatestSolidMilestone() throws MilestoneException { int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); if (currentSolidRoundIndex < latestMilestoneTracker.getLatestRoundIndex()) { RoundViewModel nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); + //System.out.println("Apply Round " + (currentSolidRoundIndex + 1) + " to Ledger"); if (nextRound != null) { // check solidity of milestones // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? @@ -156,8 +156,8 @@ public void trackLatestSolidMilestone() throws MilestoneException { } } while (!Thread.currentThread().isInterrupted() && allSolid) { - syncValidatorTracker(); - syncLatestMilestoneTracker(nextRound.index()); + //syncValidatorTracker(); + //syncLatestMilestoneTracker(nextRound.index()); applySolidMilestoneToLedger(nextRound); logChange(currentSolidRoundIndex); currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 0368f355..41af7ac4 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -166,8 +166,9 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM } try { - if (RoundViewModel.get(tangle, roundIndex) != null) { + if (RoundViewModel.get(tangle, roundIndex).getHashes().contains(transactionViewModel.getHash())) { // Already validated. + System.out.println("Already validated!"); return VALID; } @@ -194,6 +195,8 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM currentRoundViewModel.addMilestone(transactionViewModel.getHash()); currentRoundViewModel.update(tangle); + //System.out.println("Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); + // if we find a NEW milestone that should have been processed before our latest solid // milestone -> reset the ledger state and check the milestones again // From cf04bd040d71e23cdd92f615d6aae74d623f7dff Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 18 Jun 2019 15:17:06 +0200 Subject: [PATCH 042/223] refactor latestRound to currentRound and fix related issues --- src/main/java/net/helix/hlx/service/API.java | 6 +-- .../milestone/LatestMilestoneTracker.java | 4 +- .../impl/LatestMilestoneTrackerImpl.java | 36 +++++++------ .../impl/LatestSolidMilestoneTrackerImpl.java | 50 +++++++++---------- .../milestone/impl/ValidatorTrackerImpl.java | 2 +- .../impl/LocalSnapshotManagerImpl.java | 2 +- .../snapshot/impl/SnapshotServiceImpl.java | 2 +- .../impl/EntryPointSelectorImpl.java | 2 +- 8 files changed, 49 insertions(+), 55 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 76d42e3a..8a2665a5 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -654,7 +654,7 @@ private AbstractResponse getNodeInfoStatement() throws Exception { System.getProperty("java.version"), Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), - latestMilestoneTracker.getLatestRoundIndex(), + latestMilestoneTracker.getCurrentRoundIndex(), snapshotProvider.getLatestSnapshot().getHash(), snapshotProvider.getLatestSnapshot().getIndex(), @@ -1490,8 +1490,8 @@ private void attachStoreAndBroadcast(final String address, final String message, public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign) throws Exception { // get tips - int latestMilestoneIndex = latestMilestoneTracker.getLatestRoundIndex(); - long nextIndex = latestMilestoneIndex+1; + int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); + long nextIndex = currentRoundIndex; List txToApprove = new ArrayList<>(); txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index a739853e..9bebff31 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -25,7 +25,7 @@ public interface LatestMilestoneTracker { * * @return the index of the latest milestone that was seen by this tracker */ - int getLatestRoundIndex(); + int getCurrentRoundIndex(); /** * Returns the transaction hash of the latest milestone that was seen by this tracker.
@@ -46,7 +46,7 @@ public interface LatestMilestoneTracker { * * @param latestRoundIndex the milestone index of the milestone */ - void setLatestRoundIndex(int latestRoundIndex); + void setCurrentRoundIndex(int latestRoundIndex); void setLatestValidators(Set validatorAddresses); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 38bdff56..371f0782 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -7,7 +7,6 @@ import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; -import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; @@ -92,7 +91,7 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { /** * Holds the round index of the latest round that we have seen / processed.
*/ - private int latestRoundIndex; + private int currentRoundIndex; /** * Holds the transaction hashes of the latest round that we have seen / processed.
@@ -175,22 +174,22 @@ public void addMilestoneToLatestRound(Hash milestoneHash, int latestRoundIndex) this.latestRoundHashes.add(milestoneHash); } @Override - public void setLatestRoundIndex(int roundIndex) { - tangle.publish("lmi %d %d", this.latestRoundIndex, roundIndex); - log.delegate().info("Latest round has changed from #{} to #{}", this.latestRoundIndex, roundIndex); - this.latestRoundIndex = roundIndex; + public void setCurrentRoundIndex(int roundIndex) { + tangle.publish("lmi %d %d", this.currentRoundIndex, roundIndex); + log.delegate().info("Latest round has changed from #{} to #{}", this.currentRoundIndex, roundIndex); + this.currentRoundIndex = roundIndex; } @Override public void setLatestValidators(Set validatorAddresses) { - tangle.publish("lv %d %d", this.latestRoundIndex, validatorAddresses); - log.delegate().info("Validators for round #{}: {}", this.latestRoundIndex, validatorAddresses); + tangle.publish("lv %d %d", this.currentRoundIndex, validatorAddresses); + log.delegate().info("Validators for round #{}: {}", this.currentRoundIndex, validatorAddresses); this.validatorAddresses = validatorAddresses; } @Override - public int getLatestRoundIndex() { - return latestRoundIndex; + public int getCurrentRoundIndex() { + return currentRoundIndex; } @Override @@ -228,7 +227,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validatorAddresses)) { case VALID: - if (roundIndex > latestRoundIndex) { + if (roundIndex == currentRoundIndex) { addMilestoneToLatestRound(transaction.getHash(), roundIndex); } @@ -274,7 +273,7 @@ public boolean isInitialScanComplete() { public void start() { executorService1.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); - executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION * 2, ROUND_DURATION, + executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION, ROUND_DURATION, TimeUnit.MILLISECONDS); } @@ -298,14 +297,13 @@ public void incrementRound() throws MilestoneException{ try { // init new round roundStart = System.currentTimeMillis(); - RoundViewModel currentRoundViewModel = new RoundViewModel(latestRoundIndex + 1, new HashSet<>()); + RoundViewModel currentRoundViewModel = new RoundViewModel(currentRoundIndex + 1, new HashSet<>()); currentRoundViewModel.store(tangle); - System.out.println("Stored round #" + currentRoundViewModel.index()); // clear and increment latest round clearLatestRoundHashes(); - setLatestRoundIndex(latestRoundIndex + 1); + setCurrentRoundIndex(currentRoundIndex + 1); } catch (Exception e) { - throw new MilestoneException("unexpected error while incrementing round #{}" + (latestRoundIndex + 1), e); + throw new MilestoneException("unexpected error while incrementing round #{}" + (currentRoundIndex + 1), e); } } @@ -427,12 +425,12 @@ private void checkIfInitializationComplete() { */ private void bootstrapLatestRoundIndex() { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - setLatestRoundIndex(latestSnapshot.getIndex()); + setCurrentRoundIndex(latestSnapshot.getIndex()); try { RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); - if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getLatestRoundIndex()) { - setLatestRoundIndex(lastMilestoneInDatabase.index()); + if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getCurrentRoundIndex()) { + setCurrentRoundIndex(lastMilestoneInDatabase.index()); } } catch (Exception e) { log.error("unexpectedly failed to retrieve the latest milestone from the database", e); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 1ebf7f08..e6e432b2 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -142,29 +142,25 @@ public void shutdown() { public void trackLatestSolidMilestone() throws MilestoneException { try { int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); - if (currentSolidRoundIndex < latestMilestoneTracker.getLatestRoundIndex()) { - RoundViewModel nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); - //System.out.println("Apply Round " + (currentSolidRoundIndex + 1) + " to Ledger"); - if (nextRound != null) { - // check solidity of milestones - // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? - // TODO: This solution is definitly wrong, we should continue even there are non solid milestones - boolean allSolid = true; - for (Hash milestoneHash : nextRound.getHashes()) { - if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { - allSolid = false; - } - } - while (!Thread.currentThread().isInterrupted() && allSolid) { - //syncValidatorTracker(); - //syncLatestMilestoneTracker(nextRound.index()); - applySolidMilestoneToLedger(nextRound); - logChange(currentSolidRoundIndex); - currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + RoundViewModel nextRound; + while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex() - 1) && + (nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1)) != null) { + // check solidity of milestones + // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? + // TODO: This solution is definitly wrong, we should continue even there are non solid milestones + boolean allSolid = true; + for (Hash milestoneHash : nextRound.getHashes()) { + if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { + allSolid = false; } } - } else { - syncLatestMilestoneTracker(currentSolidRoundIndex); + if (allSolid) { + //syncValidatorTracker(); + //syncLatestMilestoneTracker(nextRound.index()); + applySolidMilestoneToLedger(nextRound); + logChange(currentSolidRoundIndex); + currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + } } } catch (Exception e) { throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); @@ -259,8 +255,8 @@ private void stopRepair() { * @param roundIndex milestone index */ private void syncLatestMilestoneTracker(int roundIndex) { - if(roundIndex > latestMilestoneTracker.getLatestRoundIndex()) { - latestMilestoneTracker.setLatestRoundIndex(roundIndex); + if(roundIndex > latestMilestoneTracker.getCurrentRoundIndex()) { + latestMilestoneTracker.setCurrentRoundIndex(roundIndex); } } @@ -278,13 +274,13 @@ private void syncValidatorTracker() { */ private void logChange(int prevSolidRoundIndex) { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - int latestMilestoneIndex = latestSnapshot.getIndex(); + int latestRoundIndex = latestSnapshot.getIndex(); Hash latestMilestoneHash = latestSnapshot.getHash(); - if (prevSolidRoundIndex != latestMilestoneIndex) { - log.info("Latest SOLID milestone index changed from #" + prevSolidRoundIndex + " to #" + latestMilestoneIndex); + if (prevSolidRoundIndex != latestRoundIndex) { + log.info("Round #{} is SOLID" + latestRoundIndex); - tangle.publish("lmsi %d %d", prevSolidRoundIndex, latestMilestoneIndex); + tangle.publish("lmsi %d %d", prevSolidRoundIndex, latestRoundIndex); tangle.publish("lmhs %s", latestMilestoneHash.hexString()); } } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java index 5752b1c9..ddf0c6a4 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java @@ -96,7 +96,7 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception validation.add(Trustee_Address); switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validation)) { case VALID: - if (roundIndex > latestMilestoneTracker.getLatestRoundIndex()) { + if (roundIndex > latestMilestoneTracker.getCurrentRoundIndex()) { latestValidators = getValidatorAddresses(transaction.getHash()); } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java index e865e4be..0a473b87 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java @@ -118,7 +118,7 @@ public void shutdown() { private void monitorThread(LatestMilestoneTracker latestMilestoneTracker) { while (!Thread.currentThread().isInterrupted()) { int localSnapshotInterval = latestMilestoneTracker.isInitialScanComplete() && - snapshotProvider.getLatestSnapshot().getIndex() == latestMilestoneTracker.getLatestRoundIndex() + snapshotProvider.getLatestSnapshot().getIndex() == latestMilestoneTracker.getCurrentRoundIndex() ? config.getLocalSnapshotsIntervalSynced() : config.getLocalSnapshotsIntervalUnsynced(); diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 30f85b8b..39a674e3 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -300,7 +300,7 @@ public List generateSeenRounds(LatestMilestoneTracker latestMilestoneTr try { RoundViewModel seenRound = targetRound; while ((seenRound = RoundViewModel.findClosestNextRound(tangle, seenRound.index(), - latestMilestoneTracker.getLatestRoundIndex())) != null) { + latestMilestoneTracker.getCurrentRoundIndex())) != null) { seenRounds.add(seenRound.index()); diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index 8b8ba90b..b093c186 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -39,7 +39,7 @@ public Hash getEntryPoint(int depth) throws Exception { int milestoneIndex = Math.max(snapshotProvider.getLatestSnapshot().getIndex() - depth - 1, snapshotProvider.getInitialSnapshot().getIndex()); RoundViewModel roundViewModel = RoundViewModel.findClosestNextRound(tangle, milestoneIndex, - latestMilestoneTracker.getLatestRoundIndex()); + latestMilestoneTracker.getCurrentRoundIndex()); //todo which transaction using as solid entry point when there are multiple milestones / confirmed tips? //temporary solution: select random if (roundViewModel != null && roundViewModel.getHashes() != null) { From 2b32e7b6caa789f0896f08b31b8070c7cc4baa02 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 18 Jun 2019 15:21:23 +0200 Subject: [PATCH 043/223] typo --- .../service/milestone/impl/LatestSolidMilestoneTrackerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index e6e432b2..5838770f 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -278,7 +278,7 @@ private void logChange(int prevSolidRoundIndex) { Hash latestMilestoneHash = latestSnapshot.getHash(); if (prevSolidRoundIndex != latestRoundIndex) { - log.info("Round #{} is SOLID" + latestRoundIndex); + log.info("Round #" + latestRoundIndex + " is SOLID"); tangle.publish("lmsi %d %d", prevSolidRoundIndex, latestRoundIndex); tangle.publish("lmhs %s", latestMilestoneHash.hexString()); From e4e0c980a85cc6fb3b3926af7886701f1f99ea0c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 18 Jun 2019 23:39:39 +0200 Subject: [PATCH 044/223] fix findLatestProcessedSolidRoundInDatabase --- .../ledger/impl/LedgerServiceImpl.java | 7 ++++- .../impl/LatestMilestoneTrackerImpl.java | 1 + .../impl/LatestSolidMilestoneTrackerImpl.java | 4 +++ .../milestone/impl/MilestoneServiceImpl.java | 21 +++++++++++---- .../rocksDB/RocksDBPersistenceProvider.java | 26 +++++++++++++++++++ .../net/helix/hlx/service/MilestoneTest.java | 0 6 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 src/test/java/net/helix/hlx/service/MilestoneTest.java diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index 637dcc95..3b3cb6c0 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -92,7 +92,9 @@ public boolean updateDiff(Set approvedHashes, final Map diff, public void restoreLedgerState() throws LedgerException { try { Optional milestone = milestoneService.findLatestProcessedSolidRoundInDatabase(); + System.out.println(milestone); if (milestone.isPresent()) { + System.out.println(milestone.get().index()); snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.get().index()); } } catch (Exception e) { @@ -102,6 +104,7 @@ public void restoreLedgerState() throws LedgerException { @Override public boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerException { + System.out.println("Apply Round " + milestone.index() + " to ledger"); if(generateStateDiff(milestone)) { try { snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.index()); @@ -168,6 +171,8 @@ public boolean isBalanceDiffConsistent(Set approvedHashes, Map public Map generateBalanceDiff(Set visitedTransactions, Set startTransactions, int milestoneIndex) throws LedgerException { + System.out.println("Generate balance diff for round " + milestoneIndex); + Map state = new HashMap<>(); Set countedTx = new HashSet<>(); @@ -288,7 +293,7 @@ private boolean generateStateDiff(RoundViewModel round) throws LedgerException { try { Set comfirmedTips = milestoneService.getConfirmedTips(round.index()); Map balanceChanges = generateBalanceDiff(new HashSet<>(), comfirmedTips, - snapshotProvider.getLatestSnapshot().getIndex()); + snapshotProvider.getLatestSnapshot().getIndex() + 1); successfullyProcessed = balanceChanges != null; if (successfullyProcessed) { successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 371f0782..91d1088f 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -299,6 +299,7 @@ public void incrementRound() throws MilestoneException{ roundStart = System.currentTimeMillis(); RoundViewModel currentRoundViewModel = new RoundViewModel(currentRoundIndex + 1, new HashSet<>()); currentRoundViewModel.store(tangle); + System.out.println("Store round " + (currentRoundIndex + 1)); // clear and increment latest round clearLatestRoundHashes(); setCurrentRoundIndex(currentRoundIndex + 1); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 5838770f..8c96d9ce 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -145,6 +145,10 @@ public void trackLatestSolidMilestone() throws MilestoneException { RoundViewModel nextRound; while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex() - 1) && (nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1)) != null) { + System.out.println("current solid round: " + currentSolidRoundIndex); + System.out.println("latest round: " + (latestMilestoneTracker.getCurrentRoundIndex() - 1)); + System.out.println("current round: " + (latestMilestoneTracker.getCurrentRoundIndex())); + System.out.println("hashes size: " + nextRound.size()); // check solidity of milestones // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? // TODO: This solution is definitly wrong, we should continue even there are non solid milestones diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 41af7ac4..fbb6efa1 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -112,11 +112,15 @@ public Optional findLatestProcessedSolidRoundInDatabase() throws // if we have no milestone in our database -> abort RoundViewModel latestRound = RoundViewModel.latest(tangle); if (latestRound == null) { + System.out.println("we have no milestone in our database"); return Optional.empty(); } + //System.out.println("Latest Round: " + latestRound.index()); + System.out.println("Was applied to ledger: " + wasRoundAppliedToLedger(latestRound)); // trivial case #1: the node was fully synced if (wasRoundAppliedToLedger(latestRound)) { + System.out.println("the node was fully synced"); return Optional.of(latestRound); } @@ -127,6 +131,8 @@ public Optional findLatestProcessedSolidRoundInDatabase() throws return Optional.of(latestRoundPredecessor); } + System.out.println("Was applied to ledger: " + wasRoundAppliedToLedger(latestRoundPredecessor)); + // non-trivial case: do a binary search in the database return binarySearchLatestProcessedSolidRoundInDatabase(latestRound); } catch (Exception e) { @@ -345,11 +351,16 @@ private RoundViewModel getRoundInMiddleOfRange(int rangeStart, int rangeEnd) thr * @throws Exception if anything unexpected happens while checking the milestone */ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { - //TODO: find a solution for this - /*TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash()); - return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT && - milestoneTransaction.snapshotIndex() != 0;*/ - return true; + //TODO: snapshot index of which milestone should be checked? + if (round.size() > 0) { + TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, (Hash) round.getHashes().toArray()[0]); + System.out.println("round: " + round.index() + ", snapshot: " + milestoneTransaction.snapshotIndex()); + return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT && + milestoneTransaction.snapshotIndex() != 0; + } + else { + return false; + } } /** diff --git a/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java index 5d18fa79..404255e7 100644 --- a/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java @@ -1,6 +1,7 @@ package net.helix.hlx.storage.rocksDB; import net.helix.hlx.model.HashFactory; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.storage.Indexable; import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.PersistenceProvider; @@ -16,6 +17,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.SystemUtils; +import org.bouncycastle.util.encoders.Hex; import org.rocksdb.*; import org.rocksdb.util.SizeUnit; import org.slf4j.Logger; @@ -88,6 +90,14 @@ public boolean save(Persistable thing, Indexable index) throws Exception { if (referenceHandle != null) { db.put(referenceHandle, index.bytes(), thing.metadata()); } + + if (thing.getClass() == Round.class) { + System.out.println("Save " + thing.getClass().getName()); + System.out.println("Indexable Bytes : " + Hex.toHexString(index.bytes())); + System.out.println("Persistable Bytes : " + Hex.toHexString(thing.bytes())); + System.out.println("Persistable Metadata : " + Hex.toHexString(thing.metadata())); + } + return true; } @@ -129,6 +139,14 @@ public Persistable get(Class model, Indexable index) throws Exception { if (referenceHandle != null) { object.readMetadata(db.get(referenceHandle, index == null ? new byte[0] : index.bytes())); } + + if (model == Round.class) { + System.out.println("Get " + model.getName()); + System.out.println("Indexable Bytes : " + Hex.toHexString(index.bytes())); + System.out.println("Persistable Bytes : " + Hex.toHexString(object.bytes())); + System.out.println("Persistable Metadata : " + Hex.toHexString(object.metadata())); + } + return object; } @@ -364,6 +382,14 @@ public boolean update(Persistable thing, Indexable index, String item) throws Ex if (referenceHandle != null) { db.put(referenceHandle, index.bytes(), thing.metadata()); } + + if (thing.getClass() == Round.class) { + System.out.println("Get " + thing.getClass().getName()); + System.out.println("Indexable Bytes : " + Hex.toHexString(index.bytes())); + System.out.println("Persistable Bytes : " + Hex.toHexString(thing.bytes())); + System.out.println("Persistable Metadata : " + Hex.toHexString(thing.metadata())); + } + return false; } diff --git a/src/test/java/net/helix/hlx/service/MilestoneTest.java b/src/test/java/net/helix/hlx/service/MilestoneTest.java new file mode 100644 index 00000000..e69de29b From 675865eacf4e14a4d9f6d80dc0b177f7ed905f5d Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 19 Jun 2019 13:50:14 +0200 Subject: [PATCH 045/223] debug findLatestProcessedSolidRoundInDatabase --- .../impl/LatestMilestoneTrackerImpl.java | 2 ++ .../milestone/impl/MilestoneServiceImpl.java | 17 ++++++++++++-- .../impl/SeenMilestonesRetrieverImpl.java | 8 +++++++ .../java/net/helix/hlx/storage/Tangle.java | 2 +- .../rocksDB/RocksDBPersistenceProvider.java | 23 +++++++++++-------- 5 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 91d1088f..2554cfe5 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -216,6 +216,8 @@ public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneE @Override public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { try { + System.out.println("Process Milestone"); + System.out.println("Hash: " + transaction.getHash().hexString() + ", round: " + milestoneService.getRoundIndex(transaction)); if (validatorAddresses.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { int roundIndex = milestoneService.getRoundIndex(transaction); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index fbb6efa1..cad522ac 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -13,6 +13,7 @@ import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.StateDiff; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.service.ledger.LedgerException; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; @@ -109,13 +110,22 @@ public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvide @Override public Optional findLatestProcessedSolidRoundInDatabase() throws MilestoneException { try { + + // all milestones in db + long count = tangle.getCount(Round.class); + System.out.println("rounds in db: " + count); + for (int i = 1; i < (int) count; i++) { + RoundViewModel round = RoundViewModel.get(tangle, i); + System.out.println("round: " + round.index() + ", size: " + round.size()); + } + // if we have no milestone in our database -> abort RoundViewModel latestRound = RoundViewModel.latest(tangle); if (latestRound == null) { System.out.println("we have no milestone in our database"); return Optional.empty(); } - //System.out.println("Latest Round: " + latestRound.index()); + System.out.println("Latest Round: " + latestRound.index()); System.out.println("Was applied to ledger: " + wasRoundAppliedToLedger(latestRound)); // trivial case #1: the node was fully synced @@ -131,6 +141,7 @@ public Optional findLatestProcessedSolidRoundInDatabase() throws return Optional.of(latestRoundPredecessor); } + System.out.println("Closest Prev Round: " + latestRoundPredecessor.index()); System.out.println("Was applied to ledger: " + wasRoundAppliedToLedger(latestRoundPredecessor)); // non-trivial case: do a binary search in the database @@ -201,7 +212,9 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM currentRoundViewModel.addMilestone(transactionViewModel.getHash()); currentRoundViewModel.update(tangle); - //System.out.println("Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); + System.out.println("Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); + long latest = tangle.getCount(Round.class); + System.out.println("Database number of rounds: " + latest); // if we find a NEW milestone that should have been processed before our latest solid // milestone -> reset the ledger state and check the milestones again diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java index d116a4a0..37bf6d74 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java @@ -115,14 +115,22 @@ public SeenMilestonesRetrieverImpl init(Tangle tangle, SnapshotProvider snapshot */ @Override public void retrieveSeenMilestones() { + System.out.println("Retrieve seen milestones (#" + seenMilestones.size() + ")"); + System.out.println("seen milestones: "); + seenMilestones.forEach(r -> System.out.println(r)); seenMilestones.forEach((roundIndex) -> { try { + System.out.println("round index: " + roundIndex); if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { seenMilestones.remove(roundIndex); } else if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() + RETRIEVE_RANGE) { RoundViewModel round = RoundViewModel.get(tangle, roundIndex); for (Hash milestoneHash : round.getHashes()) { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); + System.out.println("Milestone: " + milestoneHash.hexString()); + System.out.println("Type: " + milestoneTransaction.getType()); + System.out.println("Slot: " + TransactionViewModel.PREFILLED_SLOT); + System.out.println("is requested: " + transactionRequester.isTransactionRequested(milestoneHash, true)); if (milestoneTransaction.getType() == TransactionViewModel.PREFILLED_SLOT && !transactionRequester.isTransactionRequested(milestoneHash, true)) { diff --git a/src/main/java/net/helix/hlx/storage/Tangle.java b/src/main/java/net/helix/hlx/storage/Tangle.java index ef7e3cef..a38ecfa9 100644 --- a/src/main/java/net/helix/hlx/storage/Tangle.java +++ b/src/main/java/net/helix/hlx/storage/Tangle.java @@ -21,7 +21,7 @@ public class Tangle { public static final Map> COLUMN_FAMILIES = new LinkedHashMap>() {{ put("transaction", Transaction.class); - put("milestone", Round.class); + put("round", Round.class); put("stateDiff", StateDiff.class); put("address", Address.class); put("approvee", Approvee.class); diff --git a/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java index 404255e7..771508c4 100644 --- a/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/net/helix/hlx/storage/rocksDB/RocksDBPersistenceProvider.java @@ -91,12 +91,13 @@ public boolean save(Persistable thing, Indexable index) throws Exception { db.put(referenceHandle, index.bytes(), thing.metadata()); } - if (thing.getClass() == Round.class) { + /*if (thing.getClass() == Round.class) { System.out.println("Save " + thing.getClass().getName()); + System.out.println("Indexable (key) : " + index); + System.out.println("Persistable (value) : " + thing); System.out.println("Indexable Bytes : " + Hex.toHexString(index.bytes())); System.out.println("Persistable Bytes : " + Hex.toHexString(thing.bytes())); - System.out.println("Persistable Metadata : " + Hex.toHexString(thing.metadata())); - } + }*/ return true; } @@ -140,12 +141,13 @@ public Persistable get(Class model, Indexable index) throws Exception { object.readMetadata(db.get(referenceHandle, index == null ? new byte[0] : index.bytes())); } - if (model == Round.class) { + /*if (model == Round.class) { System.out.println("Get " + model.getName()); + System.out.println("Indexable (key) : " + index); + System.out.println("Persistable (value) : " + object); System.out.println("Indexable Bytes : " + Hex.toHexString(index.bytes())); System.out.println("Persistable Bytes : " + Hex.toHexString(object.bytes())); - System.out.println("Persistable Metadata : " + Hex.toHexString(object.metadata())); - } + }*/ return object; } @@ -383,12 +385,13 @@ public boolean update(Persistable thing, Indexable index, String item) throws Ex db.put(referenceHandle, index.bytes(), thing.metadata()); } - if (thing.getClass() == Round.class) { - System.out.println("Get " + thing.getClass().getName()); + /*if (thing.getClass() == Round.class) { + System.out.println("Update " + thing.getClass().getName()); + System.out.println("Indexable (key) : " + index); + System.out.println("Persistable (value) : " + thing); System.out.println("Indexable Bytes : " + Hex.toHexString(index.bytes())); System.out.println("Persistable Bytes : " + Hex.toHexString(thing.bytes())); - System.out.println("Persistable Metadata : " + Hex.toHexString(thing.metadata())); - } + }*/ return false; } From 9c5c3c607b486aad52660761463738eb0c640524 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 09:53:53 +0200 Subject: [PATCH 046/223] rename NomineeTracker --- src/main/java/net/helix/hlx/Helix.java | 4 +- .../hlx/service/milestone/NomineeTracker.java | 25 +++ .../milestone/impl/NomineeTrackerImpl.java | 197 ++++++++++++++++++ 3 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java create mode 100644 src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index 6b094624..7fed7c72 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -78,7 +78,7 @@ public class Helix { public final LocalSnapshotManagerImpl localSnapshotManager; public final MilestoneServiceImpl milestoneService; public final LatestMilestoneTrackerImpl latestMilestoneTracker; - public final ValidatorTrackerImpl validatorTracker; + public final NomineeTrackerImpl validatorTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; public final SeenMilestonesRetrieverImpl seenMilestonesRetriever; public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); @@ -124,7 +124,7 @@ public Helix(HelixConfig configuration) throws TransactionPruningException, Snap : null; milestoneService = new MilestoneServiceImpl(); latestMilestoneTracker = new LatestMilestoneTrackerImpl(); - validatorTracker = new ValidatorTrackerImpl(); + validatorTracker = new NomineeTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); diff --git a/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java b/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java new file mode 100644 index 00000000..7f5f0292 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java @@ -0,0 +1,25 @@ +package net.helix.hlx.service.milestone; + +import net.helix.hlx.model.Hash; + +import java.util.Set; + +public interface NomineeTracker { + + public Set getLatestValidators(); + + Set getValidatorsOfRound(int roundIndex) throws Exception; + + boolean processTrusteeTransaction(Hash transactionHash) throws Exception; + + Set getValidatorAddresses(Hash transaction) throws Exception; + + void analyzeTrusteeTransactions() throws Exception; + + void collectNewTrusteeTransactions() throws Exception; + + void start(); + + void shutdown(); + +} diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java new file mode 100644 index 00000000..7af8f28b --- /dev/null +++ b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java @@ -0,0 +1,197 @@ +package net.helix.hlx.service.milestone.impl; + +import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.controllers.AddressViewModel; +import net.helix.hlx.controllers.BundleViewModel; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.SpongeFactory; +import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; +import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneService; +import net.helix.hlx.service.milestone.MilestoneSolidifier; +import net.helix.hlx.service.milestone.NomineeTracker; +import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.storage.Tangle; +import net.helix.hlx.utils.log.interval.IntervalLogger; +import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; +import net.helix.hlx.utils.thread.SilentScheduledExecutorService; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +public class NomineeTrackerImpl implements NomineeTracker { + + private static final int MAX_CANDIDATES_TO_ANALYZE = 5000; + private static final int RESCAN_INTERVAL = 1000; + private Hash Trustee_Address; + + private static final IntervalLogger log = new IntervalLogger(LatestMilestoneTrackerImpl.class); + private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Validator Tracker", log.delegate()); + + private Tangle tangle; + private LatestMilestoneTracker latestMilestoneTracker; + private SnapshotProvider snapshotProvider; + private MilestoneService milestoneService; + private MilestoneSolidifier milestoneSolidifier; + private Set latestValidators = new HashSet<>(); + private final Set seenTrusteeTransactions = new HashSet<>(); + private final Deque trusteeTransactionsToAnalyze = new ArrayDeque<>(); + + public NomineeTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, HelixConfig config) { + + this.tangle = tangle; + this.latestMilestoneTracker = latestMilestoneTracker; + this.Trustee_Address = config.getTrusteeAddress(); + this.snapshotProvider = snapshotProvider; + this.milestoneService = milestoneService; + this.milestoneSolidifier = milestoneSolidifier; + //bootstrapLatestValidators(); + + return this; + } + + @Override + public Set getLatestValidators() { + return latestValidators; + } + + + @Override + public Set getValidatorsOfRound(int roundIndex) throws Exception{ + try { + Set validators = new HashSet<>(); + for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { + TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, hash); + if (milestoneService.getRoundIndex(transaction) == roundIndex) { + validators = getValidatorAddresses(hash); + } + } + return validators; + }catch (Exception e) { + throw new Exception("unexpected error while getting Validators of round #{}" + roundIndex, e); + } + } + + @Override + public boolean processTrusteeTransaction(Hash transactionHash) throws Exception { + TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); + try { + if (Trustee_Address.equals(transaction.getAddressHash())) { + + int roundIndex = milestoneService.getRoundIndex(transaction); + + // if the trustee transaction is older than our ledger start point: we already processed it in the past + if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { + return true; + } + + // we use milestone validation because its the same process (TODO: Rename function and implement more general) + Set validation = new HashSet<>(); + validation.add(Trustee_Address); + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validation)) { + case VALID: + if (roundIndex > latestMilestoneTracker.getCurrentRoundIndex()) { + latestValidators = getValidatorAddresses(transaction.getHash()); + } + + if (!transaction.isSolid()) { + milestoneSolidifier.add(transaction.getHash(), roundIndex); + } + + break; + + case INCOMPLETE: + milestoneSolidifier.add(transaction.getHash(), roundIndex); + + return false; + + default: + } + }else { + return false; + } + return true; + } catch (Exception e) { + throw new Exception("unexpected error while processing trustee transaction " + transaction, e); + } + } + + @Override + public Set getValidatorAddresses(Hash transaction) throws Exception{ + TransactionViewModel tail = TransactionViewModel.fromHash(tangle, transaction); + BundleViewModel bundle = BundleViewModel.load(tangle, tail.getBundleHash()); + int security = 1; + Set validators = new HashSet<>(); + + for (Hash txHash : bundle.getHashes()) { + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, txHash); + // get transactions with validator addresses in signatureMessageFragment + // -> we have to count from last index + // n-security-1 to n: tx with signature + // n-security-1: tx with merkle path + // 0 to n-security: tx with validator addresses + if ((tx.getCurrentIndex() <= 0) && (tx.getCurrentIndex() < bundle.size() - security - 1)) { + for (int i = 0; i < TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE / Hash.SIZE_IN_BYTES; i++) { + Hash address = HashFactory.ADDRESS.create(Arrays.copyOfRange(tx.getSignature(), i * Hash.SIZE_IN_BYTES, (i+1) * Hash.SIZE_IN_BYTES)); + // TODO: Do we send all validators or only adding and removing ones + if (address.equals(Hash.NULL_HASH)){ + break; + } + validators.add(address); + } + } + } + return validators; + } + + @Override + public void analyzeTrusteeTransactions() throws Exception { + int transactionsToAnalyze = Math.min(trusteeTransactionsToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); + for (int i = 0; i < transactionsToAnalyze; i++) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Hash trusteeTransactionHash = trusteeTransactionsToAnalyze.pollFirst(); + if(!processTrusteeTransaction(trusteeTransactionHash)) { + seenTrusteeTransactions.remove(trusteeTransactionHash); + } + } + } + + @Override + public void collectNewTrusteeTransactions() throws Exception { + try { + for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { + if (Thread.currentThread().isInterrupted()) { + return; + } + if (seenTrusteeTransactions.add(hash)) { + trusteeTransactionsToAnalyze.addFirst(hash); + } + } + } catch (Exception e) { + throw new Exception("failed to collect the new trustee transactions", e); + } + } + + private void validatorTrackerThread() { + try { + collectNewTrusteeTransactions(); + analyzeTrusteeTransactions(); + } catch (Exception e) { + log.error("error while scaning for trustee transactions", e); + } + } + + public void start(){ + executorService.silentScheduleWithFixedDelay(this::validatorTrackerThread, 0, RESCAN_INTERVAL, + TimeUnit.MILLISECONDS); + } + + public void shutdown(){ + executorService.shutdownNow(); + } +} From 42f09229a10d9c6fb687c5a48d3a716aea3cbe02 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 09:57:57 +0200 Subject: [PATCH 047/223] fix load, get tips from correct transaction(s), implement toString() --- .../helix/hlx/controllers/RoundViewModel.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 761fecf1..1e3f6ed2 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -1,11 +1,9 @@ package net.helix.hlx.controllers; -import net.helix.hlx.BundleValidator; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.persistables.Round; -import net.helix.hlx.model.persistables.Transaction; import net.helix.hlx.storage.Indexable; import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.Tangle; @@ -59,7 +57,7 @@ public static void clear(int index) { public RoundViewModel(final int index, final Set milestoneHashes) { this.round = new Round(); this.round.index = new IntegerIndex(index); - round.set = milestoneHashes; + this.round.set = milestoneHashes; } /** @@ -93,7 +91,7 @@ public static RoundViewModel get(Tangle tangle, int index) throws Exception { */ public static boolean load(Tangle tangle, int index) throws Exception { Round round = (Round) tangle.load(Round.class, new IntegerIndex(index)); - if(round != null && round.set != null) { + if (round != null) { rounds.put(index, new RoundViewModel(round)); return true; } @@ -224,10 +222,10 @@ public Set getTipSet(Tangle tangle, Hash milestone) throws Exception { for (Hash bundleTxHash : bundle.getHashes()) { TransactionViewModel bundleTx = TransactionViewModel.fromHash(tangle, bundleTxHash); - if ((bundleTx.getCurrentIndex() >= 0) && (bundleTx.getCurrentIndex() < bundle.size() - security - 1)) { + if ((bundleTx.getCurrentIndex() > bundle.size() - security - 1)) { for (int i = 0; i < 16; i++) { - Hash tip = HashFactory.TRANSACTION.create(bundleTx.getSignature(), i * Hash.SIZE_IN_BYTES, (i + 1) * Hash.SIZE_IN_BYTES); + Hash tip = HashFactory.TRANSACTION.create(bundleTx.getSignature(), i * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); if (tip.equals(Hash.NULL_HASH)) { break; } @@ -235,6 +233,12 @@ public Set getTipSet(Tangle tangle, Hash milestone) throws Exception { } } } + System.out.println("Tip Set:"); + if (tips.size() == 0){ + System.out.println("empty!"); + } else { + tips.forEach(tip -> System.out.println(tip.hexString())); + } return tips; } @@ -254,10 +258,17 @@ public Set getConfirmedTips(Tangle tangle) throws Exception { } } } - return occurrences.entrySet().stream() + Set tips = occurrences.entrySet().stream() .filter(entry -> entry.getValue() >= quorum) .map(entry -> entry.getKey()) .collect(Collectors.toSet()); + System.out.println("Confirmed Tips:"); + if (tips.size() == 0){ + System.out.println("empty!"); + } else { + tips.forEach(tip -> System.out.println(tip.hexString())); + } + return tips; } public Set getConfirmingMilestones(Tangle tangle) throws Exception { @@ -266,15 +277,21 @@ public Set getConfirmingMilestones(Tangle tangle) throws Exception { for (Hash milestoneHash : getHashes()) { Set tips = getTipSet(tangle, milestoneHash); - if (confirmedTips.equals(tips)) { + if (confirmedTips.equals(tips) || (confirmedTips.isEmpty() && tips.isEmpty())) { confirmingMilestones.add(milestoneHash); } } + System.out.println("Confirming Milestones"); + if (confirmingMilestones.size() == 0){ + System.out.println("empty!"); + } else { + confirmingMilestones.forEach(ms -> System.out.println(ms.hexString())); + } return confirmingMilestones; } public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { - Set confirmingMilestones = getConfirmingMilestones(tangle); + Set confirmingMilestones = getHashes(); // todo getConfirmingMilestones(tangle); int item = new Random().nextInt(confirmingMilestones.size()); int i = 0; for(Hash obj : confirmingMilestones) { @@ -338,6 +355,7 @@ public void delete(Tangle tangle) throws Exception { */ @Override public String toString() { - return "milestone #" + index() + " (" + getHashes().toString() + ")"; + String hashes = getHashes().stream().map(Hash::hexString).collect(Collectors.joining(",")); + return "round #" + index() + " (" + hashes + ")"; } } From 4307708da94d59ddb0cb50e16cf84df5fc0dd40b Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 10:02:15 +0200 Subject: [PATCH 048/223] update snapshot index of milestones, fix already validated check (not sure if correct, review this) --- .../milestone/impl/MilestoneServiceImpl.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index cad522ac..e9bba908 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -111,14 +111,6 @@ public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvide public Optional findLatestProcessedSolidRoundInDatabase() throws MilestoneException { try { - // all milestones in db - long count = tangle.getCount(Round.class); - System.out.println("rounds in db: " + count); - for (int i = 1; i < (int) count; i++) { - RoundViewModel round = RoundViewModel.get(tangle, i); - System.out.println("round: " + round.index() + ", size: " + round.size()); - } - // if we have no milestone in our database -> abort RoundViewModel latestRound = RoundViewModel.latest(tangle); if (latestRound == null) { @@ -183,7 +175,9 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM } try { - if (RoundViewModel.get(tangle, roundIndex).getHashes().contains(transactionViewModel.getHash())) { + //todo don't know if this is correct + RoundViewModel round = RoundViewModel.get(tangle, roundIndex); + if (round != null && !round.getHashes().isEmpty()) { // Already validated. System.out.println("Already validated!"); return VALID; @@ -199,7 +193,8 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM final TransactionViewModel tail = bundleTransactionViewModels.get(0); // milestone transaction with signature if (tail.getHash().equals(transactionViewModel.getHash())) { - if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { + //todo implement when sure how bundle structure has to look like + //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); @@ -208,13 +203,13 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { - RoundViewModel currentRoundViewModel = RoundViewModel.get(tangle, roundIndex); + /*RoundViewModel currentRoundViewModel = RoundViewModel.get(tangle, roundIndex); currentRoundViewModel.addMilestone(transactionViewModel.getHash()); currentRoundViewModel.update(tangle); System.out.println("Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); long latest = tangle.getCount(Round.class); - System.out.println("Database number of rounds: " + latest); + System.out.println("Database number of rounds: " + latest);*/ // if we find a NEW milestone that should have been processed before our latest solid // milestone -> reset the ledger state and check the milestones again @@ -231,7 +226,6 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM } else { return INVALID; } - } } } } @@ -389,9 +383,18 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, Set processedTransactions) throws MilestoneException { + System.out.println("UPDATE ROUND INDEX"); Set inconsistentMilestones = new HashSet<>(); try { + // update milestones + RoundViewModel round = RoundViewModel.get(tangle, newIndex); + for (Hash milestoneHash : round.getHashes()){ + TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); + updateRoundIndexOfSingleTransaction(milestoneTx, newIndex); + System.out.println("milestone: " + milestoneHash.hexString() + ", Snapshot: " + milestoneTx.snapshotIndex()); + } + // update confirmed transactions final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex)); Hash transactionPointer; while ((transactionPointer = transactionsToUpdate.poll()) != null) { From 6beab53bfa4bf0c6bb6f1eb0e7a67c4802daedfe Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 10:06:55 +0200 Subject: [PATCH 049/223] bootstrap current round index after ledger state is restored, rename NomineeTracker --- .../impl/LatestSolidMilestoneTrackerImpl.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 8c96d9ce..6ae57194 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -8,7 +8,7 @@ import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; import net.helix.hlx.service.milestone.LatestSolidMilestoneTracker; -import net.helix.hlx.service.milestone.ValidatorTracker; +import net.helix.hlx.service.milestone.NomineeTracker; import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; @@ -65,7 +65,7 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac */ private LedgerService ledgerService; - private ValidatorTracker validatorTracker; + private NomineeTracker validatorTracker; /** * Holds a reference to the manager of the background worker.
@@ -109,7 +109,7 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac */ public LatestSolidMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, LedgerService ledgerService, - LatestMilestoneTracker latestMilestoneTracker, ValidatorTracker validatorTracker) { + LatestMilestoneTracker latestMilestoneTracker, NomineeTracker validatorTracker) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; @@ -145,10 +145,7 @@ public void trackLatestSolidMilestone() throws MilestoneException { RoundViewModel nextRound; while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex() - 1) && (nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1)) != null) { - System.out.println("current solid round: " + currentSolidRoundIndex); - System.out.println("latest round: " + (latestMilestoneTracker.getCurrentRoundIndex() - 1)); - System.out.println("current round: " + (latestMilestoneTracker.getCurrentRoundIndex())); - System.out.println("hashes size: " + nextRound.size()); + // check solidity of milestones // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? // TODO: This solution is definitly wrong, we should continue even there are non solid milestones @@ -183,6 +180,7 @@ private void latestSolidMilestoneTrackerThread() { firstRun = false; ledgerService.restoreLedgerState(); + latestMilestoneTracker.bootstrapCurrentRoundIndex(); logChange(snapshotProvider.getInitialSnapshot().getIndex()); } From 768a8cd11c2f9cd15e30e093559711dd8e2865d9 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 10:09:17 +0200 Subject: [PATCH 050/223] make bootstrapCurrentRoundIndex public, store milestone at the end of the round --- .../net/helix/hlx/service/milestone/LatestMilestoneTracker.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index 9bebff31..b9b94653 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -93,4 +93,6 @@ public interface LatestMilestoneTracker { * This method stops the background worker that updates the latest milestones.
*/ void shutdown(); + + void bootstrapCurrentRoundIndex(); } From 89e6aa92da25e960f6bf9ea0941ddc16c9145a30 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 10:10:11 +0200 Subject: [PATCH 051/223] make bootstrapCurrentRoundIndex public, store milestone at the end of the round --- .../impl/LatestMilestoneTrackerImpl.java | 73 ++++++++++--------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 2554cfe5..d0281835 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -18,10 +18,7 @@ import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; import net.helix.hlx.utils.thread.SilentScheduledExecutorService; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; /** @@ -96,7 +93,7 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { /** * Holds the transaction hashes of the latest round that we have seen / processed.
*/ - private Set latestRoundHashes; + private Set latestRoundHashes = new HashSet<>(); /** * A set that allows us to keep track of the candidates that have been seen and added to the {@link @@ -151,11 +148,10 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP validatorAddresses = new HashSet<>(); validatorAddresses.add(HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a")); - latestRoundHashes = new HashSet<>(); roundStart = System.currentTimeMillis(); - bootstrapLatestRoundIndex(); + //bootstrapLatestRoundIndex(); return this; } @@ -275,7 +271,7 @@ public boolean isInitialScanComplete() { public void start() { executorService1.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); - executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION, ROUND_DURATION, + executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION*2, ROUND_DURATION, TimeUnit.MILLISECONDS); } @@ -297,11 +293,18 @@ public void roundCounter(){ public void incrementRound() throws MilestoneException{ try { + System.out.println("INCREMENT ROUND"); // init new round roundStart = System.currentTimeMillis(); - RoundViewModel currentRoundViewModel = new RoundViewModel(currentRoundIndex + 1, new HashSet<>()); - currentRoundViewModel.store(tangle); - System.out.println("Store round " + (currentRoundIndex + 1)); + if (currentRoundIndex > 0) { + System.out.println("Round hashes: " + getLatestRoundHashes().size()); + RoundViewModel currentRoundViewModel = new RoundViewModel(currentRoundIndex, getLatestRoundHashes()); + currentRoundViewModel.store(tangle); + System.out.println("Store round " + currentRoundIndex); + } + boolean stored = RoundViewModel.load(tangle, currentRoundIndex); + System.out.println("stored: " + stored); + // clear and increment latest round clearLatestRoundHashes(); setCurrentRoundIndex(currentRoundIndex + 1); @@ -310,6 +313,32 @@ public void incrementRound() throws MilestoneException{ } } + /** + * This method bootstraps this tracker with the latest milestone values that can easily be retrieved without + * analyzing any transactions (for faster startup).
+ *
+ * It first sets the latest milestone to the values found in the latest snapshot and then check if there is a younger + * milestone at the end of our database. While this last entry in the database doesn't necessarily have to be the + * latest one we know it at least gives a reasonable value most of the times.
+ */ + @Override + public void bootstrapCurrentRoundIndex() { + Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); + setCurrentRoundIndex(latestSnapshot.getIndex() + 1); + + try { + RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); + if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getCurrentRoundIndex()) { + setCurrentRoundIndex(lastMilestoneInDatabase.index()); + for (Hash milestoneHash : lastMilestoneInDatabase.getHashes()) { + addMilestoneToLatestRound(milestoneHash, lastMilestoneInDatabase.index()); + } + } + } catch (Exception e) { + log.error("unexpectedly failed to retrieve the latest milestone from the database", e); + } + } + /** * This method contains the logic for scanning for new latest milestones that gets executed in a background @@ -417,26 +446,4 @@ private void checkIfInitializationComplete() { log.info("Processing milestone candidates ... [DONE]").triggerOutput(true); } } - - /** - * This method bootstraps this tracker with the latest milestone values that can easily be retrieved without - * analyzing any transactions (for faster startup).
- *
- * It first sets the latest milestone to the values found in the latest snapshot and then check if there is a younger - * milestone at the end of our database. While this last entry in the database doesn't necessarily have to be the - * latest one we know it at least gives a reasonable value most of the times.
- */ - private void bootstrapLatestRoundIndex() { - Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - setCurrentRoundIndex(latestSnapshot.getIndex()); - - try { - RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); - if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getCurrentRoundIndex()) { - setCurrentRoundIndex(lastMilestoneInDatabase.index()); - } - } catch (Exception e) { - log.error("unexpectedly failed to retrieve the latest milestone from the database", e); - } - } } From 63226a2105508da9c22807229afce8aafdb99bc7 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 10:11:31 +0200 Subject: [PATCH 052/223] log confirmation status when doing balance diff --- .../net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index 3b3cb6c0..e6c350fb 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -189,6 +189,7 @@ public Map generateBalanceDiff(Set visitedTransactions, Set Date: Wed, 26 Jun 2019 10:13:51 +0200 Subject: [PATCH 053/223] add tips transaction to milestone bundle --- src/main/java/net/helix/hlx/service/API.java | 50 ++++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 8a2665a5..d4c2dffb 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1496,20 +1496,26 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); - // A milestone consists of two transactions. - // The last transaction (currentIndex == lastIndex) contains the siblings for the merkle tree. + // list of confirming tips + byte[] txTips = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(2L), 0, txTips, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(2L), 0, txTips, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txTips, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + + // siblings for merkle tree. byte[] txSibling = new byte[TransactionViewModel.SIZE]; System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(2L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - // The other transactions contain a signature that signs the siblings and thereby ensures the integrity. + // contain a signature that signs the siblings and thereby ensures the integrity. byte[] txMilestone = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, txMilestone, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(1L), 0, txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(2L), 0, txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txMilestone, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); System.arraycopy(Serializer.serialize(nextIndex), 0, txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + // calculate bundle hash Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); @@ -1517,9 +1523,12 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri sponge.absorb(milestoneEssence, 0, milestoneEssence.length); byte[] siblingEssence = Arrays.copyOfRange(txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); sponge.absorb(siblingEssence, 0, siblingEssence.length); + byte[] tipsEssence = Arrays.copyOfRange(txTips, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(tipsEssence, 0, tipsEssence.length); byte[] bundleHash = new byte[32]; sponge.squeeze(bundleHash, 0, bundleHash.length); + System.arraycopy(bundleHash, 0, txTips, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); System.arraycopy(bundleHash, 0, txMilestone, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); @@ -1543,8 +1552,39 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri System.arraycopy(signature, 0, txMilestone, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); } + // get confirming tips + List confirmingTips = new LinkedList<>(); + + System.out.println("Tips (" + tipsViewModel.getTips().size() + ")"); + snapshotProvider.getLatestSnapshot().lockRead(); + try { + WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); + for (Hash transaction : tipsViewModel.getTips()) { + System.out.println(transaction.hexString()); + TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); + if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && + txVM.getCurrentIndex() == 0 && + txVM.isSolid() && + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { + if (walkValidator.isValid(transaction)) { + System.out.println("confirmed!"); + confirmingTips.add(transaction); + } + } + } + } finally { + snapshotProvider.getLatestSnapshot().unlockRead(); + } + + + // write confirming tips into signature message fragment of txTips + for (int i=0; i transactions = new ArrayList<>(); + transactions.add(Hex.toHexString(txTips)); transactions.add(Hex.toHexString(txSibling)); transactions.add(Hex.toHexString(txMilestone)); List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); From 4ff9a79b385f8be9c940ecde9f3d0aa04abf9ce1 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 17:28:33 +0200 Subject: [PATCH 054/223] hardcode address for new nominee keyfile, access nominee keyfile for signing milestones --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- src/main/java/net/helix/hlx/service/API.java | 2 +- src/main/java/net/helix/hlx/service/milestone/MSS.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index e0824bb4..d6b8450e 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -902,7 +902,7 @@ public interface Defaults { String PREVIOUS_EPOCHS_SPENT_ADDRESSES_SIG = "/previousEpochsSpentAddresses.sig"; long GLOBAL_SNAPSHOT_TIME = 1522235533L; int MILESTONE_START_INDEX = 0; - int NUM_KEYS_IN_MILESTONE = 15; + int NUM_KEYS_IN_MILESTONE = 10; int MAX_ANALYZED_TXS = 20_000; //Logging diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index d4c2dffb..82ece85e 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1535,7 +1535,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri if (sign) { // Get merkle path and store in signatureMessageFragment of Sibling Transaction StringBuilder seedBuilder = new StringBuilder(); - byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Coordinator.key"), seedBuilder); + byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); String seed = seedBuilder.toString(), coordinatorAddress = Hex.toHexString(merkleTree[merkleTree.length - 1][0]); // create merkle path from keyfile byte[] merklePath = Merkle.getMerklePath(merkleTree, (int) nextIndex); diff --git a/src/main/java/net/helix/hlx/service/milestone/MSS.java b/src/main/java/net/helix/hlx/service/milestone/MSS.java index baa2ac1d..8b35400a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MSS.java +++ b/src/main/java/net/helix/hlx/service/milestone/MSS.java @@ -31,7 +31,7 @@ public MSS(HelixConfig configuration, API api) { int minDelay = this.config.getMinDelay(); this.mwm = this.config.getMwm(); this.message = StringUtils.repeat('0', 1024); - this.address = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; + this.address = "6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"; this.sign = !this.config.isDontValidateTestnetMilestoneSig(); if(this.delay < minDelay) { From c50b80f2714f52b50c96066fe674f5a284240e41 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 17:31:59 +0200 Subject: [PATCH 055/223] store milestone when validated --- .../impl/LatestMilestoneTrackerImpl.java | 25 ++++++++++--------- .../milestone/impl/MilestoneServiceImpl.java | 20 +++++++++++---- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index d0281835..93dee51b 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -83,7 +83,7 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { private final SilentScheduledExecutorService executorService2 = new DedicatedScheduledExecutorService( "Round Counter", log.delegate()); - private long roundStart; + private long genesisTime; /** * Holds the round index of the latest round that we have seen / processed.
@@ -146,10 +146,13 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP this.milestoneSolidifier = milestoneSolidifier; validatorAddresses = new HashSet<>(); - validatorAddresses.add(HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a")); + validatorAddresses.add(HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db")); + validatorAddresses.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); - roundStart = System.currentTimeMillis(); + genesisTime = System.currentTimeMillis(); + //System.out.println("current time: " + System.currentTimeMillis()); + //System.out.println("current round: " + getRound(System.currentTimeMillis())); //bootstrapLatestRoundIndex(); @@ -295,15 +298,13 @@ public void incrementRound() throws MilestoneException{ try { System.out.println("INCREMENT ROUND"); // init new round - roundStart = System.currentTimeMillis(); - if (currentRoundIndex > 0) { - System.out.println("Round hashes: " + getLatestRoundHashes().size()); - RoundViewModel currentRoundViewModel = new RoundViewModel(currentRoundIndex, getLatestRoundHashes()); - currentRoundViewModel.store(tangle); - System.out.println("Store round " + currentRoundIndex); - } - boolean stored = RoundViewModel.load(tangle, currentRoundIndex); - System.out.println("stored: " + stored); + genesisTime = System.currentTimeMillis(); + //System.out.println("Round hashes: " + getLatestRoundHashes().size()); + //RoundViewModel currentRoundViewModel = new RoundViewModel(currentRoundIndex + 1, new HashSet<>()); + //currentRoundViewModel.store(tangle); + //System.out.println("Store round " + (currentRoundIndex + 1)); + //boolean stored = RoundViewModel.load(tangle, currentRoundIndex + 1); + //System.out.println("stored: " + stored); // clear and increment latest round clearLatestRoundHashes(); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index e9bba908..c66b75c8 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -203,13 +203,23 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { - /*RoundViewModel currentRoundViewModel = RoundViewModel.get(tangle, roundIndex); - currentRoundViewModel.addMilestone(transactionViewModel.getHash()); - currentRoundViewModel.update(tangle); + log.info("Milestone " + transactionViewModel.getHash().hexString() + " is valid!"); + + RoundViewModel currentRoundViewModel; + if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { + currentRoundViewModel.addMilestone(transactionViewModel.getHash()); + currentRoundViewModel.update(tangle); + System.out.println("Update: Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); + } else { + Set milestones = new HashSet<>(); + milestones.add(transactionViewModel.getHash()); + currentRoundViewModel = new RoundViewModel(roundIndex, milestones); + currentRoundViewModel.store(tangle); + System.out.println("Store: Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); + } - System.out.println("Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); long latest = tangle.getCount(Round.class); - System.out.println("Database number of rounds: " + latest);*/ + System.out.println("Database number of rounds: " + latest); // if we find a NEW milestone that should have been processed before our latest solid // milestone -> reset the ledger state and check the milestones again From 5d7e015aab45651986735e389e569ef2988f9236 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 17:34:37 +0200 Subject: [PATCH 056/223] return false when trying to load a round without milestones (not sure if this is correct, review this) --- .../helix/hlx/controllers/RoundViewModel.java | 2 +- .../service/milestone/ValidatorTracker.java | 25 --- .../milestone/impl/ValidatorTrackerImpl.java | 201 ------------------ 3 files changed, 1 insertion(+), 227 deletions(-) delete mode 100644 src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java delete mode 100644 src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 1e3f6ed2..df9b4aa0 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -91,7 +91,7 @@ public static RoundViewModel get(Tangle tangle, int index) throws Exception { */ public static boolean load(Tangle tangle, int index) throws Exception { Round round = (Round) tangle.load(Round.class, new IntegerIndex(index)); - if (round != null) { + if (round != null && !round.set.isEmpty()) { rounds.put(index, new RoundViewModel(round)); return true; } diff --git a/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java b/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java deleted file mode 100644 index c8c7a472..00000000 --- a/src/main/java/net/helix/hlx/service/milestone/ValidatorTracker.java +++ /dev/null @@ -1,25 +0,0 @@ -package net.helix.hlx.service.milestone; - -import net.helix.hlx.model.Hash; - -import java.util.Set; - -public interface ValidatorTracker { - - public Set getLatestValidators(); - - Set getValidatorsOfRound(int roundIndex) throws Exception; - - boolean processTrusteeTransaction(Hash transactionHash) throws Exception; - - Set getValidatorAddresses(Hash transaction) throws Exception; - - void analyzeTrusteeTransactions() throws Exception; - - void collectNewTrusteeTransactions() throws Exception; - - void start(); - - void shutdown(); - -} diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java deleted file mode 100644 index ddf0c6a4..00000000 --- a/src/main/java/net/helix/hlx/service/milestone/impl/ValidatorTrackerImpl.java +++ /dev/null @@ -1,201 +0,0 @@ -package net.helix.hlx.service.milestone.impl; - -import net.helix.hlx.conf.HelixConfig; -import net.helix.hlx.controllers.AddressViewModel; -import net.helix.hlx.controllers.BundleViewModel; -import net.helix.hlx.controllers.RoundViewModel; -import net.helix.hlx.controllers.TransactionViewModel; -import net.helix.hlx.crypto.SpongeFactory; -import net.helix.hlx.model.Hash; -import net.helix.hlx.model.HashFactory; -import net.helix.hlx.model.IntegerIndex; -import net.helix.hlx.model.persistables.Round; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; -import net.helix.hlx.service.milestone.MilestoneService; -import net.helix.hlx.service.milestone.MilestoneSolidifier; -import net.helix.hlx.service.milestone.ValidatorTracker; -import net.helix.hlx.service.snapshot.Snapshot; -import net.helix.hlx.service.snapshot.SnapshotProvider; -import net.helix.hlx.storage.Tangle; -import net.helix.hlx.utils.log.interval.IntervalLogger; -import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; -import net.helix.hlx.utils.thread.SilentScheduledExecutorService; - -import java.util.*; -import java.util.concurrent.TimeUnit; - -public class ValidatorTrackerImpl implements ValidatorTracker { - - private static final int MAX_CANDIDATES_TO_ANALYZE = 5000; - private static final int RESCAN_INTERVAL = 1000; - private Hash Trustee_Address; - - private static final IntervalLogger log = new IntervalLogger(LatestMilestoneTrackerImpl.class); - private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Validator Tracker", log.delegate()); - - private Tangle tangle; - private LatestMilestoneTracker latestMilestoneTracker; - private SnapshotProvider snapshotProvider; - private MilestoneService milestoneService; - private MilestoneSolidifier milestoneSolidifier; - private Set latestValidators = new HashSet<>(); - private final Set seenTrusteeTransactions = new HashSet<>(); - private final Deque trusteeTransactionsToAnalyze = new ArrayDeque<>(); - - public ValidatorTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, HelixConfig config) { - - this.tangle = tangle; - this.latestMilestoneTracker = latestMilestoneTracker; - this.Trustee_Address = config.getTrusteeAddress(); - this.snapshotProvider = snapshotProvider; - this.milestoneService = milestoneService; - this.milestoneSolidifier = milestoneSolidifier; - //bootstrapLatestValidators(); - - return this; - } - - @Override - public Set getLatestValidators() { - return latestValidators; - } - - - @Override - public Set getValidatorsOfRound(int roundIndex) throws Exception{ - try { - Set validators = new HashSet<>(); - for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { - TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, hash); - if (milestoneService.getRoundIndex(transaction) == roundIndex) { - validators = getValidatorAddresses(hash); - } - } - return validators; - }catch (Exception e) { - throw new Exception("unexpected error while getting Validators of round #{}" + roundIndex, e); - } - } - - @Override - public boolean processTrusteeTransaction(Hash transactionHash) throws Exception { - TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); - try { - if (Trustee_Address.equals(transaction.getAddressHash())) { - - int roundIndex = milestoneService.getRoundIndex(transaction); - - // if the trustee transaction is older than our ledger start point: we already processed it in the past - if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { - return true; - } - - // we use milestone validation because its the same process (TODO: Rename function and implement more general) - Set validation = new HashSet<>(); - validation.add(Trustee_Address); - switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validation)) { - case VALID: - if (roundIndex > latestMilestoneTracker.getCurrentRoundIndex()) { - latestValidators = getValidatorAddresses(transaction.getHash()); - } - - if (!transaction.isSolid()) { - milestoneSolidifier.add(transaction.getHash(), roundIndex); - } - - break; - - case INCOMPLETE: - milestoneSolidifier.add(transaction.getHash(), roundIndex); - - return false; - - default: - } - }else { - return false; - } - return true; - } catch (Exception e) { - throw new Exception("unexpected error while processing trustee transaction " + transaction, e); - } - } - - @Override - public Set getValidatorAddresses(Hash transaction) throws Exception{ - TransactionViewModel tail = TransactionViewModel.fromHash(tangle, transaction); - BundleViewModel bundle = BundleViewModel.load(tangle, tail.getBundleHash()); - int security = 1; - Set validators = new HashSet<>(); - - for (Hash txHash : bundle.getHashes()) { - TransactionViewModel tx = TransactionViewModel.fromHash(tangle, txHash); - // get transactions with validator addresses in signatureMessageFragment - // -> we have to count from last index - // n-security-1 to n: tx with signature - // n-security-1: tx with merkle path - // 0 to n-security: tx with validator addresses - if ((tx.getCurrentIndex() <= 0) && (tx.getCurrentIndex() < bundle.size() - security - 1)) { - for (int i = 0; i < TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE / Hash.SIZE_IN_BYTES; i++) { - Hash address = HashFactory.ADDRESS.create(Arrays.copyOfRange(tx.getSignature(), i * Hash.SIZE_IN_BYTES, (i+1) * Hash.SIZE_IN_BYTES)); - // TODO: Do we send all validators or only adding and removing ones - if (address.equals(Hash.NULL_HASH)){ - break; - } - validators.add(address); - } - } - } - return validators; - } - - @Override - public void analyzeTrusteeTransactions() throws Exception { - int transactionsToAnalyze = Math.min(trusteeTransactionsToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); - for (int i = 0; i < transactionsToAnalyze; i++) { - if (Thread.currentThread().isInterrupted()) { - return; - } - - Hash trusteeTransactionHash = trusteeTransactionsToAnalyze.pollFirst(); - if(!processTrusteeTransaction(trusteeTransactionHash)) { - seenTrusteeTransactions.remove(trusteeTransactionHash); - } - } - } - - @Override - public void collectNewTrusteeTransactions() throws Exception { - try { - for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { - if (Thread.currentThread().isInterrupted()) { - return; - } - if (seenTrusteeTransactions.add(hash)) { - trusteeTransactionsToAnalyze.addFirst(hash); - } - } - } catch (Exception e) { - throw new Exception("failed to collect the new trustee transactions", e); - } - } - - private void validatorTrackerThread() { - try { - collectNewTrusteeTransactions(); - analyzeTrusteeTransactions(); - } catch (Exception e) { - log.error("error while scaning for trustee transactions", e); - } - } - - public void start(){ - executorService.silentScheduleWithFixedDelay(this::validatorTrackerThread, 0, RESCAN_INTERVAL, - TimeUnit.MILLISECONDS); - } - - public void shutdown(){ - executorService.shutdownNow(); - } -} From 0186b59a646b2586f8663b61169b54d38c0f12fa Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 26 Jun 2019 18:02:46 +0200 Subject: [PATCH 057/223] get round depending on time instead of getting it from roundCounter --- .../milestone/impl/LatestMilestoneTrackerImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 93dee51b..d203cc12 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -150,7 +150,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP validatorAddresses.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); - genesisTime = System.currentTimeMillis(); + genesisTime = 1561563357884L; //System.out.println("current time: " + System.currentTimeMillis()); //System.out.println("current round: " + getRound(System.currentTimeMillis())); @@ -191,6 +191,10 @@ public int getCurrentRoundIndex() { return currentRoundIndex; } + public int getRound(long time) { + return (int) (time - genesisTime) / ROUND_DURATION; + } + @Override public Set getLatestRoundHashes() { return this.latestRoundHashes; } @@ -217,6 +221,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw try { System.out.println("Process Milestone"); System.out.println("Hash: " + transaction.getHash().hexString() + ", round: " + milestoneService.getRoundIndex(transaction)); + System.out.println("Current Round: " + getRound(System.currentTimeMillis())); if (validatorAddresses.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { int roundIndex = milestoneService.getRoundIndex(transaction); @@ -228,7 +233,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validatorAddresses)) { case VALID: - if (roundIndex == currentRoundIndex) { + if (roundIndex == getRound(System.currentTimeMillis())) { addMilestoneToLatestRound(transaction.getHash(), roundIndex); } @@ -274,8 +279,7 @@ public boolean isInitialScanComplete() { public void start() { executorService1.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); - executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION*2, ROUND_DURATION, - TimeUnit.MILLISECONDS); + //executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION*2, ROUND_DURATION, TimeUnit.MILLISECONDS); } From a56f928640d64eaba9e9b94958c15812b6803cbd Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 11:53:13 +0200 Subject: [PATCH 058/223] get milestone of a given address --- .../helix/hlx/controllers/RoundViewModel.java | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index df9b4aa0..be6c5b0a 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -8,6 +8,7 @@ import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Pair; +import org.bouncycastle.util.encoders.Hex; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -78,6 +79,17 @@ public static RoundViewModel get(Tangle tangle, int index) throws Exception { return roundViewModel; } + public static TransactionViewModel getMilestone(Tangle tangle, int index, Hash address) throws Exception { + RoundViewModel roundViewModel = get(tangle, index); + for (Hash milestoneHash : roundViewModel.getHashes()){ + TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); + if (milestoneTx.getAddressHash() == address){ + return milestoneTx; + } + } + return null; + } + /** * Fetches a {@link Round} object from the database using its integer index. If the {@link Round} and the * associated {@link Hash} identifier are not null, a new {@link RoundViewModel} is created for the @@ -91,7 +103,8 @@ public static RoundViewModel get(Tangle tangle, int index) throws Exception { */ public static boolean load(Tangle tangle, int index) throws Exception { Round round = (Round) tangle.load(Round.class, new IntegerIndex(index)); - if (round != null && !round.set.isEmpty()) { + // todo didn't really get this, because tangle.load never returns null + if (round != null && round.index != null) { rounds.put(index, new RoundViewModel(round)); return true; } @@ -233,12 +246,6 @@ public Set getTipSet(Tangle tangle, Hash milestone) throws Exception { } } } - System.out.println("Tip Set:"); - if (tips.size() == 0){ - System.out.println("empty!"); - } else { - tips.forEach(tip -> System.out.println(tip.hexString())); - } return tips; } @@ -262,12 +269,6 @@ public Set getConfirmedTips(Tangle tangle) throws Exception { .filter(entry -> entry.getValue() >= quorum) .map(entry -> entry.getKey()) .collect(Collectors.toSet()); - System.out.println("Confirmed Tips:"); - if (tips.size() == 0){ - System.out.println("empty!"); - } else { - tips.forEach(tip -> System.out.println(tip.hexString())); - } return tips; } @@ -281,12 +282,6 @@ public Set getConfirmingMilestones(Tangle tangle) throws Exception { confirmingMilestones.add(milestoneHash); } } - System.out.println("Confirming Milestones"); - if (confirmingMilestones.size() == 0){ - System.out.println("empty!"); - } else { - confirmingMilestones.forEach(ms -> System.out.println(ms.hexString())); - } return confirmingMilestones; } From 571805f47f170f2ddffdea602f0db8111374363e Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 11:58:13 +0200 Subject: [PATCH 059/223] define genesis time, delete round tracker, store milestone after valid and timestamp belongs to correct round --- .../impl/LatestMilestoneTrackerImpl.java | 78 +++++++++---------- .../milestone/impl/MilestoneServiceImpl.java | 27 ++----- 2 files changed, 44 insertions(+), 61 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index d203cc12..5bcf1e30 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -80,8 +80,6 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { private final SilentScheduledExecutorService executorService1 = new DedicatedScheduledExecutorService( "Latest Milestone Tracker", log.delegate()); - private final SilentScheduledExecutorService executorService2 = new DedicatedScheduledExecutorService( - "Round Counter", log.delegate()); private long genesisTime; @@ -150,8 +148,9 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP validatorAddresses.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); - genesisTime = 1561563357884L; - //System.out.println("current time: " + System.currentTimeMillis()); + genesisTime = System.currentTimeMillis(); + //currentRoundIndex = getRound(System.currentTimeMillis()); + System.out.println("current time: " + System.currentTimeMillis()); //System.out.println("current round: " + getRound(System.currentTimeMillis())); //bootstrapLatestRoundIndex(); @@ -188,7 +187,7 @@ public void setLatestValidators(Set validatorAddresses) { @Override public int getCurrentRoundIndex() { - return currentRoundIndex; + return getRound(System.currentTimeMillis()); } public int getRound(long time) { @@ -219,9 +218,9 @@ public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneE @Override public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { try { - System.out.println("Process Milestone"); - System.out.println("Hash: " + transaction.getHash().hexString() + ", round: " + milestoneService.getRoundIndex(transaction)); - System.out.println("Current Round: " + getRound(System.currentTimeMillis())); + //System.out.println("Process Milestone"); + //System.out.println("Hash: " + transaction.getHash().hexString() + ", round: " + milestoneService.getRoundIndex(transaction)); + //System.out.println("Current Round: " + getCurrentRoundIndex()); if (validatorAddresses.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { int roundIndex = milestoneService.getRoundIndex(transaction); @@ -233,8 +232,38 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validatorAddresses)) { case VALID: - if (roundIndex == getRound(System.currentTimeMillis())) { - addMilestoneToLatestRound(transaction.getHash(), roundIndex); + // TODO: 1.review conditions, 2.is okay to store here? + // before a milestone can be added to a round the following conditions have to be checked: + // - index is bigger than initial snapshot (above) + // - VALID: + // - senderAddress must be part of validator addresses + // - signature belongs to senderAddress + // - index is bigger than snapshot index + // - attachment timestamp is in correct time window for the index + // - there doesn't already exist a milestone with the same address for that round + if (roundIndex == getRound(transaction.getAttachmentTimestamp())) { + + RoundViewModel currentRoundViewModel; + + // there already arrived a milestone for that round, just update + if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { + // check if there is already a milestone with the same address + if (currentRoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { + currentRoundViewModel.addMilestone(transaction.getHash()); + currentRoundViewModel.update(tangle); + log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex + "(" + currentRoundViewModel.size() + ")"); + } else { + log.error("Round #" + roundIndex + " has already received a milestone from " + transaction.getAddressHash().hexString()); + } + } + // this is the first milestone for that round, make new database entry + else { + Set milestones = new HashSet<>(); + milestones.add(transaction.getHash()); + currentRoundViewModel = new RoundViewModel(roundIndex, milestones); + currentRoundViewModel.store(tangle); + log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex); + } } if (!transaction.isSolid()) { @@ -279,7 +308,6 @@ public boolean isInitialScanComplete() { public void start() { executorService1.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); - //executorService2.silentScheduleWithFixedDelay(this::roundCounter, ROUND_DURATION*2, ROUND_DURATION, TimeUnit.MILLISECONDS); } @@ -290,34 +318,6 @@ public void shutdown() { } - public void roundCounter(){ - try { - incrementRound(); - }catch (Exception e) { - log.error("error while incrementing round", e); - } - } - - public void incrementRound() throws MilestoneException{ - try { - System.out.println("INCREMENT ROUND"); - // init new round - genesisTime = System.currentTimeMillis(); - //System.out.println("Round hashes: " + getLatestRoundHashes().size()); - //RoundViewModel currentRoundViewModel = new RoundViewModel(currentRoundIndex + 1, new HashSet<>()); - //currentRoundViewModel.store(tangle); - //System.out.println("Store round " + (currentRoundIndex + 1)); - //boolean stored = RoundViewModel.load(tangle, currentRoundIndex + 1); - //System.out.println("stored: " + stored); - - // clear and increment latest round - clearLatestRoundHashes(); - setCurrentRoundIndex(currentRoundIndex + 1); - } catch (Exception e) { - throw new MilestoneException("unexpected error while incrementing round #{}" + (currentRoundIndex + 1), e); - } - } - /** * This method bootstraps this tracker with the latest milestone values that can easily be retrieved without * analyzing any transactions (for faster startup).
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index c66b75c8..00d8142c 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -199,30 +199,13 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); - // TODO: we should also check here if there is already a milestone with the same address if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { - log.info("Milestone " + transactionViewModel.getHash().hexString() + " is valid!"); - - RoundViewModel currentRoundViewModel; - if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { - currentRoundViewModel.addMilestone(transactionViewModel.getHash()); - currentRoundViewModel.update(tangle); - System.out.println("Update: Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); - } else { - Set milestones = new HashSet<>(); - milestones.add(transactionViewModel.getHash()); - currentRoundViewModel = new RoundViewModel(roundIndex, milestones); - currentRoundViewModel.store(tangle); - System.out.println("Store: Milestone " + transactionViewModel.getHash().hexString() + " is stored in round #" + roundIndex); - } - - long latest = tangle.getCount(Round.class); - System.out.println("Database number of rounds: " + latest); - // if we find a NEW milestone that should have been processed before our latest solid - // milestone -> reset the ledger state and check the milestones again + // if we find a NEW milestone for a round that already has been processed + // and considered as solid (there is already a snapshot without this milestone) + // -> reset the ledger state and check the milestones again // // NOTE: this can happen if a new subtangle becomes solid before a previous one while // syncing @@ -393,7 +376,7 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, Set processedTransactions) throws MilestoneException { - System.out.println("UPDATE ROUND INDEX"); + //System.out.println("UPDATE ROUND INDEX"); Set inconsistentMilestones = new HashSet<>(); try { @@ -402,7 +385,7 @@ private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIn for (Hash milestoneHash : round.getHashes()){ TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); updateRoundIndexOfSingleTransaction(milestoneTx, newIndex); - System.out.println("milestone: " + milestoneHash.hexString() + ", Snapshot: " + milestoneTx.snapshotIndex()); + //System.out.println("milestone: " + milestoneHash.hexString() + ", Snapshot: " + milestoneTx.snapshotIndex()); } // update confirmed transactions final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex)); From 10c399c2e9f76e68350db2b38186b58980354e50 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 11:59:28 +0200 Subject: [PATCH 060/223] delete second executor service --- .../service/milestone/impl/LatestMilestoneTrackerImpl.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 5bcf1e30..4b70c582 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -77,7 +77,7 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { /** * Holds a reference to the manager of the background worker.
*/ - private final SilentScheduledExecutorService executorService1 = new DedicatedScheduledExecutorService( + private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( "Latest Milestone Tracker", log.delegate()); @@ -306,15 +306,14 @@ public boolean isInitialScanComplete() { */ @Override public void start() { - executorService1.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, + executorService.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); } @Override public void shutdown() { - executorService1.shutdownNow(); - executorService2.shutdownNow(); + executorService.shutdownNow(); } From 5e8c677c0e7fe7b7614496fb5da59b8eccccf4d2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 12:02:51 +0200 Subject: [PATCH 061/223] enable incrementing index --- src/main/java/net/helix/hlx/service/API.java | 7 +++---- src/main/java/net/helix/hlx/service/milestone/MSS.java | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 82ece85e..7e3092eb 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1487,11 +1487,11 @@ private void attachStoreAndBroadcast(final String address, final String message, broadcastTransactionsStatement(powResult); } - public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign) throws Exception { + public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int incrementIndex) throws Exception { // get tips int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); - long nextIndex = currentRoundIndex; + long nextIndex = currentRoundIndex + incrementIndex; List txToApprove = new ArrayList<>(); txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); @@ -1560,14 +1560,13 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri try { WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); for (Hash transaction : tipsViewModel.getTips()) { - System.out.println(transaction.hexString()); TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && txVM.getCurrentIndex() == 0 && txVM.isSolid() && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { if (walkValidator.isValid(transaction)) { - System.out.println("confirmed!"); + System.out.println(transaction.hexString()); confirmingTips.add(transaction); } } diff --git a/src/main/java/net/helix/hlx/service/milestone/MSS.java b/src/main/java/net/helix/hlx/service/milestone/MSS.java index 8b35400a..1c39579c 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MSS.java +++ b/src/main/java/net/helix/hlx/service/milestone/MSS.java @@ -47,7 +47,8 @@ public void startScheduledExecutorService() { private void publishMilestone() throws Exception { log.info("Publishing next Milestone..."); - this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign); + this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, 0); + //this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, 2); } private Runnable getRunnablePublishMilestone() { From abb859090f49c28fed97c543f2a8637eb31fbde6 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 12:13:56 +0200 Subject: [PATCH 062/223] bootstrap current round happens in latest milestone tracker --- .../service/milestone/impl/LatestSolidMilestoneTrackerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 6ae57194..c8a1fc8a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -180,7 +180,7 @@ private void latestSolidMilestoneTrackerThread() { firstRun = false; ledgerService.restoreLedgerState(); - latestMilestoneTracker.bootstrapCurrentRoundIndex(); + //latestMilestoneTracker.bootstrapCurrentRoundIndex(); logChange(snapshotProvider.getInitialSnapshot().getIndex()); } From 36c34b819020953496484e952599301859ebd572 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 12:14:23 +0200 Subject: [PATCH 063/223] prints --- .../hlx/service/ledger/impl/LedgerServiceImpl.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index e6c350fb..873560d4 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -104,7 +104,7 @@ public void restoreLedgerState() throws LedgerException { @Override public boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerException { - System.out.println("Apply Round " + milestone.index() + " to ledger"); + //System.out.println("Apply Round " + milestone.index() + " to ledger"); if(generateStateDiff(milestone)) { try { snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.index()); @@ -171,7 +171,7 @@ public boolean isBalanceDiffConsistent(Set approvedHashes, Map public Map generateBalanceDiff(Set visitedTransactions, Set startTransactions, int milestoneIndex) throws LedgerException { - System.out.println("Generate balance diff for round " + milestoneIndex); + //System.out.println("Generate balance diff for round " + milestoneIndex); Map state = new HashMap<>(); Set countedTx = new HashSet<>(); @@ -189,7 +189,7 @@ public Map generateBalanceDiff(Set visitedTransactions, Set comfirmedTips = milestoneService.getConfirmedTips(round.index()); - Map balanceChanges = generateBalanceDiff(new HashSet<>(), comfirmedTips, + Set confirmedTips = milestoneService.getConfirmedTips(round.index()); + System.out.println("Confirmed Tips:"); + confirmedTips.forEach(tip -> System.out.println(tip.hexString())); + Map balanceChanges = generateBalanceDiff(new HashSet<>(), confirmedTips == null? new HashSet<>() : confirmedTips, snapshotProvider.getLatestSnapshot().getIndex() + 1); successfullyProcessed = balanceChanges != null; if (successfullyProcessed) { From cacb62c7d1d8bc823cabb00b1de5ebc68fb09075 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 12:34:49 +0200 Subject: [PATCH 064/223] implement getLatestRoundHashes --- .../service/milestone/LatestMilestoneTracker.java | 2 +- .../impl/LatestMilestoneTrackerImpl.java | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index b9b94653..ce535921 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -34,7 +34,7 @@ public interface LatestMilestoneTracker { * * @return the transaction hash of the latest milestone that was seen by this tracker */ - Set getLatestRoundHashes(); + Set getLatestRoundHashes() throws Exception; /** * Sets the latest milestone.
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 4b70c582..f3fd7ab0 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -7,6 +7,7 @@ import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; @@ -195,7 +196,19 @@ public int getRound(long time) { } @Override - public Set getLatestRoundHashes() { return this.latestRoundHashes; } + public Set getLatestRoundHashes() throws Exception{ + int index = getCurrentRoundIndex(); + try { + RoundViewModel currentRound = RoundViewModel.get(tangle, index); + if (currentRound == null) { + return null; + } else { + return currentRound.getHashes(); + } + } catch (Exception e) { + throw new MilestoneException("unexpected error while getting latest milestones (#" + index + ")", e); + } + } @Override public void clearLatestRoundHashes() {this.latestRoundHashes.clear();} From 3b86f16de069fad28aeb2957d6a61474d22f95a3 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 30 Jun 2019 12:35:58 +0200 Subject: [PATCH 065/223] make normal tip selection for branch and trunc --- src/main/java/net/helix/hlx/service/API.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 7e3092eb..e975d7df 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1493,8 +1493,13 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); long nextIndex = currentRoundIndex + incrementIndex; List txToApprove = new ArrayList<>(); - txToApprove.add(Hash.NULL_HASH); - txToApprove.add(Hash.NULL_HASH); + System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); + if(latestMilestoneTracker.getCurrentRoundIndex() == 1) { + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); + } else { + txToApprove = getTransactionToApproveTips(3, Optional.empty()); + } // list of confirming tips byte[] txTips = new byte[TransactionViewModel.SIZE]; From 9c80fc45f42987d112750cab71807275e8db08a2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 1 Jul 2019 13:36:33 +0200 Subject: [PATCH 066/223] move on to round with milestones --- src/main/java/net/helix/hlx/controllers/RoundViewModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index be6c5b0a..a7ae575d 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -218,7 +218,7 @@ public static RoundViewModel findClosestNextRound(Tangle tangle, int index, int // search for the next milestone following our index RoundViewModel nextRoundViewModel = null; int currentIndex = index; - while(nextRoundViewModel == null && ++currentIndex <= maxIndex) { + while((nextRoundViewModel == null || nextRoundViewModel.getHashes().isEmpty()) && ++currentIndex <= maxIndex ) { nextRoundViewModel = RoundViewModel.get(tangle, currentIndex); } From e7d6c3e6dd106cf36d4cc9ae092ecbf597f74c80 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 1 Jul 2019 13:38:46 +0200 Subject: [PATCH 067/223] check if there is already a milestone with the same address --- .../service/milestone/impl/LatestMilestoneTrackerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index f3fd7ab0..da5a3a6e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -149,7 +149,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP validatorAddresses.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); - genesisTime = System.currentTimeMillis(); + genesisTime = 1561979153933L; //currentRoundIndex = getRound(System.currentTimeMillis()); System.out.println("current time: " + System.currentTimeMillis()); //System.out.println("current round: " + getRound(System.currentTimeMillis())); @@ -261,7 +261,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw // there already arrived a milestone for that round, just update if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { // check if there is already a milestone with the same address - if (currentRoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { + if (RoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { currentRoundViewModel.addMilestone(transaction.getHash()); currentRoundViewModel.update(tangle); log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex + "(" + currentRoundViewModel.size() + ")"); From 2fcad9bf57808bcdf3f2c418958c934c84dbda2b Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 1 Jul 2019 13:39:59 +0200 Subject: [PATCH 068/223] check if round has finished without milestones --- .../impl/LatestSolidMilestoneTrackerImpl.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index c8a1fc8a..c8dcddb8 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -16,6 +16,7 @@ import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; import net.helix.hlx.utils.thread.SilentScheduledExecutorService; +import java.util.HashSet; import java.util.concurrent.TimeUnit; /** @@ -143,8 +144,23 @@ public void trackLatestSolidMilestone() throws MilestoneException { try { int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); RoundViewModel nextRound; - while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex() - 1) && - (nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1)) != null) { + while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex() - 1)) { + + nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); + + if (nextRound == null) { + // round has finished without milestones + RoundViewModel latest = RoundViewModel.latest(tangle); + if (latest != null && latest.index() > currentSolidRoundIndex + 1) { + nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); + nextRound.store(tangle); + } + // round hasn't finished yet + else { + break; + } + } + // check solidity of milestones // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? From c04035e5337bd6aa7c1ad0f525833f1549fbe54d Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 1 Jul 2019 13:42:23 +0200 Subject: [PATCH 069/223] if there is no latest round take null hash as branch and trunk --- src/main/java/net/helix/hlx/service/API.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index e975d7df..a9655921 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -622,6 +622,7 @@ public void storeTransactionsStatement(final List txHex) throws Exceptio transactionValidator.updateStatus(transactionViewModel); transactionViewModel.updateSender("local"); transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); + System.out.println("published tx: " + transactionViewModel.getHash().hexString()); } if (graph != null) { @@ -1494,7 +1495,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri long nextIndex = currentRoundIndex + incrementIndex; List txToApprove = new ArrayList<>(); System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); - if(latestMilestoneTracker.getCurrentRoundIndex() == 1) { + if(RoundViewModel.latest(tangle) == null) { txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); } else { From 57fe1f9fabe65dc5467f69a066dbdcf0777943c2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 1 Jul 2019 13:44:44 +0200 Subject: [PATCH 070/223] check if round has milestones --- .../hlx/service/tipselection/impl/EntryPointSelectorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index b093c186..9bab0d64 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -42,7 +42,7 @@ public Hash getEntryPoint(int depth) throws Exception { latestMilestoneTracker.getCurrentRoundIndex()); //todo which transaction using as solid entry point when there are multiple milestones / confirmed tips? //temporary solution: select random - if (roundViewModel != null && roundViewModel.getHashes() != null) { + if (roundViewModel != null && !roundViewModel.getHashes().isEmpty()) { return roundViewModel.getRandomConfirmingMilestone(tangle); } From 4a81056ba3826db6c617e1e2e97a96416002e49f Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 1 Jul 2019 13:45:20 +0200 Subject: [PATCH 071/223] show confirmed status in graphstream --- .../net/helix/hlx/service/Graphstream.java | 32 ++++++++++++------- .../ledger/impl/LedgerServiceImpl.java | 21 ++++-------- .../service/milestone/MilestoneService.java | 4 ++- .../milestone/impl/MilestoneServiceImpl.java | 18 +++++++---- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/Graphstream.java b/src/main/java/net/helix/hlx/service/Graphstream.java index 2f567a58..481a2113 100644 --- a/src/main/java/net/helix/hlx/service/Graphstream.java +++ b/src/main/java/net/helix/hlx/service/Graphstream.java @@ -20,23 +20,31 @@ public void setLabel(boolean toLabel) { this.LABEL = toLabel; } - public void setValidity(String hash, Integer validity) { - Node graphNode = graph.getNode(hash); - if (validity.equals(-1)){ - graphNode.addAttribute("ui.style", "fill-color: rgb(255,0,0);"); - } - else { - if (validity.equals(0)) { - graphNode.addAttribute("ui.style", "fill-color: rgb(255,165,0);"); + public void setConfirmed(String hash, Integer validity) { + try { + Node graphNode = graph.getNode(hash); + if (validity.equals(-1)) { + graphNode.addAttribute("ui.style", "fill-color: rgb(255,0,0);"); } else { - graphNode.addAttribute("ui.style", "fill-color: rgb(0,255,0);"); + if (validity.equals(0)) { + graphNode.addAttribute("ui.style", "fill-color: rgb(255,165,0);"); + } else { + graphNode.addAttribute("ui.style", "fill-color: rgb(0,255,0);"); + } } + } catch(Exception e) { + log.error("couldn't find a node with label " + hash, e); } } - public void setMilestone(String hash) { - Node graphNode = graph.getNode(hash); - graphNode.addAttribute("ui.style", "size: 20px; stroke-mode: plain;"); + public void setMilestone(String hash, int index) { + try { + Node graphNode = graph.getNode(hash); + graphNode.addAttribute("ui.style", "fill-color: rgb(30,144,255); size: 20px; stroke-mode: plain;"); + graphNode.addAttribute("ui.label", index); + } catch(Exception e) { + log.error("couldn't find a node with label " + hash, e); + } } public void addNode(String hash, String branch, String trunk) { diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index 873560d4..4de8ecd0 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -104,7 +104,12 @@ public void restoreLedgerState() throws LedgerException { @Override public boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerException { - //System.out.println("Apply Round " + milestone.index() + " to ledger"); + System.out.println("Apply Round " + milestone.index() + " to ledger"); + if (graph != null) { + for (Hash milestoneHash : milestone.getHashes()) { + graph.setMilestone(milestoneHash.hexString(), milestone.index()); + } + } if(generateStateDiff(milestone)) { try { snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.index()); @@ -200,20 +205,8 @@ public Map generateBalanceDiff(Set visitedTransactions, Set> bundleTransactions = BundleValidator.validate( tangle, snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); - //TODO: graphstream - if (graph != null) { - graph.setValidity(transactionViewModel.getHash().hexString(), transactionViewModel.getValidity()); - } for (final List bundleTransactionViewModels : bundleTransactions) { - //TODO: graphstream - if (graph != null) { - for (final TransactionViewModel bundleTransactionViewModel : bundleTransactionViewModels) { - graph.setValidity(bundleTransactionViewModel.getHash().hexString(), bundleTransactionViewModel.getValidity()); - } - } - - if (BundleValidator.isInconsistent(bundleTransactionViewModels)) { break; } @@ -302,7 +295,7 @@ private boolean generateStateDiff(RoundViewModel round) throws LedgerException { successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); if (successfullyProcessed) { - milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); + milestoneService.updateRoundIndexOfMilestoneTransactions(round.index(), graph); if (!balanceChanges.isEmpty()) { new StateDiffViewModel(balanceChanges, round.index()).store(tangle); diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index 277703f6..83066310 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -4,10 +4,12 @@ import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; +import net.helix.hlx.service.Graphstream; import java.util.Optional; import java.util.Set; + /** * Represents the service that contains all the relevant business logic for interacting with milestones.
*
@@ -65,7 +67,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i * @param newIndex the milestone index that shall be set * @throws MilestoneException if anything unexpected happens while updating the milestone index */ - void updateRoundIndexOfMilestoneTransactions(int newIndex) throws MilestoneException; + void updateRoundIndexOfMilestoneTransactions(int newIndex, Graphstream graph) throws MilestoneException; /** * Resets all milestone related information of the transactions that were "confirmed" by the given milestone and diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 00d8142c..f33d01c2 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -14,6 +14,7 @@ import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.StateDiff; import net.helix.hlx.model.persistables.Round; +import net.helix.hlx.service.Graphstream; import net.helix.hlx.service.ledger.LedgerException; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; @@ -24,6 +25,7 @@ import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Serializer; import net.helix.hlx.utils.dag.DAGHelper; +import net.helix.hlx.service.Graphstream; import net.helix.hlx.utils.dag.TraversalException; import org.bouncycastle.util.encoders.Hex; @@ -145,13 +147,13 @@ public Optional findLatestProcessedSolidRoundInDatabase() throws } @Override - public void updateRoundIndexOfMilestoneTransactions(int index) throws MilestoneException { + public void updateRoundIndexOfMilestoneTransactions(int index, Graphstream graph) throws MilestoneException { if (index <= 0) { throw new MilestoneException("the new index needs to be bigger than 0 " + "(use resetCorruptedMilestone to reset the milestone index)"); } - updateRoundIndexOfMilestoneTransactions(index, index, new HashSet<>()); + updateRoundIndexOfMilestoneTransactions(index, index, new HashSet<>(), graph); } /** @@ -374,9 +376,9 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { * @param processedTransactions a set of transactions that have been processed already (for the recursive calls) */ private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, - Set processedTransactions) throws MilestoneException { + Set processedTransactions, Graphstream graph) throws MilestoneException { - //System.out.println("UPDATE ROUND INDEX"); + System.out.println("UPDATE ROUND INDEX"); Set inconsistentMilestones = new HashSet<>(); try { @@ -385,7 +387,7 @@ private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIn for (Hash milestoneHash : round.getHashes()){ TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); updateRoundIndexOfSingleTransaction(milestoneTx, newIndex); - //System.out.println("milestone: " + milestoneHash.hexString() + ", Snapshot: " + milestoneTx.snapshotIndex()); + System.out.println("milestone: " + milestoneHash.hexString() + ", Snapshot: " + milestoneTx.snapshotIndex()); } // update confirmed transactions final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex)); @@ -400,6 +402,10 @@ private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIn prepareRoundIndexUpdate(transactionViewModel, correctIndex, newIndex, inconsistentMilestones, transactionsToUpdate); updateRoundIndexOfSingleTransaction(transactionViewModel, newIndex); + System.out.println("tx: " + transactionViewModel.getHash().hexString() + ", Snapshot: " + transactionViewModel.snapshotIndex()); + if (graph != null) { + graph.setConfirmed(transactionViewModel.getHash().hexString(), 1); + } if (!transactionsToUpdate.contains(transactionViewModel.getTrunkTransactionHash())) { transactionsToUpdate.offer(transactionViewModel.getTrunkTransactionHash()); } @@ -550,7 +556,7 @@ private void resetCorruptedRound(int index, Set processedTransactions) thr snapshotService.rollBackMilestones(snapshotProvider.getLatestSnapshot(), roundToRepair.index()); } updateRoundIndexOfMilestoneTransactions(roundToRepair.index(), 0, - processedTransactions); + processedTransactions, new Graphstream()); tangle.delete(StateDiff.class, new IntegerIndex(roundToRepair.index())); } } catch (Exception e) { From 7b0dc930cb1fa43dbaaabf44c822a62f27ba5e54 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 5 Jul 2019 12:10:53 +0200 Subject: [PATCH 072/223] comment bug --- .../hlx/service/tipselection/impl/EntryPointSelectorImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index 9bab0d64..f879a199 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -41,6 +41,7 @@ public Hash getEntryPoint(int depth) throws Exception { RoundViewModel roundViewModel = RoundViewModel.findClosestNextRound(tangle, milestoneIndex, latestMilestoneTracker.getCurrentRoundIndex()); //todo which transaction using as solid entry point when there are multiple milestones / confirmed tips? + //todo sometimes produces error here because entry point is not consistent (not sure under what conditions) //temporary solution: select random if (roundViewModel != null && !roundViewModel.getHashes().isEmpty()) { return roundViewModel.getRandomConfirmingMilestone(tangle); From b404f832e54d5b7f9063493117d5e50008200f77 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 5 Jul 2019 12:12:21 +0200 Subject: [PATCH 073/223] if round already contains hash -> already validated --- .../helix/hlx/service/milestone/impl/MilestoneServiceImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index f33d01c2..225d5330 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -177,9 +177,8 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM } try { - //todo don't know if this is correct RoundViewModel round = RoundViewModel.get(tangle, roundIndex); - if (round != null && !round.getHashes().isEmpty()) { + if (round != null && round.getHashes().contains(transactionViewModel.getHash())) { // Already validated. System.out.println("Already validated!"); return VALID; From 7db124e3bc6a93696690c7e680a6508c92978cb8 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 5 Jul 2019 12:13:39 +0200 Subject: [PATCH 074/223] get correct transactions in bundle --- src/main/java/net/helix/hlx/controllers/RoundViewModel.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index a7ae575d..679d2420 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -235,7 +235,7 @@ public Set getTipSet(Tangle tangle, Hash milestone) throws Exception { for (Hash bundleTxHash : bundle.getHashes()) { TransactionViewModel bundleTx = TransactionViewModel.fromHash(tangle, bundleTxHash); - if ((bundleTx.getCurrentIndex() > bundle.size() - security - 1)) { + if ((bundleTx.getCurrentIndex() > security)) { for (int i = 0; i < 16; i++) { Hash tip = HashFactory.TRANSACTION.create(bundleTx.getSignature(), i * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); @@ -346,7 +346,6 @@ public void delete(Tangle tangle) throws Exception { * It can be used to directly append the milestone in error and debug messages. * * @return human readable string representation of the milestone - * TODO: Either get a specific milestone from the set and cast that to string or cast the whole set to string in a loop and print. */ @Override public String toString() { From a1407e6c603c2332588443f7d41c7fef54191e2c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 5 Jul 2019 12:14:20 +0200 Subject: [PATCH 075/223] integrate nomineeTracker --- src/main/java/net/helix/hlx/Helix.java | 10 +- .../milestone/LatestMilestoneTracker.java | 4 +- .../hlx/service/milestone/NomineeTracker.java | 14 +- .../impl/LatestMilestoneTrackerImpl.java | 75 +++++---- .../impl/LatestSolidMilestoneTrackerImpl.java | 4 +- .../milestone/impl/NomineeTrackerImpl.java | 146 ++++++++++++------ 6 files changed, 163 insertions(+), 90 deletions(-) diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index 7fed7c72..45e89f71 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -78,7 +78,7 @@ public class Helix { public final LocalSnapshotManagerImpl localSnapshotManager; public final MilestoneServiceImpl milestoneService; public final LatestMilestoneTrackerImpl latestMilestoneTracker; - public final NomineeTrackerImpl validatorTracker; + public final NomineeTrackerImpl nomineeTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; public final SeenMilestonesRetrieverImpl seenMilestonesRetriever; public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); @@ -124,7 +124,7 @@ public Helix(HelixConfig configuration) throws TransactionPruningException, Snap : null; milestoneService = new MilestoneServiceImpl(); latestMilestoneTracker = new LatestMilestoneTrackerImpl(); - validatorTracker = new NomineeTrackerImpl(); + nomineeTracker = new NomineeTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); @@ -210,10 +210,10 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); } milestoneService.init(tangle, snapshotProvider, snapshotService, configuration); - latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); - validatorTracker.init(tangle, latestMilestoneTracker, snapshotProvider, milestoneService, milestoneSolidifier, configuration); + nomineeTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); + latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, nomineeTracker, configuration); latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService, - latestMilestoneTracker, validatorTracker); + latestMilestoneTracker, nomineeTracker); seenMilestonesRetriever.init(tangle, snapshotProvider, transactionRequester); milestoneSolidifier.init(snapshotProvider, transactionValidator); ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, graph); diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index ce535921..9072711a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -14,7 +14,7 @@ */ public interface LatestMilestoneTracker { - void addMilestoneToLatestRound(Hash milestoneHash, int latestRoundIndex); + void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numberOfMilestones, int numberOfNominees); void clearLatestRoundHashes(); @@ -48,7 +48,7 @@ public interface LatestMilestoneTracker { */ void setCurrentRoundIndex(int latestRoundIndex); - void setLatestValidators(Set validatorAddresses); + void setCurrentNominees(Set nomineeAddresses); /** * Analyzes the given transaction to determine if it is a valid milestone.
diff --git a/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java b/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java index 7f5f0292..540de21b 100644 --- a/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java @@ -6,17 +6,19 @@ public interface NomineeTracker { - public Set getLatestValidators(); + Set getLatestNominees(); - Set getValidatorsOfRound(int roundIndex) throws Exception; + int getStartRound(); - boolean processTrusteeTransaction(Hash transactionHash) throws Exception; + Set getNomineesOfRound(int roundIndex) throws Exception; - Set getValidatorAddresses(Hash transaction) throws Exception; + boolean processNominees(Hash transactionHash) throws Exception; - void analyzeTrusteeTransactions() throws Exception; + Set getNomineeAddresses(Hash transaction) throws Exception; - void collectNewTrusteeTransactions() throws Exception; + void analyzeCuratorTransactions() throws Exception; + + void collectNewCuratorTransactions() throws Exception; void start(); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index da5a3a6e..b468c399 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -8,10 +8,7 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.persistables.Round; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; -import net.helix.hlx.service.milestone.MilestoneException; -import net.helix.hlx.service.milestone.MilestoneService; -import net.helix.hlx.service.milestone.MilestoneSolidifier; +import net.helix.hlx.service.milestone.*; import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; @@ -70,10 +67,17 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { */ private MilestoneSolidifier milestoneSolidifier; + /** + * Holds a reference to the manager that tracks nominees.
+ */ + private NomineeTracker nomineeTracker; + /** * Holds the addresses which are used to filter possible milestone candidates.
*/ - private Set validatorAddresses; + private Set currentNominees; + + private Set allNominees; /** * Holds a reference to the manager of the background worker.
@@ -137,19 +141,20 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { * @return the initialized instance itself to allow chaining */ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, HelixConfig config) { + MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, NomineeTracker nomineeTracker, HelixConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; + this.nomineeTracker = nomineeTracker; - validatorAddresses = new HashSet<>(); - validatorAddresses.add(HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db")); - validatorAddresses.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); + setCurrentNominees(nomineeTracker.getLatestNominees()); + allNominees = new HashSet<>(); + allNominees.addAll(nomineeTracker.getLatestNominees()); - genesisTime = 1561979153933L; + genesisTime = 1562320680150L; //currentRoundIndex = getRound(System.currentTimeMillis()); System.out.println("current time: " + System.currentTimeMillis()); //System.out.println("current round: " + getRound(System.currentTimeMillis())); @@ -166,11 +171,10 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP * message processor so external receivers get informed about this change.
*/ @Override - public void addMilestoneToLatestRound(Hash milestoneHash, int latestRoundIndex) { - tangle.publish("lmi %d %d", milestoneHash.hexString(), latestRoundIndex); - log.delegate().info("New milestone ({}/{}) added to round #{}", latestRoundHashes.size() + 1, validatorAddresses.size(), latestRoundIndex); + public void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numberOfMilestones, int numberOfNominees) { + tangle.publish("lmi %d %d", milestoneHash.hexString(), roundIndex); + log.delegate().info("New milestone {} ({}/{}) added to round #{}", milestoneHash.hexString(), numberOfMilestones, numberOfNominees, roundIndex); - this.latestRoundHashes.add(milestoneHash); } @Override public void setCurrentRoundIndex(int roundIndex) { @@ -180,10 +184,10 @@ public void setCurrentRoundIndex(int roundIndex) { } @Override - public void setLatestValidators(Set validatorAddresses) { - tangle.publish("lv %d %d", this.currentRoundIndex, validatorAddresses); - log.delegate().info("Validators for round #{}: {}", this.currentRoundIndex, validatorAddresses); - this.validatorAddresses = validatorAddresses; + public void setCurrentNominees(Set nominees) { + tangle.publish("lv %d %d", getCurrentRoundIndex(), nominees); + log.delegate().info("Validators of round #{}: {}", getCurrentRoundIndex(), nominees); + this.currentNominees = nominees; } @Override @@ -234,17 +238,31 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw //System.out.println("Process Milestone"); //System.out.println("Hash: " + transaction.getHash().hexString() + ", round: " + milestoneService.getRoundIndex(transaction)); //System.out.println("Current Round: " + getCurrentRoundIndex()); - if (validatorAddresses.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { - int roundIndex = milestoneService.getRoundIndex(transaction); + int roundIndex = milestoneService.getRoundIndex(transaction); + int currentRound = getCurrentRoundIndex(); + + // todo what happens if this method isn't called for that round and it passes the start round (do we need <= here?) + if (nomineeTracker.getStartRound() == currentRound) { + setCurrentNominees(nomineeTracker.getLatestNominees()); + allNominees.addAll(nomineeTracker.getLatestNominees()); + } + + Set nominees = currentNominees; + if (roundIndex != currentRound) { + nominees = nomineeTracker.getNomineesOfRound(roundIndex); + } + + if (nominees.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { // if the milestone is older than our ledger start point: we already processed it in the past if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { return true; } - switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validatorAddresses)) { + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, nominees)) { case VALID: + System.out.println("VALID"); // TODO: 1.review conditions, 2.is okay to store here? // before a milestone can be added to a round the following conditions have to be checked: // - index is bigger than initial snapshot (above) @@ -254,6 +272,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw // - index is bigger than snapshot index // - attachment timestamp is in correct time window for the index // - there doesn't already exist a milestone with the same address for that round + System.out.println("attachment timestamp round: " + getRound(transaction.getAttachmentTimestamp())); if (roundIndex == getRound(transaction.getAttachmentTimestamp())) { RoundViewModel currentRoundViewModel; @@ -264,9 +283,9 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw if (RoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { currentRoundViewModel.addMilestone(transaction.getHash()); currentRoundViewModel.update(tangle); - log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex + "(" + currentRoundViewModel.size() + ")"); + //log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex + " (" + currentRoundViewModel.size() + ")"); } else { - log.error("Round #" + roundIndex + " has already received a milestone from " + transaction.getAddressHash().hexString()); + //log.error("Round #" + roundIndex + " has already received a milestone from " + transaction.getAddressHash().hexString()); } } // this is the first milestone for that round, make new database entry @@ -275,8 +294,10 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw milestones.add(transaction.getHash()); currentRoundViewModel = new RoundViewModel(roundIndex, milestones); currentRoundViewModel.store(tangle); - log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex); + //log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex); } + + addMilestoneToRoundLog(transaction.getHash(), roundIndex, currentRoundViewModel.size(), nominees.size()); } if (!transaction.isSolid()) { @@ -287,6 +308,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw break; case INCOMPLETE: + System.out.println("INCOMPLETE"); milestoneSolidifier.add(transaction.getHash(), roundIndex); transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); @@ -347,9 +369,6 @@ public void bootstrapCurrentRoundIndex() { RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getCurrentRoundIndex()) { setCurrentRoundIndex(lastMilestoneInDatabase.index()); - for (Hash milestoneHash : lastMilestoneInDatabase.getHashes()) { - addMilestoneToLatestRound(milestoneHash, lastMilestoneInDatabase.index()); - } } } catch (Exception e) { log.error("unexpectedly failed to retrieve the latest milestone from the database", e); @@ -409,7 +428,7 @@ private void logProgress() { */ private void collectNewMilestoneCandidates() throws MilestoneException { try { - for (Hash address : validatorAddresses) { + for (Hash address : allNominees) { for (Hash hash : AddressViewModel.load(tangle, address).getHashes()) { if (Thread.currentThread().isInterrupted()) { return; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index c8dcddb8..f8b298cf 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -278,8 +278,8 @@ private void syncLatestMilestoneTracker(int roundIndex) { } } - private void syncValidatorTracker() { - latestMilestoneTracker.setLatestValidators(validatorTracker.getLatestValidators()); + private void syncNomineeTracker() { + latestMilestoneTracker.setCurrentNominees(validatorTracker.getLatestNominees()); } /** diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java index 7af8f28b..cda3244a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java @@ -1,22 +1,26 @@ package net.helix.hlx.service.milestone.impl; import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.BundleValidator; import net.helix.hlx.controllers.AddressViewModel; import net.helix.hlx.controllers.BundleViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; +import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.HashFactory; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; -import net.helix.hlx.service.milestone.MilestoneService; -import net.helix.hlx.service.milestone.MilestoneSolidifier; -import net.helix.hlx.service.milestone.NomineeTracker; +import net.helix.hlx.service.milestone.*; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; import net.helix.hlx.utils.thread.SilentScheduledExecutorService; +import static net.helix.hlx.service.milestone.MilestoneValidity.INCOMPLETE; +import static net.helix.hlx.service.milestone.MilestoneValidity.INVALID; +import static net.helix.hlx.service.milestone.MilestoneValidity.VALID; + + import java.util.*; import java.util.concurrent.TimeUnit; @@ -24,48 +28,58 @@ public class NomineeTrackerImpl implements NomineeTracker { private static final int MAX_CANDIDATES_TO_ANALYZE = 5000; private static final int RESCAN_INTERVAL = 1000; - private Hash Trustee_Address; + private Hash Curator_Address; - private static final IntervalLogger log = new IntervalLogger(LatestMilestoneTrackerImpl.class); + private static final IntervalLogger log = new IntervalLogger(NomineeTrackerImpl.class); private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Validator Tracker", log.delegate()); + "Nominee Tracker", log.delegate()); private Tangle tangle; - private LatestMilestoneTracker latestMilestoneTracker; + private HelixConfig config; private SnapshotProvider snapshotProvider; private MilestoneService milestoneService; private MilestoneSolidifier milestoneSolidifier; - private Set latestValidators = new HashSet<>(); - private final Set seenTrusteeTransactions = new HashSet<>(); - private final Deque trusteeTransactionsToAnalyze = new ArrayDeque<>(); + private Set latestNominees; + private int startRound; + private final Set seenCuratorTransactions = new HashSet<>(); + private final Deque curatorTransactionsToAnalyze = new ArrayDeque<>(); - public NomineeTrackerImpl init(Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, HelixConfig config) { + public NomineeTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, HelixConfig config) { this.tangle = tangle; - this.latestMilestoneTracker = latestMilestoneTracker; - this.Trustee_Address = config.getTrusteeAddress(); + this.config = config; + this.Curator_Address = config.getTrusteeAddress(); this.snapshotProvider = snapshotProvider; this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; - //bootstrapLatestValidators(); + + latestNominees = new HashSet<>(); + latestNominees.add(HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db")); + latestNominees.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); + + startRound = 1; + //bootstrapLatestNominees(); return this; } @Override - public Set getLatestValidators() { - return latestValidators; + public Set getLatestNominees() { + return latestNominees; } + @Override + public int getStartRound() {return startRound; } + @Override - public Set getValidatorsOfRound(int roundIndex) throws Exception{ + public Set getNomineesOfRound(int roundIndex) throws Exception{ try { Set validators = new HashSet<>(); - for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { + for (Hash hash : AddressViewModel.load(tangle, Curator_Address).getHashes()) { TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, hash); if (milestoneService.getRoundIndex(transaction) == roundIndex) { - validators = getValidatorAddresses(hash); + validators = getNomineeAddresses(hash); } } return validators; @@ -74,11 +88,50 @@ public Set getValidatorsOfRound(int roundIndex) throws Exception{ } } + + public MilestoneValidity validateNominees(TransactionViewModel transactionViewModel, int roundIndex, + SpongeFactory.Mode mode, int securityLevel) throws Exception { + + if (roundIndex < 0 || roundIndex >= 0x200000) { + return INVALID; + } + + try { + + final List> bundleTransactions = BundleValidator.validate(tangle, + snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); + + if (bundleTransactions.isEmpty()) { + return INCOMPLETE; + } else { + for (final List bundleTransactionViewModels : bundleTransactions) { + final TransactionViewModel tail = bundleTransactionViewModels.get(0); // transaction with signature + if (tail.getHash().equals(transactionViewModel.getHash())) { + + // validate signature + Hash senderAddress = tail.getAddressHash(); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); + + if (Curator_Address == senderAddress && validSignature) { + return VALID; + } else { + return INVALID; + } + } + } + } + } catch (Exception e) { + throw new MilestoneException("error while validating nominees for round #" + roundIndex, e); + } + + return INVALID; + } + @Override - public boolean processTrusteeTransaction(Hash transactionHash) throws Exception { + public boolean processNominees(Hash transactionHash) throws Exception { TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); try { - if (Trustee_Address.equals(transaction.getAddressHash())) { + if (Curator_Address.equals(transaction.getAddressHash())) { int roundIndex = milestoneService.getRoundIndex(transaction); @@ -87,13 +140,12 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception return true; } - // we use milestone validation because its the same process (TODO: Rename function and implement more general) - Set validation = new HashSet<>(); - validation.add(Trustee_Address); - switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, validation)) { + // validate + switch (validateNominees(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { case VALID: - if (roundIndex > latestMilestoneTracker.getCurrentRoundIndex()) { - latestValidators = getValidatorAddresses(transaction.getHash()); + if (roundIndex > startRound) { + latestNominees = getNomineeAddresses(transaction.getHash()); + startRound = roundIndex; } if (!transaction.isSolid()) { @@ -119,7 +171,7 @@ public boolean processTrusteeTransaction(Hash transactionHash) throws Exception } @Override - public Set getValidatorAddresses(Hash transaction) throws Exception{ + public Set getNomineeAddresses(Hash transaction) throws Exception{ TransactionViewModel tail = TransactionViewModel.fromHash(tangle, transaction); BundleViewModel bundle = BundleViewModel.load(tangle, tail.getBundleHash()); int security = 1; @@ -128,14 +180,14 @@ public Set getValidatorAddresses(Hash transaction) throws Exception{ for (Hash txHash : bundle.getHashes()) { TransactionViewModel tx = TransactionViewModel.fromHash(tangle, txHash); // get transactions with validator addresses in signatureMessageFragment - // -> we have to count from last index - // n-security-1 to n: tx with signature - // n-security-1: tx with merkle path - // 0 to n-security: tx with validator addresses - if ((tx.getCurrentIndex() <= 0) && (tx.getCurrentIndex() < bundle.size() - security - 1)) { + // 0 - security: tx with signature + // security: tx with merkle path + // security+1 - n: tx with validator addresses + if ((tx.getCurrentIndex() > security)) { + for (int i = 0; i < TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE / Hash.SIZE_IN_BYTES; i++) { - Hash address = HashFactory.ADDRESS.create(Arrays.copyOfRange(tx.getSignature(), i * Hash.SIZE_IN_BYTES, (i+1) * Hash.SIZE_IN_BYTES)); - // TODO: Do we send all validators or only adding and removing ones + Hash address = HashFactory.ADDRESS.create(tx.getSignature(), i * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); + // TODO: Do we send all validators or only adding and removing ones ? if (address.equals(Hash.NULL_HASH)){ break; } @@ -147,29 +199,29 @@ public Set getValidatorAddresses(Hash transaction) throws Exception{ } @Override - public void analyzeTrusteeTransactions() throws Exception { - int transactionsToAnalyze = Math.min(trusteeTransactionsToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); + public void analyzeCuratorTransactions() throws Exception { + int transactionsToAnalyze = Math.min(curatorTransactionsToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); for (int i = 0; i < transactionsToAnalyze; i++) { if (Thread.currentThread().isInterrupted()) { return; } - Hash trusteeTransactionHash = trusteeTransactionsToAnalyze.pollFirst(); - if(!processTrusteeTransaction(trusteeTransactionHash)) { - seenTrusteeTransactions.remove(trusteeTransactionHash); + Hash trusteeTransactionHash = curatorTransactionsToAnalyze.pollFirst(); + if(!processNominees(trusteeTransactionHash)) { + seenCuratorTransactions.remove(trusteeTransactionHash); } } } @Override - public void collectNewTrusteeTransactions() throws Exception { + public void collectNewCuratorTransactions() throws Exception { try { - for (Hash hash : AddressViewModel.load(tangle, Trustee_Address).getHashes()) { + for (Hash hash : AddressViewModel.load(tangle, Curator_Address).getHashes()) { if (Thread.currentThread().isInterrupted()) { return; } - if (seenTrusteeTransactions.add(hash)) { - trusteeTransactionsToAnalyze.addFirst(hash); + if (seenCuratorTransactions.add(hash)) { + curatorTransactionsToAnalyze.addFirst(hash); } } } catch (Exception e) { @@ -179,8 +231,8 @@ public void collectNewTrusteeTransactions() throws Exception { private void validatorTrackerThread() { try { - collectNewTrusteeTransactions(); - analyzeTrusteeTransactions(); + collectNewCuratorTransactions(); + analyzeCuratorTransactions(); } catch (Exception e) { log.error("error while scaning for trustee transactions", e); } From 961d70bba750b1747680bf8dceb9e9a9aa81d469 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 5 Jul 2019 14:21:55 +0200 Subject: [PATCH 076/223] merkle tree representation as List --- .../java/net/helix/hlx/crypto/Merkle.java | 54 +++++++++++-------- src/main/java/net/helix/hlx/service/API.java | 26 +++++---- .../impl/LatestMilestoneTrackerImpl.java | 6 +-- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index 19e3b67c..8af8148b 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -13,7 +13,11 @@ import java.io.File; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; public class Merkle { @@ -48,37 +52,43 @@ public static byte[] getMerklePath(byte[][][] merkleTree, int keyIndex){ return merklePath; } - public static byte[][][] buildMerkleTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount){ - byte[][] keys = new byte[1 << pubkeyDepth][32]; + public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount){ + List keys = new ArrayList<>(1 << pubkeyDepth); for (int i = 0; i < pubkeyCount; i++) { int idx = firstIndex + i; - keys[idx] = Winternitz.generateAddress(Hex.decode(seed), idx, 1); + keys.set(idx, HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, 1))); } + return buildMerkleTree(keys); + } + + + public static List> buildMerkleTree(List leaves){ byte[] buffer; Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); - byte[][][] merkleTree = new byte[pubkeyDepth+1][][]; - merkleTree[0] = keys; + int depth = (int) Math.ceil(Math.sqrt(leaves.size())); + List> merkleTree = new ArrayList<>(depth); + merkleTree.set(0, leaves); int row = 1; // hash two following keys together until only one is left -> merkle tree - while (keys.length > 1) { + while (leaves.size() > 1) { // Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them - byte[][] nextKeys = new byte[keys.length / 2][32]; - for (int i = 0; i < nextKeys.length; i++) { - if (keys[i * 2] == null && keys[i * 2 + 1] == null) { + List nextKeys = new ArrayList<>(leaves.size() / 2); + for (int i = 0; i < nextKeys.size(); i++) { + if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { // leave the combined key null as well continue; } sha3.reset(); - byte[] k1 = keys[i * 2], k2 = keys[i * 2 + 1]; + byte[] k1 = leaves.get(i * 2).bytes(), k2 = leaves.get(i * 2 + 1).bytes(); buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1, 0, 32); sha3.absorb(buffer, 0, buffer.length); buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2, 0, 32); sha3.absorb(buffer, 0, buffer.length); sha3.squeeze(buffer, 0, buffer.length); - nextKeys[i] = buffer; + nextKeys.set(i, HashFactory.GENERIC.create(buffer)); } - keys = nextKeys; - merkleTree[row++] = keys; + leaves = nextKeys; + merkleTree.set(row++, leaves); } return merkleTree; } @@ -130,30 +140,30 @@ public static byte[][][] readKeyfile(File keyfile, StringBuilder seedBuilder) th } } - public static void createKeyfile(byte[][][] merkleTree, byte[] seed, int pubkeyDepth, String filename) throws IOException { + public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, String filename) throws IOException { // fill buffer try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { // write pubkey depth and seed into buffer bw.write(pubkeyDepth + " " + Hex.toHexString(seed)); bw.newLine(); - writeKeys(bw, merkleTree[0]); - for (int i = 1; i < merkleTree.length; i++) { - writeKeys(bw, merkleTree[i]); + writeKeys(bw, merkleTree.get(0)); + for (int i = 1; i < merkleTree.size(); i++) { + writeKeys(bw, merkleTree.get(0)); } } } - private static void writeKeys(BufferedWriter bw, byte[][] keys) throws IOException { + private static void writeKeys(BufferedWriter bw, List keys) throws IOException { int leadingNulls = 0; - while (keys[leadingNulls] == null) { + while (keys.get(leadingNulls) == null) { leadingNulls++; } bw.write(leadingNulls + " "); - for (int i = leadingNulls; i < keys.length; i++) { - if (keys[i] == null) { + for (int i = leadingNulls; i < keys.size(); i++) { + if (keys.get(i) == null) { break; } - bw.write(Hex.toHexString(keys[i])); + bw.write(keys.get(i).hexString()); } bw.newLine(); } diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index a9655921..0114e5b2 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1490,17 +1490,9 @@ private void attachStoreAndBroadcast(final String address, final String message, public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int incrementIndex) throws Exception { - // get tips + // get round int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); long nextIndex = currentRoundIndex + incrementIndex; - List txToApprove = new ArrayList<>(); - System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); - if(RoundViewModel.latest(tangle) == null) { - txToApprove.add(Hash.NULL_HASH); - txToApprove.add(Hash.NULL_HASH); - } else { - txToApprove = getTransactionToApproveTips(3, Optional.empty()); - } // list of confirming tips byte[] txTips = new byte[TransactionViewModel.SIZE]; @@ -1581,12 +1573,26 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri snapshotProvider.getLatestSnapshot().unlockRead(); } - // write confirming tips into signature message fragment of txTips for (int i=0; i txToApprove = new ArrayList<>(); + System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); + if(RoundViewModel.latest(tangle) == null) { + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); + } else { + txToApprove = getTransactionToApproveTips(3, Optional.empty()); + /* + List> merkleTreeTips = Merkle.buildMerkleTree(confirmingTips); + Hash branch = merkleTreeTips.get(merkleTreeTips.size() - 1).get(0); // get root + List> merkleTreeLatestRound = Merkle.buildMerkleTree(confirmingTips); + */ + } + // attach, broadcast and store List transactions = new ArrayList<>(); transactions.add(Hex.toHexString(txTips)); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index b468c399..44a7bbd2 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -154,7 +154,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP allNominees = new HashSet<>(); allNominees.addAll(nomineeTracker.getLatestNominees()); - genesisTime = 1562320680150L; + genesisTime = 1562328163995L; //currentRoundIndex = getRound(System.currentTimeMillis()); System.out.println("current time: " + System.currentTimeMillis()); //System.out.println("current round: " + getRound(System.currentTimeMillis())); @@ -172,7 +172,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP */ @Override public void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numberOfMilestones, int numberOfNominees) { - tangle.publish("lmi %d %d", milestoneHash.hexString(), roundIndex); + tangle.publish("lmi %s %d", milestoneHash.hexString(), roundIndex); log.delegate().info("New milestone {} ({}/{}) added to round #{}", milestoneHash.hexString(), numberOfMilestones, numberOfNominees, roundIndex); } @@ -185,7 +185,7 @@ public void setCurrentRoundIndex(int roundIndex) { @Override public void setCurrentNominees(Set nominees) { - tangle.publish("lv %d %d", getCurrentRoundIndex(), nominees); + //tangle.publish("lv %d %d", getCurrentRoundIndex(), nominees); log.delegate().info("Validators of round #{}: {}", getCurrentRoundIndex(), nominees); this.currentNominees = nominees; } From e14d9cce9a6f77037c058bf3af117e6a3078d5bf Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 5 Jul 2019 16:21:36 +0200 Subject: [PATCH 077/223] getBalances fix --- src/main/java/net/helix/hlx/service/API.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 0114e5b2..4b51cbd5 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1075,7 +1075,8 @@ private AbstractResponse getBalancesStatement(List addresses, final int index = snapshotProvider.getLatestSnapshot().getIndex(); if (tips == null || tips.size() == 0) { - hashes = Collections.singletonList(snapshotProvider.getLatestSnapshot().getHash()); + //todo merkle root of latest milestones + hashes = Collections.singletonList(RoundViewModel.latest(tangle).getRandomConfirmingMilestone(tangle)); } else { hashes = tips.stream() .map(tip -> (HashFactory.TRANSACTION.create(tip))) From c347ae49f3b05dda62e24bc3364fc0203be08a59 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 7 Jul 2019 13:50:34 +0200 Subject: [PATCH 078/223] correct merkle tree size, fixed size nextKeys --- src/main/java/net/helix/hlx/crypto/Merkle.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index 8af8148b..e9d06ea8 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -66,29 +66,29 @@ public static List> buildMerkleTree(List leaves){ byte[] buffer; Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); int depth = (int) Math.ceil(Math.sqrt(leaves.size())); - List> merkleTree = new ArrayList<>(depth); - merkleTree.set(0, leaves); + List> merkleTree = new ArrayList<>(depth + 1); + merkleTree.add(0, leaves); int row = 1; // hash two following keys together until only one is left -> merkle tree while (leaves.size() > 1) { // Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them - List nextKeys = new ArrayList<>(leaves.size() / 2); + List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); for (int i = 0; i < nextKeys.size(); i++) { if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { // leave the combined key null as well continue; } sha3.reset(); - byte[] k1 = leaves.get(i * 2).bytes(), k2 = leaves.get(i * 2 + 1).bytes(); - buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1, 0, 32); + Hash k1 = leaves.get(i * 2), k2 = leaves.get(i * 2 + 1); + buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32); sha3.absorb(buffer, 0, buffer.length); - buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2, 0, 32); + buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32); sha3.absorb(buffer, 0, buffer.length); sha3.squeeze(buffer, 0, buffer.length); - nextKeys.set(i, HashFactory.GENERIC.create(buffer)); + nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); } leaves = nextKeys; - merkleTree.set(row++, leaves); + merkleTree.add(row++, leaves); } return merkleTree; } From be03ff5cabeece5a5f2154f7a5184b30c012b7eb Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 7 Jul 2019 13:53:29 +0200 Subject: [PATCH 079/223] get merkle root of milestones to store as snapshot hash --- .../snapshot/impl/SnapshotServiceImpl.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 39a674e3..9e49ef28 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -5,6 +5,7 @@ import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.StateDiffViewModel; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.Hash; import net.helix.hlx.service.milestone.LatestMilestoneTracker; import net.helix.hlx.service.snapshot.*; @@ -123,7 +124,7 @@ public SnapshotServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws SnapshotException { Map balanceChanges = new HashMap<>(); Set skippedMilestones = new HashSet<>(); - RoundViewModel lastAppliedMilestone = null; + RoundViewModel lastAppliedRound = null; try { for (int currentRoundIndex = snapshot.getIndex() + 1; currentRoundIndex <= targetRoundIndex; @@ -138,24 +139,32 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna }); } - lastAppliedMilestone = currentRound; + lastAppliedRound = currentRound; } else { skippedMilestones.add(currentRoundIndex); } } - if (lastAppliedMilestone != null) { + if (lastAppliedRound != null) { try { snapshot.lockWrite(); snapshot.applyStateDiff(new SnapshotStateDiffImpl(balanceChanges)); - snapshot.setIndex(lastAppliedMilestone.index()); + snapshot.setIndex(lastAppliedRound.index()); - //TODO: Store all milestone hashes in snapshot or merkle root? - //snapshot.setHash(lastAppliedMilestone.getHash()); + //store merkle root + if (!lastAppliedRound.getHashes().isEmpty()) { + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(lastAppliedRound.getHashes())); + snapshot.setHash(merkleTree.get(merkleTree.size() - 1).get(0)); + System.out.println("Hashes:"); + lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); + } else { + snapshot.setHash(Hash.NULL_HASH); + } + System.out.println("snapshot hash: " + snapshot.getHash().hexString()); - //TODO: From where we should get the timestamp of the round/snapshot? + //TODO: get start time of round (need genesisTime) /*TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, lastAppliedMilestone.getHash()); if(milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT) { From 04586a8e5c148f5099aa184c0183b6de20831a76 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 7 Jul 2019 13:55:18 +0200 Subject: [PATCH 080/223] reference merkle root of confirmed tips and snapshot hash as branch and trunk --- src/main/java/net/helix/hlx/service/API.java | 23 ++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 4b51cbd5..c9430d0f 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1552,7 +1552,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri } // get confirming tips - List confirmingTips = new LinkedList<>(); + List confirmedTips = new LinkedList<>(); System.out.println("Tips (" + tipsViewModel.getTips().size() + ")"); snapshotProvider.getLatestSnapshot().lockRead(); @@ -1566,7 +1566,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { if (walkValidator.isValid(transaction)) { System.out.println(transaction.hexString()); - confirmingTips.add(transaction); + confirmedTips.add(transaction); } } } @@ -1574,9 +1574,9 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri snapshotProvider.getLatestSnapshot().unlockRead(); } - // write confirming tips into signature message fragment of txTips - for (int i=0; i> merkleTreeTips = Merkle.buildMerkleTree(confirmingTips); - Hash branch = merkleTreeTips.get(merkleTreeTips.size() - 1).get(0); // get root - List> merkleTreeLatestRound = Merkle.buildMerkleTree(confirmingTips); - */ + if (!confirmedTips.isEmpty()) { + List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); + txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips + } else { + txToApprove.add(Hash.NULL_HASH); + } + txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); // merkle root of latest milestones } // attach, broadcast and store From 4eea5fc74c38431b2e6c843ffd2d7dca7b4fad46 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 7 Jul 2019 13:56:43 +0200 Subject: [PATCH 081/223] getTipSet() -> static --- src/main/java/net/helix/hlx/controllers/RoundViewModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 679d2420..abab2d47 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -225,7 +225,7 @@ public static RoundViewModel findClosestNextRound(Tangle tangle, int index, int return nextRoundViewModel; } - public Set getTipSet(Tangle tangle, Hash milestone) throws Exception { + public static Set getTipSet(Tangle tangle, Hash milestone) throws Exception { int security = 1; TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestone); From 4789a9153e44eaa008b59685a514ee73da612a13 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 7 Jul 2019 13:57:05 +0200 Subject: [PATCH 082/223] print snapshot --- .../net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index 4de8ecd0..eafaa41c 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -113,6 +113,8 @@ public boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerExc if(generateStateDiff(milestone)) { try { snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.index()); + //System.out.println("Snapshot"); + //snapshotProvider.getLatestSnapshot().getBalances().forEach((address, balance) -> System.out.println("Address: " + address.hexString() + ", " + balance)); } catch (SnapshotException e) { throw new LedgerException("failed to apply the balance changes to the ledger state", e); } From 268d5f1da9e30e77cab51d0eed6792d7be71c501 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 9 Jul 2019 17:57:43 +0200 Subject: [PATCH 083/223] add method to check if transaction is part of milestone bundle --- .../helix/hlx/controllers/TransactionViewModel.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java index 244a6ff6..88793237 100644 --- a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java @@ -662,6 +662,16 @@ public boolean isMilestone() { return transaction.milestone; } + public TransactionViewModel isMilestoneBundle(Tangle tangle) throws Exception{ + for (Hash bundleTx : BundleViewModel.load(tangle, getBundleHash()).getHashes()){ + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, bundleTx); + if (tx.getCurrentIndex() == 0 && tx.isMilestone()){ + return tx; + } + } + return null; + } + /** @return The current {@link Transaction#height} */ public long getHeight() { return transaction.height; From 8690ead9ae9cb79e2ad0da04d674042e1ddbc4a4 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 9 Jul 2019 18:08:30 +0200 Subject: [PATCH 084/223] if leaves size=1 return leave --- src/main/java/net/helix/hlx/crypto/Merkle.java | 3 +++ src/main/java/net/helix/hlx/service/API.java | 10 ++++------ .../service/snapshot/impl/SnapshotServiceImpl.java | 12 ++++-------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index e9d06ea8..f22c2955 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -63,6 +63,9 @@ public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, public static List> buildMerkleTree(List leaves){ + if (leaves.isEmpty()) { + leaves.add(Hash.NULL_HASH); + } byte[] buffer; Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); int depth = (int) Math.ceil(Math.sqrt(leaves.size())); diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index c9430d0f..6b442085 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1586,13 +1586,11 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); } else { - if (!confirmedTips.isEmpty()) { - List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); - txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips - } else { - txToApprove.add(Hash.NULL_HASH); - } + // trunk txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); // merkle root of latest milestones + //branch + List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); + txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips } // attach, broadcast and store diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 9e49ef28..efd08b87 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -154,14 +154,10 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna snapshot.setIndex(lastAppliedRound.index()); //store merkle root - if (!lastAppliedRound.getHashes().isEmpty()) { - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(lastAppliedRound.getHashes())); - snapshot.setHash(merkleTree.get(merkleTree.size() - 1).get(0)); - System.out.println("Hashes:"); - lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); - } else { - snapshot.setHash(Hash.NULL_HASH); - } + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(lastAppliedRound.getHashes())); + snapshot.setHash(merkleTree.get(merkleTree.size() - 1).get(0)); + System.out.println("Hashes:"); + lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); System.out.println("snapshot hash: " + snapshot.getHash().hexString()); //TODO: get start time of round (need genesisTime) From 2ce932f8d71b4284adc9b66446ae8fada1602892 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 9 Jul 2019 18:16:19 +0200 Subject: [PATCH 085/223] move getRoundIndex (from tag) to RoundViewModel for bette access --- .../java/net/helix/hlx/controllers/RoundViewModel.java | 8 ++++++++ .../helix/hlx/service/milestone/MilestoneService.java | 10 ---------- .../service/milestone/impl/MilestoneServiceImpl.java | 7 ------- .../hlx/service/milestone/impl/NomineeTrackerImpl.java | 5 +++-- 4 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index abab2d47..19210e7b 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -1,5 +1,6 @@ package net.helix.hlx.controllers; +import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.IntegerIndex; @@ -8,6 +9,7 @@ import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Pair; +import net.helix.hlx.utils.Serializer; import org.bouncycastle.util.encoders.Hex; import java.util.*; @@ -225,6 +227,12 @@ public static RoundViewModel findClosestNextRound(Tangle tangle, int index, int return nextRoundViewModel; } + public static int getRoundIndex(TransactionViewModel milestoneTransaction) { + return (int) Serializer.getLong(milestoneTransaction.getBytes(), TransactionViewModel.TAG_OFFSET); + //TODO: why is index stored in bundle nonce (obsolete tag) in new version? + //return (int) Serializer.getLong(milestoneTransaction.getBytes(), BUNDLE_NONCE_OFFSET); + } + public static Set getTipSet(Tangle tangle, Hash milestone) throws Exception { int security = 1; diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java index 83066310..6d11e15e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneService.java @@ -112,14 +112,4 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i Set getConfirmedTips(int roundNumber) throws Exception; - /** - * Retrieves the milestone index of the given transaction by decoding the {@code OBSOLETE_TAG}.
- *
- * The returned result will of cause only have a reasonable value if we hand in a transaction that represents a real - * milestone.
- * - * @param milestoneTransaction the transaction that shall have its milestone index retrieved - * @return the milestone index of the transaction - */ - int getRoundIndex(TransactionViewModel milestoneTransaction); } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 225d5330..07555811 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -230,13 +230,6 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM return INVALID; } - @Override - public int getRoundIndex(TransactionViewModel milestoneTransaction) { - return (int) Serializer.getLong(milestoneTransaction.getBytes(), TransactionViewModel.TAG_OFFSET); - //TODO: why is index stored in bundle nonce (obsolete tag) in new version? - //return (int) Serializer.getLong(milestoneTransaction.getBytes(), BUNDLE_NONCE_OFFSET); - } - @Override public Set getConfirmedTips(int roundNumber) throws Exception { diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java index cda3244a..8e9e9b5e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java @@ -5,6 +5,7 @@ import net.helix.hlx.controllers.AddressViewModel; import net.helix.hlx.controllers.BundleViewModel; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.crypto.Merkle; @@ -78,7 +79,7 @@ public Set getNomineesOfRound(int roundIndex) throws Exception{ Set validators = new HashSet<>(); for (Hash hash : AddressViewModel.load(tangle, Curator_Address).getHashes()) { TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, hash); - if (milestoneService.getRoundIndex(transaction) == roundIndex) { + if (RoundViewModel.getRoundIndex(transaction) == roundIndex) { validators = getNomineeAddresses(hash); } } @@ -133,7 +134,7 @@ public boolean processNominees(Hash transactionHash) throws Exception { try { if (Curator_Address.equals(transaction.getAddressHash())) { - int roundIndex = milestoneService.getRoundIndex(transaction); + int roundIndex = RoundViewModel.getRoundIndex(transaction); // if the trustee transaction is older than our ledger start point: we already processed it in the past if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { From d24de21f329af777b1279ef7d66c50531f12fc73 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 9 Jul 2019 18:18:25 +0200 Subject: [PATCH 086/223] implement getMilestoneTrunk and getMilestoneBranch --- .../helix/hlx/controllers/RoundViewModel.java | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 19210e7b..7e770bb3 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -232,7 +232,88 @@ public static int getRoundIndex(TransactionViewModel milestoneTransaction) { //TODO: why is index stored in bundle nonce (obsolete tag) in new version? //return (int) Serializer.getLong(milestoneTransaction.getBytes(), BUNDLE_NONCE_OFFSET); } - + + // todo this may be very inefficient + public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ + System.out.println("Get Milestone Trunk"); + Set trunk = new HashSet<>(); + int round = RoundViewModel.getRoundIndex(milestoneTx); + System.out.println("round: " + round); + System.out.println("index: " + transaction.getCurrentIndex()); + // idx = n: milestone merkle root in trunk and tips merkle root in branch + if (transaction.getCurrentIndex() == transaction.lastIndex()) { + // add previous milestones to non analyzed transactions + Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); + System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + System.out.println("merkleRoot: " + transaction.getTrunkTransactionHash()); + System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0)); + if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { + System.out.println("trunk: "); + if (prevMilestones.isEmpty()){ + trunk.add(Hash.NULL_HASH); + System.out.println(Hash.NULL_HASH.hexString()); + } else { + trunk.addAll(prevMilestones); + prevMilestones.forEach(c -> System.out.println(c.hexString())); + } + } + } + else { + // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash + trunk.add(transaction.getTrunkTransactionHash()); + System.out.println("trunk: " + transaction.getTrunkTransactionHash().hexString()); + + } + return trunk; + } + + public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ + System.out.println("Get Milestone Branch"); + Set branch = new HashSet<>(); + int round = RoundViewModel.getRoundIndex(milestoneTx); + System.out.println("round: " + round); + System.out.println("index: " + transaction.getCurrentIndex()); + // idx = n: milestone merkle root in trunk and tips merkle root in branch + if (transaction.getCurrentIndex() == transaction.lastIndex()) { + // tips merkle root + Set confirmedTips = getTipSet(tangle, milestoneTx.getHash()); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); + System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); + System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); + if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { + System.out.println("branch: "); + if (confirmedTips.isEmpty()){ + branch.add(Hash.NULL_HASH); + System.out.println(Hash.NULL_HASH.hexString()); + } else { + branch.addAll(confirmedTips); + confirmedTips.forEach(c -> System.out.println(c.hexString())); + } + } + } + else { + // add previous milestones to non analyzed transactions + Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); + System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + System.out.println("merkleRoot: " + transaction.getBranchTransactionHash()); + System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0)); + if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { + System.out.println("branch: "); + if (prevMilestones.isEmpty()){ + branch.add(Hash.NULL_HASH); + System.out.println(Hash.NULL_HASH.hexString()); + } else { + branch.addAll(prevMilestones); + prevMilestones.forEach(c -> System.out.println(c.hexString())); + } + } + } + + return branch; + } + public static Set getTipSet(Tangle tangle, Hash milestone) throws Exception { int security = 1; From f701fcddb9fa974293444f708e31e6b26907b5ef Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 9 Jul 2019 18:21:24 +0200 Subject: [PATCH 087/223] use specific getMilestonTrunk/Branch for transaction of milestone bundle in the solidification process --- .../net/helix/hlx/TransactionValidator.java | 58 ++++++++++++++++--- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/helix/hlx/TransactionValidator.java b/src/main/java/net/helix/hlx/TransactionValidator.java index ccf74bd3..89ab8b91 100644 --- a/src/main/java/net/helix/hlx/TransactionValidator.java +++ b/src/main/java/net/helix/hlx/TransactionValidator.java @@ -1,13 +1,16 @@ package net.helix.hlx; import net.helix.hlx.conf.APIConfig; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TipsViewModel; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; import net.helix.hlx.crypto.Sponge; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.model.TransactionHash; import net.helix.hlx.network.TransactionRequester; +import net.helix.hlx.service.milestone.MilestoneService; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import org.slf4j.Logger; @@ -243,6 +246,7 @@ public boolean checkSolidity(Hash hash, boolean milestone) throws Exception { * @throws Exception if anything goes wrong while trying to solidify the transaction */ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTransactions) throws Exception { + System.out.println("Check Solidity"); if(fromHash(tangle, hash).isSolid()) { return true; } @@ -260,6 +264,7 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans } final TransactionViewModel transaction = fromHash(tangle, hashPointer); + System.out.println(hashPointer.hexString() + ", solid: " + transaction.isSolid() + ", has solid entry point: " + snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); if(!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { if (transaction.getType() == PREFILLED_SLOT) { solid = false; @@ -269,8 +274,20 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans break; } } else { - nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); + // transaction of milestone bundle + TransactionViewModel milestoneTx; + if ((milestoneTx = transaction.isMilestoneBundle(tangle)) != null){ + Set parents = RoundViewModel.getMilestoneTrunk(tangle, transaction, milestoneTx); + parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transaction, milestoneTx)); + for (Hash parent : parents){ + nonAnalyzedTransactions.offer(parent); + } + } + // normal transaction + else { + nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); + nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); + } } } } @@ -379,9 +396,19 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); if(transactionViewModel.getApprovers(tangle).size() == 0) { tipsViewModel.addTipHash(transactionViewModel.getHash()); + } else { + TransactionViewModel milestoneTx; + if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ + Set parents = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); + parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx)); + for (Hash parent : parents){ + tipsViewModel.removeTipHash(parent); + } + } else { + tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); + tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); + } } - tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); - tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); if(quickSetSolid(transactionViewModel)) { transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); @@ -412,14 +439,27 @@ private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { * @throws Exception */ private boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { + System.out.println("Quick Set Solid"); if(!transactionViewModel.isSolid()) { boolean solid = true; - if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { - solid = false; - } - if (!checkApproovee(transactionViewModel.getBranchTransaction(tangle))) { - solid = false; + TransactionViewModel milestoneTx; + if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ + Set parents = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); + parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx)); + for (Hash parent : parents){ + if (!checkApproovee(TransactionViewModel.fromHash(tangle, parent))) { + solid = false; + } + } + } else { + if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { + solid = false; + } + if (!checkApproovee(transactionViewModel.getBranchTransaction(tangle))) { + solid = false; + } } + if(solid) { transactionViewModel.updateSolid(true); transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); From 51edd16532f504ed56b5882c6f0418048fb641a9 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 9 Jul 2019 18:24:43 +0200 Subject: [PATCH 088/223] set correct references for milestone bundle in graphstream --- src/main/java/net/helix/hlx/network/Node.java | 18 ++++++++++++++++-- src/main/java/net/helix/hlx/service/API.java | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/network/Node.java b/src/main/java/net/helix/hlx/network/Node.java index 539a33e4..c5113ca2 100644 --- a/src/main/java/net/helix/hlx/network/Node.java +++ b/src/main/java/net/helix/hlx/network/Node.java @@ -463,8 +463,22 @@ public void processReceivedData(TransactionViewModel receivedTransactionViewMode //store new transaction try { stored = receivedTransactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot()); - if (this.graph != null) { - this.graph.addNode(receivedTransactionViewModel.getHash().hexString(), receivedTransactionViewModel.getTrunkTransactionHash().hexString(), receivedTransactionViewModel.getBranchTransactionHash().hexString()); + if (graph != null) { + TransactionViewModel milestoneTx; + if ((milestoneTx = receivedTransactionViewModel.isMilestoneBundle(tangle)) != null){ + Set trunk = RoundViewModel.getMilestoneTrunk(tangle, receivedTransactionViewModel, milestoneTx); + Set branch =RoundViewModel.getMilestoneBranch(tangle, receivedTransactionViewModel, milestoneTx); + for (Hash t : trunk){ + this.graph.graph.addEdge(receivedTransactionViewModel.getHash().hexString()+t.hexString(), receivedTransactionViewModel.getHash().hexString(), t.hexString()); // h -> t + } + for (Hash b : branch){ + this.graph.graph.addEdge(receivedTransactionViewModel.getHash().hexString()+b.hexString(), receivedTransactionViewModel.getHash().hexString(), b.hexString()); // h -> t + } + org.graphstream.graph.Node graphNode = graph.graph.getNode(receivedTransactionViewModel.getHash().hexString()); + graphNode.addAttribute("ui.label", receivedTransactionViewModel.getHash().hexString().substring(0,10)); + graphNode.addAttribute("ui.style", "fill-color: rgb(255,165,0); stroke-color: rgb(30,144,255); stroke-width: 2px;"); + } + graph.addNode(receivedTransactionViewModel.getHash().hexString(), receivedTransactionViewModel.getTrunkTransactionHash().hexString(), receivedTransactionViewModel.getBranchTransactionHash().hexString()); } } catch (Exception e) { log.error("Error accessing persistence store.", e); diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 6b442085..f044d3b3 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -626,6 +626,20 @@ public void storeTransactionsStatement(final List txHex) throws Exceptio } if (graph != null) { + TransactionViewModel milestoneTx; + if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ + Set trunk = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); + Set branch =RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx); + for (Hash t : trunk){ + this.graph.graph.addEdge(transactionViewModel.getHash().hexString()+t.hexString(), transactionViewModel.getHash().hexString(), t.hexString()); // h -> t + } + for (Hash b : branch){ + this.graph.graph.addEdge(transactionViewModel.getHash().hexString()+b.hexString(), transactionViewModel.getHash().hexString(), b.hexString()); // h -> t + } + org.graphstream.graph.Node graphNode = graph.graph.getNode(transactionViewModel.getHash().hexString()); + graphNode.addAttribute("ui.label", transactionViewModel.getHash().hexString().substring(0,10)); + graphNode.addAttribute("ui.style", "fill-color: rgb(255,165,0); stroke-color: rgb(30,144,255); stroke-width: 2px;"); + } graph.addNode(transactionViewModel.getHash().hexString(), transactionViewModel.getTrunkTransactionHash().hexString(), transactionViewModel.getBranchTransactionHash().hexString()); } } From 5fa134b50509f6956573deac6ac5ee05112706d7 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 9 Jul 2019 18:26:53 +0200 Subject: [PATCH 089/223] getRoundIndex from RoundViewModel, new genesis time --- .../service/milestone/impl/LatestMilestoneTrackerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 44a7bbd2..47ff63f8 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -154,7 +154,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP allNominees = new HashSet<>(); allNominees.addAll(nomineeTracker.getLatestNominees()); - genesisTime = 1562328163995L; + genesisTime = 1562590985837L; //currentRoundIndex = getRound(System.currentTimeMillis()); System.out.println("current time: " + System.currentTimeMillis()); //System.out.println("current round: " + getRound(System.currentTimeMillis())); @@ -239,7 +239,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw //System.out.println("Hash: " + transaction.getHash().hexString() + ", round: " + milestoneService.getRoundIndex(transaction)); //System.out.println("Current Round: " + getCurrentRoundIndex()); - int roundIndex = milestoneService.getRoundIndex(transaction); + int roundIndex = RoundViewModel.getRoundIndex(transaction); int currentRound = getCurrentRoundIndex(); // todo what happens if this method isn't called for that round and it passes the start round (do we need <= here?) From 065bc7807ce73d89e43160c0308a4f7c7baaf92e Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:39:12 +0200 Subject: [PATCH 090/223] add GENESIS_TIME and ROUND_DURATION to config --- .../java/net/helix/hlx/conf/BaseHelixConfig.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index d6b8450e..9c76621f 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -113,6 +113,8 @@ public abstract class BaseHelixConfig implements HelixConfig { protected int msDelay = Defaults.MS_DELAY; protected int minDelay = Defaults.MS_MIN_DELAY; protected Hash cooAddress = HashFactory.ADDRESS.create(Defaults.COORDINATOR_ADDRESS); + protected long genesisTime = Defaults.GENESIS_TIME; + protected int roundDuration = Defaults.ROUND_DURATION; //Spammer protected int spamDelay = Defaults.SPAM_DELAY; @@ -777,6 +779,16 @@ public int getMinDelay() { @Parameter(names = {"--min-delay"}, description = MilestoneConfig.Descriptions.MS_MIN_DELAY) protected void setMinDelay(int minDelay) { this.minDelay = minDelay; } + @Override + public long getGenesisTime() { + return genesisTime; + } + + @Override + public int getRoundDuration() { + return roundDuration; + } + @Override public boolean isPoWDisabled() { return powDisabled; @@ -887,6 +899,8 @@ public interface Defaults { String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; int MS_DELAY = 0; int MS_MIN_DELAY = 5; + long GENESIS_TIME = 1563030832860L; + int ROUND_DURATION = 5000; //Snapshot boolean LOCAL_SNAPSHOTS_ENABLED = true; From e68c4a503b7048eb24943cc6b4297e6978353341 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:40:54 +0200 Subject: [PATCH 091/223] get tips first, get previous round and build merkle tree instead of taking it from latest snapshot --- src/main/java/net/helix/hlx/service/API.java | 66 ++++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index f044d3b3..8de9eee6 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -11,6 +11,7 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.TransactionHash; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.model.persistables.Transaction; import net.helix.hlx.model.persistables.Bundle; import net.helix.hlx.network.Neighbor; @@ -619,7 +620,9 @@ public void storeTransactionsStatement(final List txHex) throws Exceptio //store transactions if(transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot())) { // v transactionViewModel.setArrivalTime(System.currentTimeMillis() / 1000L); - transactionValidator.updateStatus(transactionViewModel); + if (transactionViewModel.isMilestoneBundle(tangle) == null) { + transactionValidator.updateStatus(transactionViewModel); + } transactionViewModel.updateSender("local"); transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); System.out.println("published tx: " + transactionViewModel.getHash().hexString()); @@ -629,7 +632,7 @@ public void storeTransactionsStatement(final List txHex) throws Exceptio TransactionViewModel milestoneTx; if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ Set trunk = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); - Set branch =RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx); + Set branch = RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx); for (Hash t : trunk){ this.graph.graph.addEdge(transactionViewModel.getHash().hexString()+t.hexString(), transactionViewModel.getHash().hexString(), t.hexString()); // h -> t } @@ -1505,6 +1508,32 @@ private void attachStoreAndBroadcast(final String address, final String message, public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int incrementIndex) throws Exception { + // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) + List confirmedTips = new LinkedList<>(); + + System.out.println("Tips (" + tipsViewModel.getTips().size() + ")"); + snapshotProvider.getLatestSnapshot().lockRead(); + try { + WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); + for (Hash transaction : tipsViewModel.getTips()) { + TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); + //System.out.println("Tip: " + transaction.hexString()); + //System.out.println("bundle valid: " + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size()); + //System.out.println("walker valid: " + walkValidator.isValid(transaction)); + if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && + txVM.getCurrentIndex() == 0 && + txVM.isSolid() && + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { + if (walkValidator.isValid(transaction)) { + System.out.println(transaction.hexString()); + confirmedTips.add(transaction); + } + } + } + } finally { + snapshotProvider.getLatestSnapshot().unlockRead(); + } + // get round int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); long nextIndex = currentRoundIndex + incrementIndex; @@ -1565,29 +1594,6 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri System.arraycopy(signature, 0, txMilestone, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); } - // get confirming tips - List confirmedTips = new LinkedList<>(); - - System.out.println("Tips (" + tipsViewModel.getTips().size() + ")"); - snapshotProvider.getLatestSnapshot().lockRead(); - try { - WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); - for (Hash transaction : tipsViewModel.getTips()) { - TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); - if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && - txVM.getCurrentIndex() == 0 && - txVM.isSolid() && - BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { - if (walkValidator.isValid(transaction)) { - System.out.println(transaction.hexString()); - confirmedTips.add(transaction); - } - } - } - } finally { - snapshotProvider.getLatestSnapshot().unlockRead(); - } - // write confirmed tips into signature message fragment of txTips for (int i=0; i txToApprove = new ArrayList<>(); - System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); + //System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); if(RoundViewModel.latest(tangle) == null) { txToApprove.add(Hash.NULL_HASH); txToApprove.add(Hash.NULL_HASH); } else { // trunk - txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); // merkle root of latest milestones + RoundViewModel previousRound = RoundViewModel.get(tangle, currentRoundIndex - 1); + System.out.println("Previous Round: " + previousRound.toString()); + List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); + txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones + //txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); + System.out.println("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0).hexString()); //branch List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips + System.out.println("Branch (Tips): " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0).hexString()); } // attach, broadcast and store From edddc4d86352c7aa21586922210f596f9471fb53 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:42:19 +0200 Subject: [PATCH 092/223] logs --- .../helix/hlx/controllers/RoundViewModel.java | 28 ++++++++----------- .../ledger/impl/LedgerServiceImpl.java | 8 +++--- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 7e770bb3..7d1a08f8 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -235,21 +235,19 @@ public static int getRoundIndex(TransactionViewModel milestoneTransaction) { // todo this may be very inefficient public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ - System.out.println("Get Milestone Trunk"); Set trunk = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - System.out.println("round: " + round); - System.out.println("index: " + transaction.getCurrentIndex()); - // idx = n: milestone merkle root in trunk and tips merkle root in branch + System.out.println("Get Milestone Trunk: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); + // idx = n: milestone merkle root in trunk if (transaction.getCurrentIndex() == transaction.lastIndex()) { // add previous milestones to non analyzed transactions Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - System.out.println("merkleRoot: " + transaction.getTrunkTransactionHash()); - System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0)); + System.out.println("merkleRoot: " + transaction.getTrunkTransactionHash().hexString()); + System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("trunk: "); + System.out.println("trunk (prev. milestones): "); if (prevMilestones.isEmpty()){ trunk.add(Hash.NULL_HASH); System.out.println(Hash.NULL_HASH.hexString()); @@ -262,18 +260,16 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr else { // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash trunk.add(transaction.getTrunkTransactionHash()); - System.out.println("trunk: " + transaction.getTrunkTransactionHash().hexString()); + System.out.println("trunk (bundle): " + transaction.getTrunkTransactionHash().hexString()); } return trunk; } public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ - System.out.println("Get Milestone Branch"); Set branch = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - System.out.println("round: " + round); - System.out.println("index: " + transaction.getCurrentIndex()); + System.out.println("Get Milestone Branch: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk and tips merkle root in branch if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root @@ -282,7 +278,7 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("branch: "); + System.out.println("branch (tips): "); if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); System.out.println(Hash.NULL_HASH.hexString()); @@ -294,13 +290,13 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t } else { // add previous milestones to non analyzed transactions - Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); + Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); //todo null pointer exception System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - System.out.println("merkleRoot: " + transaction.getBranchTransactionHash()); - System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0)); + System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); + System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("branch: "); + System.out.println("branch (prev. milestones): "); if (prevMilestones.isEmpty()){ branch.add(Hash.NULL_HASH); System.out.println(Hash.NULL_HASH.hexString()); diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index eafaa41c..dc46dc82 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -92,9 +92,9 @@ public boolean updateDiff(Set approvedHashes, final Map diff, public void restoreLedgerState() throws LedgerException { try { Optional milestone = milestoneService.findLatestProcessedSolidRoundInDatabase(); - System.out.println(milestone); + //System.out.println(milestone); if (milestone.isPresent()) { - System.out.println(milestone.get().index()); + //System.out.println(milestone.get().index()); snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.get().index()); } } catch (Exception e) { @@ -288,8 +288,8 @@ private boolean generateStateDiff(RoundViewModel round) throws LedgerException { snapshotProvider.getLatestSnapshot().lockRead(); try { Set confirmedTips = milestoneService.getConfirmedTips(round.index()); - System.out.println("Confirmed Tips:"); - confirmedTips.forEach(tip -> System.out.println(tip.hexString())); + //System.out.println("Confirmed Tips:"); + //confirmedTips.forEach(tip -> System.out.println(tip.hexString())); Map balanceChanges = generateBalanceDiff(new HashSet<>(), confirmedTips == null? new HashSet<>() : confirmedTips, snapshotProvider.getLatestSnapshot().getIndex() + 1); successfullyProcessed = balanceChanges != null; From a1ee3906cbbcd82c2a2b6b22fa1ecba2a704964c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:42:35 +0200 Subject: [PATCH 093/223] calculate next round start --- .../java/net/helix/hlx/service/milestone/MSS.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MSS.java b/src/main/java/net/helix/hlx/service/milestone/MSS.java index 1c39579c..5f2a90b3 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MSS.java +++ b/src/main/java/net/helix/hlx/service/milestone/MSS.java @@ -39,10 +39,21 @@ public MSS(HelixConfig configuration, API api) { } } + private int getRound(long time) { + return (int) (time - config.getGenesisTime()) / config.getRoundDuration(); + } + + private long getStartTime(int round) { + return config.getGenesisTime() + (round * config.getRoundDuration()); + } + public void startScheduledExecutorService() { log.info("MSS scheduledExecutorService started."); - log.info("Submitting Milestones every: " + this.delay + "s."); - this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnablePublishMilestone(), 5, this.delay, TimeUnit.SECONDS); + log.info("Submitting Milestones every: " + config.getRoundDuration() + "s."); + int currentRound = getRound(System.currentTimeMillis()); + long startTimeNextRound = getStartTime(currentRound + 1); + log.info("Next Round starts in " + ((startTimeNextRound - System.currentTimeMillis()) / 1000) + "s."); + this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), config.getRoundDuration(), TimeUnit.MILLISECONDS); } private void publishMilestone() throws Exception { From 5bf20716712049a30b4f6fd831f9dfdeb46c4941 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:45:52 +0200 Subject: [PATCH 094/223] getGenesisTime and getRoundDuration --- src/main/java/net/helix/hlx/conf/MilestoneConfig.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index 30e1b39a..4229395e 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -25,10 +25,16 @@ public interface MilestoneConfig extends Config { */ int getMinDelay(); + long getGenesisTime(); + + int getRoundDuration(); + interface Descriptions { String VALIDATOR_ADDRESSES = "The addresses of nodes that are allowed to publish milestones"; String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable coordinator validation on testnet"; String MS_DELAY = "The desired milestone delay in seconds."; String MS_MIN_DELAY = "The minimum delay between publishing milestones."; + String GENESIS_TIME = "Time when the ledger started."; + String ROUND_DURATION = "Duration of a round in milli secounds."; } } From 27fff97722771e268fd2e6d49ac0138e60c4397f Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:46:51 +0200 Subject: [PATCH 095/223] take genesis time and round duration from config --- .../impl/LatestMilestoneTrackerImpl.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 47ff63f8..37dcb475 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -38,8 +38,6 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { */ private static final int RESCAN_INTERVAL = 1000; - private static final int ROUND_DURATION = 5000; - /** * Holds the logger of this class (a rate limited logger that doesn't spam the CLI output).
@@ -88,6 +86,8 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { private long genesisTime; + private int roundDuration; + /** * Holds the round index of the latest round that we have seen / processed.
*/ @@ -149,15 +149,16 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP this.milestoneSolidifier = milestoneSolidifier; this.nomineeTracker = nomineeTracker; - setCurrentNominees(nomineeTracker.getLatestNominees()); - allNominees = new HashSet<>(); allNominees.addAll(nomineeTracker.getLatestNominees()); - genesisTime = 1562590985837L; - //currentRoundIndex = getRound(System.currentTimeMillis()); - System.out.println("current time: " + System.currentTimeMillis()); - //System.out.println("current round: " + getRound(System.currentTimeMillis())); + System.out.println("Current Time: " + System.currentTimeMillis()); + System.out.println("Genesis Time: " + config.getGenesisTime()); + System.out.println("Round Duration: " + config.getRoundDuration()); + genesisTime = config.getGenesisTime(); + roundDuration = config.getRoundDuration(); + + setCurrentNominees(nomineeTracker.getLatestNominees()); //bootstrapLatestRoundIndex(); @@ -196,7 +197,7 @@ public int getCurrentRoundIndex() { } public int getRound(long time) { - return (int) (time - genesisTime) / ROUND_DURATION; + return (int) (time - genesisTime) / roundDuration; } @Override From e0ef3e36fa6a43b763e92bd7f3173b4b76302a54 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:48:19 +0200 Subject: [PATCH 096/223] use equals --- .../helix/hlx/service/milestone/impl/NomineeTrackerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java index 8e9e9b5e..05f65b7d 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java @@ -113,7 +113,7 @@ public MilestoneValidity validateNominees(TransactionViewModel transactionViewMo Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); - if (Curator_Address == senderAddress && validSignature) { + if (Curator_Address.equals(senderAddress) && validSignature) { return VALID; } else { return INVALID; From 136249ed79e777f03590ba2666818c652008d67d Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 14 Jul 2019 13:48:30 +0200 Subject: [PATCH 097/223] logs --- src/main/java/net/helix/hlx/BundleValidator.java | 14 +++++++++----- .../java/net/helix/hlx/TransactionValidator.java | 10 +++++++--- .../milestone/impl/MilestoneServiceImpl.java | 6 +++--- .../service/snapshot/impl/SnapshotServiceImpl.java | 8 +++++--- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/helix/hlx/BundleValidator.java b/src/main/java/net/helix/hlx/BundleValidator.java index 97e50051..edfd8928 100644 --- a/src/main/java/net/helix/hlx/BundleValidator.java +++ b/src/main/java/net/helix/hlx/BundleValidator.java @@ -5,6 +5,7 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.storage.Tangle; +import org.bouncycastle.util.encoders.Hex; import java.util.*; @@ -58,11 +59,13 @@ public static List> validate(Tangle tangle, Snapshot TransactionViewModel tail = TransactionViewModel.fromHash(tangle, tailHash); if (tail.getCurrentIndex() != 0 || tail.getValidity() == -1) { + System.out.println("Empty List"); return Collections.EMPTY_LIST; } List> transactions = new LinkedList<>(); final Map bundleTransactions = loadTransactionsFromTangle(tangle, tail); + // System.out.println("bundle size: " + bundleTransactions.size()); //we don't really iterate, we just pick the tail tx. See the if on the next line for (TransactionViewModel transactionViewModel : bundleTransactions.values()) { @@ -96,13 +99,9 @@ public static List> validate(Tangle tangle, Snapshot || bundleValue > TransactionViewModel.SUPPLY) ) { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); + System.out.println("Semantics Error!"); break; } - // todo address validation - /* if (transactionViewModel.value() != 0 && transactionViewModel.getAddressHash().bytes()[Sha3.HASH_LENGTH - 1] != 0) { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - break; - }*/ // It's supposed to become -3812798742493 after 3812798742493 and to go "down" to -1 but // we hope that no one will create such long bundles @@ -117,6 +116,8 @@ public static List> validate(Tangle tangle, Snapshot } sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); //verify bundle hash is correct + //System.out.println("Bundle Hash: " + instanceTransactionViewModels.get(0).getBundleHash().hexString()); + //System.out.println("recalculated Bundle Hash: " + Hex.toHexString(bundleHashBytes)); if (Arrays.equals(instanceTransactionViewModels.get(0).getBundleHash().bytes(), bundleHashBytes)) { //normalizing the bundle in preparation for signature verification Winternitz.normalizedBundle(bundleHashBytes, normalizedBundle); @@ -143,6 +144,7 @@ public static List> validate(Tangle tangle, Snapshot //signature verification if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); + System.out.println("Signature Error!"); break MAIN_LOOP; } } else { @@ -156,6 +158,7 @@ public static List> validate(Tangle tangle, Snapshot //bundle hash verification failed else { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); + System.out.println("Bundle Hash Error!"); } } //bundle validity status is known @@ -166,6 +169,7 @@ public static List> validate(Tangle tangle, Snapshot //total bundle value does not sum to 0 else { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); + System.out.println("Bundle Sum Error!"); } //break from main loop break; diff --git a/src/main/java/net/helix/hlx/TransactionValidator.java b/src/main/java/net/helix/hlx/TransactionValidator.java index 89ab8b91..4b89ab50 100644 --- a/src/main/java/net/helix/hlx/TransactionValidator.java +++ b/src/main/java/net/helix/hlx/TransactionValidator.java @@ -246,7 +246,7 @@ public boolean checkSolidity(Hash hash, boolean milestone) throws Exception { * @throws Exception if anything goes wrong while trying to solidify the transaction */ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTransactions) throws Exception { - System.out.println("Check Solidity"); + //System.out.println("Check Solidity"); if(fromHash(tangle, hash).isSolid()) { return true; } @@ -264,7 +264,7 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans } final TransactionViewModel transaction = fromHash(tangle, hashPointer); - System.out.println(hashPointer.hexString() + ", solid: " + transaction.isSolid() + ", has solid entry point: " + snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); + //System.out.println(hashPointer.hexString() + ", solid: " + transaction.isSolid() + ", has solid entry point: " + snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); if(!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { if (transaction.getType() == PREFILLED_SLOT) { solid = false; @@ -396,6 +396,7 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); if(transactionViewModel.getApprovers(tangle).size() == 0) { tipsViewModel.addTipHash(transactionViewModel.getHash()); + //System.out.println("Add Tip: " + transactionViewModel.getHash().hexString()); } else { TransactionViewModel milestoneTx; if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ @@ -403,10 +404,13 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx)); for (Hash parent : parents){ tipsViewModel.removeTipHash(parent); + //System.out.println("Remove Tip: " + parent.hexString()); } } else { tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); + //System.out.println("Remove Tip: " + transactionViewModel.getTrunkTransactionHash().hexString()); + //System.out.println("Remove Tip: " + transactionViewModel.getBranchTransactionHash().hexString()); } } @@ -439,7 +443,7 @@ private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { * @throws Exception */ private boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { - System.out.println("Quick Set Solid"); + //System.out.println("Quick Set Solid"); if(!transactionViewModel.isSolid()) { boolean solid = true; TransactionViewModel milestoneTx; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 07555811..fc7c4dd1 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -370,7 +370,7 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, Set processedTransactions, Graphstream graph) throws MilestoneException { - System.out.println("UPDATE ROUND INDEX"); + //System.out.println("UPDATE ROUND INDEX"); Set inconsistentMilestones = new HashSet<>(); try { @@ -379,7 +379,7 @@ private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIn for (Hash milestoneHash : round.getHashes()){ TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); updateRoundIndexOfSingleTransaction(milestoneTx, newIndex); - System.out.println("milestone: " + milestoneHash.hexString() + ", Snapshot: " + milestoneTx.snapshotIndex()); + //System.out.println("milestone: " + milestoneHash.hexString() + ", Snapshot: " + milestoneTx.snapshotIndex()); } // update confirmed transactions final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex)); @@ -394,7 +394,7 @@ private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIn prepareRoundIndexUpdate(transactionViewModel, correctIndex, newIndex, inconsistentMilestones, transactionsToUpdate); updateRoundIndexOfSingleTransaction(transactionViewModel, newIndex); - System.out.println("tx: " + transactionViewModel.getHash().hexString() + ", Snapshot: " + transactionViewModel.snapshotIndex()); + //System.out.println("tx: " + transactionViewModel.getHash().hexString() + ", Snapshot: " + transactionViewModel.snapshotIndex()); if (graph != null) { graph.setConfirmed(transactionViewModel.getHash().hexString(), 1); } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index efd08b87..07d83018 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -156,9 +156,11 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna //store merkle root List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(lastAppliedRound.getHashes())); snapshot.setHash(merkleTree.get(merkleTree.size() - 1).get(0)); - System.out.println("Hashes:"); - lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); - System.out.println("snapshot hash: " + snapshot.getHash().hexString()); + //System.out.println("Milestones :"); + //lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); + System.out.println("Snapshot Hash: " + snapshot.getHash().hexString()); + //System.out.println("Snapshot:"); + //snapshot.getBalances().forEach((a,b) -> System.out.println("Address: " + a.hexString() + ", " + b)); //TODO: get start time of round (need genesisTime) /*TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, From 9db5de3ae7b7e7092b09edd5ff870b9eee1758bb Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 15:28:22 +0200 Subject: [PATCH 098/223] logs --- .../helix/hlx/controllers/RoundViewModel.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 7d1a08f8..71decc94 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -237,30 +237,30 @@ public static int getRoundIndex(TransactionViewModel milestoneTransaction) { public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ Set trunk = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - System.out.println("Get Milestone Trunk: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); + //System.out.println("Get Milestone Trunk: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk if (transaction.getCurrentIndex() == transaction.lastIndex()) { // add previous milestones to non analyzed transactions Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); - System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); + //System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - System.out.println("merkleRoot: " + transaction.getTrunkTransactionHash().hexString()); - System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); + //System.out.println("merkleRoot: " + transaction.getTrunkTransactionHash().hexString()); + //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("trunk (prev. milestones): "); + //System.out.println("trunk (prev. milestones): "); if (prevMilestones.isEmpty()){ trunk.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH.hexString()); + //System.out.println(Hash.NULL_HASH.hexString()); } else { trunk.addAll(prevMilestones); - prevMilestones.forEach(c -> System.out.println(c.hexString())); + //prevMilestones.forEach(c -> System.out.println(c.hexString())); } } } else { // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash trunk.add(transaction.getTrunkTransactionHash()); - System.out.println("trunk (bundle): " + transaction.getTrunkTransactionHash().hexString()); + //System.out.println("trunk (bundle): " + transaction.getTrunkTransactionHash().hexString()); } return trunk; @@ -269,40 +269,40 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ Set branch = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - System.out.println("Get Milestone Branch: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); + //System.out.println("Get Milestone Branch: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk and tips merkle root in branch if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root Set confirmedTips = getTipSet(tangle, milestoneTx.getHash()); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); - System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); - System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); + //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); + //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("branch (tips): "); + //System.out.println("branch (tips): "); if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH.hexString()); + //System.out.println(Hash.NULL_HASH.hexString()); } else { branch.addAll(confirmedTips); - confirmedTips.forEach(c -> System.out.println(c.hexString())); + //confirmedTips.forEach(c -> System.out.println(c.hexString())); } } } else { // add previous milestones to non analyzed transactions Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); //todo null pointer exception - System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); + //System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); - System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); + //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); + //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("branch (prev. milestones): "); + //System.out.println("branch (prev. milestones): "); if (prevMilestones.isEmpty()){ branch.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH.hexString()); + //System.out.println(Hash.NULL_HASH.hexString()); } else { branch.addAll(prevMilestones); - prevMilestones.forEach(c -> System.out.println(c.hexString())); + //prevMilestones.forEach(c -> System.out.println(c.hexString())); } } } From d5434971138c5c0d330c5507c5beb5bf6e4e2c13 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 16:53:15 +0200 Subject: [PATCH 099/223] update genesis time --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 9c76621f..134cb70f 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -899,7 +899,7 @@ public interface Defaults { String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; int MS_DELAY = 0; int MS_MIN_DELAY = 5; - long GENESIS_TIME = 1563030832860L; + long GENESIS_TIME = 1563285823554L; int ROUND_DURATION = 5000; //Snapshot From 64abfff3d8fdc1193587592738d21f346b15ab3d Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 17:02:34 +0200 Subject: [PATCH 100/223] update milestone references in graph when applying to ledger --- .../ledger/impl/LedgerServiceImpl.java | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index dc46dc82..c34fe70f 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -1,10 +1,13 @@ package net.helix.hlx.service.ledger.impl; import net.helix.hlx.BundleValidator; +import net.helix.hlx.controllers.BundleViewModel; import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.StateDiffViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; +import net.helix.hlx.model.persistables.Bundle; +import net.helix.hlx.model.persistables.Transaction; import net.helix.hlx.service.ledger.LedgerException; import net.helix.hlx.service.ledger.LedgerService; import net.helix.hlx.service.milestone.MilestoneService; @@ -103,16 +106,36 @@ public void restoreLedgerState() throws LedgerException { } @Override - public boolean applyMilestoneToLedger(RoundViewModel milestone) throws LedgerException { - System.out.println("Apply Round " + milestone.index() + " to ledger"); + public boolean applyMilestoneToLedger(RoundViewModel round) throws LedgerException { + System.out.println("Apply Round " + round.index() + " to ledger"); if (graph != null) { - for (Hash milestoneHash : milestone.getHashes()) { - graph.setMilestone(milestoneHash.hexString(), milestone.index()); + for (Hash milestoneHash : round.getHashes()) { + try { + TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); + BundleViewModel bundle = BundleViewModel.load(tangle, milestoneTx.getBundleHash()); + for (Hash txHash : bundle.getHashes()) { + TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, txHash); + Set trunk = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); + Set branch = RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx); + for (Hash t : trunk) { + this.graph.graph.addEdge(transactionViewModel.getHash().hexString() + t.hexString(), transactionViewModel.getHash().hexString(), t.hexString()); // h -> t + } + for (Hash b : branch) { + this.graph.graph.addEdge(transactionViewModel.getHash().hexString() + b.hexString(), transactionViewModel.getHash().hexString(), b.hexString()); // h -> t + } + org.graphstream.graph.Node graphNode = graph.graph.getNode(transactionViewModel.getHash().hexString()); + graphNode.addAttribute("ui.label", transactionViewModel.getHash().hexString().substring(0, 10)); + graphNode.addAttribute("ui.style", "fill-color: rgb(255,165,0); stroke-color: rgb(30,144,255); stroke-width: 2px;"); + } + graph.setMilestone(milestoneHash.hexString(), round.index()); + } catch (Exception e) { + throw new LedgerException("unable to update milestone attributes in graph"); + } } } - if(generateStateDiff(milestone)) { + if(generateStateDiff(round)) { try { - snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.index()); + snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), round.index()); //System.out.println("Snapshot"); //snapshotProvider.getLatestSnapshot().getBalances().forEach((address, balance) -> System.out.println("Address: " + address.hexString() + ", " + balance)); } catch (SnapshotException e) { From e540319d8a736d0fffe0c9700248a0e5085a0ed5 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 17:03:19 +0200 Subject: [PATCH 101/223] remove round label for milestones --- src/main/java/net/helix/hlx/service/Graphstream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/Graphstream.java b/src/main/java/net/helix/hlx/service/Graphstream.java index 481a2113..a1407133 100644 --- a/src/main/java/net/helix/hlx/service/Graphstream.java +++ b/src/main/java/net/helix/hlx/service/Graphstream.java @@ -41,7 +41,7 @@ public void setMilestone(String hash, int index) { try { Node graphNode = graph.getNode(hash); graphNode.addAttribute("ui.style", "fill-color: rgb(30,144,255); size: 20px; stroke-mode: plain;"); - graphNode.addAttribute("ui.label", index); + //graphNode.addAttribute("ui.label", hash.substring(0,10) + "(" + index + ")"); } catch(Exception e) { log.error("couldn't find a node with label " + hash, e); } From 54d768dab720d65a35d2148e8c2b166da693065a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 17:07:36 +0200 Subject: [PATCH 102/223] don't update milestone references in graph before they are recognized as milestones --- src/main/java/net/helix/hlx/network/Node.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/main/java/net/helix/hlx/network/Node.java b/src/main/java/net/helix/hlx/network/Node.java index c5113ca2..45aecf59 100644 --- a/src/main/java/net/helix/hlx/network/Node.java +++ b/src/main/java/net/helix/hlx/network/Node.java @@ -464,20 +464,6 @@ public void processReceivedData(TransactionViewModel receivedTransactionViewMode try { stored = receivedTransactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot()); if (graph != null) { - TransactionViewModel milestoneTx; - if ((milestoneTx = receivedTransactionViewModel.isMilestoneBundle(tangle)) != null){ - Set trunk = RoundViewModel.getMilestoneTrunk(tangle, receivedTransactionViewModel, milestoneTx); - Set branch =RoundViewModel.getMilestoneBranch(tangle, receivedTransactionViewModel, milestoneTx); - for (Hash t : trunk){ - this.graph.graph.addEdge(receivedTransactionViewModel.getHash().hexString()+t.hexString(), receivedTransactionViewModel.getHash().hexString(), t.hexString()); // h -> t - } - for (Hash b : branch){ - this.graph.graph.addEdge(receivedTransactionViewModel.getHash().hexString()+b.hexString(), receivedTransactionViewModel.getHash().hexString(), b.hexString()); // h -> t - } - org.graphstream.graph.Node graphNode = graph.graph.getNode(receivedTransactionViewModel.getHash().hexString()); - graphNode.addAttribute("ui.label", receivedTransactionViewModel.getHash().hexString().substring(0,10)); - graphNode.addAttribute("ui.style", "fill-color: rgb(255,165,0); stroke-color: rgb(30,144,255); stroke-width: 2px;"); - } graph.addNode(receivedTransactionViewModel.getHash().hexString(), receivedTransactionViewModel.getTrunkTransactionHash().hexString(), receivedTransactionViewModel.getBranchTransactionHash().hexString()); } } catch (Exception e) { From c498e68a57c6995a4a1d27d178dea1d9066f6d3a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 17:09:26 +0200 Subject: [PATCH 103/223] enable sending multiple tx with confirmed tips when there are more than 16 --- src/main/java/net/helix/hlx/service/API.java | 72 +++++++++++--------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 8de9eee6..26bfdf74 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -629,20 +629,6 @@ public void storeTransactionsStatement(final List txHex) throws Exceptio } if (graph != null) { - TransactionViewModel milestoneTx; - if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ - Set trunk = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); - Set branch = RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx); - for (Hash t : trunk){ - this.graph.graph.addEdge(transactionViewModel.getHash().hexString()+t.hexString(), transactionViewModel.getHash().hexString(), t.hexString()); // h -> t - } - for (Hash b : branch){ - this.graph.graph.addEdge(transactionViewModel.getHash().hexString()+b.hexString(), transactionViewModel.getHash().hexString(), b.hexString()); // h -> t - } - org.graphstream.graph.Node graphNode = graph.graph.getNode(transactionViewModel.getHash().hexString()); - graphNode.addAttribute("ui.label", transactionViewModel.getHash().hexString().substring(0,10)); - graphNode.addAttribute("ui.style", "fill-color: rgb(255,165,0); stroke-color: rgb(30,144,255); stroke-width: 2px;"); - } graph.addNode(transactionViewModel.getHash().hexString(), transactionViewModel.getTrunkTransactionHash().hexString(), transactionViewModel.getBranchTransactionHash().hexString()); } } @@ -1517,15 +1503,18 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); for (Hash transaction : tipsViewModel.getTips()) { TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); - //System.out.println("Tip: " + transaction.hexString()); + System.out.println("Tip: " + transaction.hexString()); //System.out.println("bundle valid: " + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size()); + //System.out.println("type: " + txVM.getType()); + //System.out.println("index: " + txVM.getCurrentIndex()); + //System.out.println("solid: " + txVM.isSolid()); //System.out.println("walker valid: " + walkValidator.isValid(transaction)); if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && txVM.getCurrentIndex() == 0 && txVM.isSolid() && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { if (walkValidator.isValid(transaction)) { - System.out.println(transaction.hexString()); + System.out.println("(selected)"); confirmedTips.add(transaction); } } @@ -1538,25 +1527,32 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); long nextIndex = currentRoundIndex + incrementIndex; - // list of confirming tips - byte[] txTips = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(2L), 0, txTips, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(2L), 0, txTips, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txTips, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - - // siblings for merkle tree. - byte[] txSibling = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(2L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + // get number of transactions needed for tips + long n = (long) (confirmedTips.size()/16) + 1; // contain a signature that signs the siblings and thereby ensures the integrity. byte[] txMilestone = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, txMilestone, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(2L), 0, txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txMilestone, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); System.arraycopy(Serializer.serialize(nextIndex), 0, txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + // siblings for merkle tree. + byte[] txSibling = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + + // list of confirming tips + List txTips = new ArrayList<>(); + for (long i = 2L; i <= (1L + n); i++) { + byte[] tx = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(i), 0, tx, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, tx, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, tx, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + txTips.add(tx); + } + // calculate bundle hash Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); @@ -1565,14 +1561,18 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri sponge.absorb(milestoneEssence, 0, milestoneEssence.length); byte[] siblingEssence = Arrays.copyOfRange(txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); sponge.absorb(siblingEssence, 0, siblingEssence.length); - byte[] tipsEssence = Arrays.copyOfRange(txTips, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(tipsEssence, 0, tipsEssence.length); + for (byte[] tx : txTips) { + byte[] tipsEssence = Arrays.copyOfRange(tx, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(tipsEssence, 0, tipsEssence.length); + } byte[] bundleHash = new byte[32]; sponge.squeeze(bundleHash, 0, bundleHash.length); - System.arraycopy(bundleHash, 0, txTips, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); System.arraycopy(bundleHash, 0, txMilestone, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + for (byte[] tx : txTips) { + System.arraycopy(bundleHash, 0, tx, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + } if (sign) { // Get merkle path and store in signatureMessageFragment of Sibling Transaction @@ -1596,7 +1596,8 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri // write confirmed tips into signature message fragment of txTips for (int i=0; i> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); @@ -1621,7 +1623,9 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri // attach, broadcast and store List transactions = new ArrayList<>(); - transactions.add(Hex.toHexString(txTips)); + for (int i = txTips.size()-1; i >= 0; i--) { + transactions.add(Hex.toHexString(txTips.get(i))); + } transactions.add(Hex.toHexString(txSibling)); transactions.add(Hex.toHexString(txMilestone)); List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); From 7a8b35835c6a82954dd4ebfb5d6035713bd99459 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 17:11:23 +0200 Subject: [PATCH 104/223] rename --- .../hlx/service/milestone/impl/MilestoneSolidifierImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneSolidifierImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneSolidifierImpl.java index cc46a47c..651cfcda 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneSolidifierImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneSolidifierImpl.java @@ -309,7 +309,7 @@ private Map.Entry getNextSolidificationCandidate() { */ private boolean isSolid(Map.Entry currentEntry) { if (unsolidMilestonesPool.size() > 1) { - log.info("Solidifying milestone #" + currentEntry.getValue() + + log.info("Solidifying round #" + currentEntry.getValue() + " [" + milestonesToSolidify.size() + " / " + unsolidMilestonesPool.size() + "]"); } @@ -317,7 +317,7 @@ private boolean isSolid(Map.Entry currentEntry) { return transactionValidator.checkSolidity(currentEntry.getKey(), true, SOLIDIFICATION_TRANSACTIONS_LIMIT); } catch (Exception e) { - log.error("Error while solidifying milestone #" + currentEntry.getValue(), e); + log.error("Error while solidifying round #" + currentEntry.getValue(), e); return false; } From aadb9383f4365d31215f99079b98d5d4837b7d0b Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 16 Jul 2019 17:15:48 +0200 Subject: [PATCH 105/223] update approvees column in database of tx that are referenced by milestones --- src/main/java/net/helix/hlx/Helix.java | 2 +- .../hlx/controllers/ApproveeViewModel.java | 1 + .../helix/hlx/controllers/RoundViewModel.java | 36 ++++++++++++------- .../milestone/impl/MilestoneServiceImpl.java | 9 ++++- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index 45e89f71..3a8cafe2 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -209,7 +209,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx if (localSnapshotManager != null) { localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); } - milestoneService.init(tangle, snapshotProvider, snapshotService, configuration); + milestoneService.init(tangle, snapshotProvider, snapshotService, transactionValidator, configuration); nomineeTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, nomineeTracker, configuration); latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService, diff --git a/src/main/java/net/helix/hlx/controllers/ApproveeViewModel.java b/src/main/java/net/helix/hlx/controllers/ApproveeViewModel.java index 3000fb34..7a20ac33 100644 --- a/src/main/java/net/helix/hlx/controllers/ApproveeViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/ApproveeViewModel.java @@ -104,6 +104,7 @@ public Indexable getIndex() { * @return Set transaction hashes */ public Set getHashes() { + self = self == null ? new Approvee(): self; return self.set; } diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 71decc94..6b2439a5 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -1,5 +1,6 @@ package net.helix.hlx.controllers; +import net.helix.hlx.TransactionValidator; import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; @@ -237,7 +238,7 @@ public static int getRoundIndex(TransactionViewModel milestoneTransaction) { public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ Set trunk = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - //System.out.println("Get Milestone Trunk: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); + System.out.println("Get Milestone Trunk: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk if (transaction.getCurrentIndex() == transaction.lastIndex()) { // add previous milestones to non analyzed transactions @@ -247,20 +248,20 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr //System.out.println("merkleRoot: " + transaction.getTrunkTransactionHash().hexString()); //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - //System.out.println("trunk (prev. milestones): "); + System.out.println("trunk (prev. milestones): "); if (prevMilestones.isEmpty()){ trunk.add(Hash.NULL_HASH); - //System.out.println(Hash.NULL_HASH.hexString()); + System.out.println(Hash.NULL_HASH.hexString()); } else { trunk.addAll(prevMilestones); - //prevMilestones.forEach(c -> System.out.println(c.hexString())); + prevMilestones.forEach(c -> System.out.println(c.hexString())); } } } else { // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash trunk.add(transaction.getTrunkTransactionHash()); - //System.out.println("trunk (bundle): " + transaction.getTrunkTransactionHash().hexString()); + System.out.println("trunk (bundle): " + transaction.getTrunkTransactionHash().hexString()); } return trunk; @@ -269,7 +270,7 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ Set branch = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - //System.out.println("Get Milestone Branch: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); + System.out.println("Get Milestone Branch: hash = " + transaction.getHash().hexString() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk and tips merkle root in branch if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root @@ -278,13 +279,13 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - //System.out.println("branch (tips): "); + System.out.println("branch (tips): "); if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); - //System.out.println(Hash.NULL_HASH.hexString()); + System.out.println(Hash.NULL_HASH.hexString()); } else { branch.addAll(confirmedTips); - //confirmedTips.forEach(c -> System.out.println(c.hexString())); + confirmedTips.forEach(c -> System.out.println(c.hexString())); } } } @@ -296,13 +297,13 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - //System.out.println("branch (prev. milestones): "); + System.out.println("branch (prev. milestones): "); if (prevMilestones.isEmpty()){ branch.add(Hash.NULL_HASH); - //System.out.println(Hash.NULL_HASH.hexString()); + System.out.println(Hash.NULL_HASH.hexString()); } else { branch.addAll(prevMilestones); - //prevMilestones.forEach(c -> System.out.println(c.hexString())); + prevMilestones.forEach(c -> System.out.println(c.hexString())); } } } @@ -382,6 +383,17 @@ public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { return (Hash) confirmingMilestones.toArray()[0]; } + public static void updateApprovees(Tangle tangle, TransactionValidator transactionValidator, List milestoneBundle, Hash milestone) throws Exception{ + Set confirmedTips = getTipSet(tangle, milestone); + // last transaction references tips + for (Hash tip : confirmedTips){ + ApproveeViewModel approvee = new ApproveeViewModel(tip); + approvee.addHash(milestoneBundle.get(milestoneBundle.size() - 1).getHash()); + approvee.store(tangle); + transactionValidator.updateStatus(TransactionViewModel.fromHash(tangle, tip)); + } + } + /** * Save the {@link Round} object, indexed by its integer index, to the database. * diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index fc7c4dd1..cd4cc7ac 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -26,6 +26,7 @@ import net.helix.hlx.utils.Serializer; import net.helix.hlx.utils.dag.DAGHelper; import net.helix.hlx.service.Graphstream; +import net.helix.hlx.TransactionValidator; import net.helix.hlx.utils.dag.TraversalException; import org.bouncycastle.util.encoders.Hex; @@ -68,6 +69,9 @@ public class MilestoneServiceImpl implements MilestoneService { */ private SnapshotService snapshotService; + + TransactionValidator transactionValidator; + /** * Holds the config with important milestone specific settings.
*/ @@ -90,11 +94,12 @@ public class MilestoneServiceImpl implements MilestoneService { * @param config config with important milestone specific settings * @return the initialized instance itself to allow chaining */ - public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotService snapshotService, ConsensusConfig config) { + public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotService snapshotService, TransactionValidator transactionValidator, ConsensusConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.snapshotService = snapshotService; + this.transactionValidator = transactionValidator; this.config = config; return this; @@ -203,6 +208,8 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { + //update approvees + RoundViewModel.updateApprovees(tangle, transactionValidator, bundleTransactionViewModels, transactionViewModel.getHash()); // if we find a NEW milestone for a round that already has been processed // and considered as solid (there is already a snapshot without this milestone) From 14d4dbe6829b42faa0647500161886d382e3f4a2 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 23 Jul 2019 14:17:56 +0200 Subject: [PATCH 106/223] fix getBalances, reference NULL_HASH if previous round doesn't exist --- src/main/java/net/helix/hlx/service/API.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 26bfdf74..13389b63 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1078,8 +1078,7 @@ private AbstractResponse getBalancesStatement(List addresses, final int index = snapshotProvider.getLatestSnapshot().getIndex(); if (tips == null || tips.size() == 0) { - //todo merkle root of latest milestones - hashes = Collections.singletonList(RoundViewModel.latest(tangle).getRandomConfirmingMilestone(tangle)); + hashes = new LinkedList<>(RoundViewModel.get(tangle, index).getHashes()); } else { hashes = tips.stream() .map(tip -> (HashFactory.TRANSACTION.create(tip))) @@ -1609,12 +1608,17 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri } else { // trunk // todo what happens if there is no entry for the previous round ? + // System.out.println("Get previous round #" + (currentRoundIndex - 1)); RoundViewModel previousRound = RoundViewModel.get(tangle, currentRoundIndex - 1); - System.out.println("Previous Round: " + previousRound.toString()); - List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); - txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones - //txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); - System.out.println("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0).hexString()); + if (previousRound == null){ + txToApprove.add(Hash.NULL_HASH); + } else { + System.out.println("Previous Round: " + previousRound.toString()); + List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); + txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones + //txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); + System.out.println("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0).hexString()); + } //branch List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips From 8108a299e98ae1356fc59301469196c400cd1fd3 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 23 Jul 2019 14:20:50 +0200 Subject: [PATCH 107/223] do a 1s round pause between each round to make sure previous round is applied to ledger before next round starts --- .../hlx/service/milestone/LatestMilestoneTracker.java | 4 ++++ .../milestone/impl/LatestMilestoneTrackerImpl.java | 11 ++++++++++- .../impl/LatestSolidMilestoneTrackerImpl.java | 3 ++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java index 9072711a..ad8edf95 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java @@ -27,6 +27,10 @@ public interface LatestMilestoneTracker { */ int getCurrentRoundIndex(); + int getRound(long time); + + boolean isRoundActive(long time); + /** * Returns the transaction hash of the latest milestone that was seen by this tracker.
*
diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 37dcb475..31069287 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -88,6 +88,8 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { private int roundDuration; + private int roundPause; + /** * Holds the round index of the latest round that we have seen / processed.
*/ @@ -157,6 +159,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP System.out.println("Round Duration: " + config.getRoundDuration()); genesisTime = config.getGenesisTime(); roundDuration = config.getRoundDuration(); + roundPause = 1000; //ms setCurrentNominees(nomineeTracker.getLatestNominees()); @@ -196,10 +199,16 @@ public int getCurrentRoundIndex() { return getRound(System.currentTimeMillis()); } + @Override public int getRound(long time) { return (int) (time - genesisTime) / roundDuration; } + @Override + public boolean isRoundActive(long time) { + return (time - genesisTime) % roundDuration < roundDuration - roundPause; + } + @Override public Set getLatestRoundHashes() throws Exception{ int index = getCurrentRoundIndex(); @@ -274,7 +283,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw // - attachment timestamp is in correct time window for the index // - there doesn't already exist a milestone with the same address for that round System.out.println("attachment timestamp round: " + getRound(transaction.getAttachmentTimestamp())); - if (roundIndex == getRound(transaction.getAttachmentTimestamp())) { + if (roundIndex == getRound(transaction.getAttachmentTimestamp()) && isRoundActive(transaction.getAttachmentTimestamp())) { RoundViewModel currentRoundViewModel; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index f8b298cf..4982b315 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -144,7 +144,8 @@ public void trackLatestSolidMilestone() throws MilestoneException { try { int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); RoundViewModel nextRound; - while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex() - 1)) { + while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex()) + && (currentSolidRoundIndex != latestMilestoneTracker.getCurrentRoundIndex() - 1 || !latestMilestoneTracker.isRoundActive(System.currentTimeMillis()))) { nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); From e7380b8e2eae6c52666172b1ac4359f3995e8f7a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 23 Jul 2019 14:22:10 +0200 Subject: [PATCH 108/223] comment --- .../helix/hlx/service/milestone/impl/MilestoneServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index cd4cc7ac..7fec7f6a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -208,7 +208,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { - //update approvees + //update tip status of approved tips RoundViewModel.updateApprovees(tangle, transactionValidator, bundleTransactionViewModels, transactionViewModel.getHash()); // if we find a NEW milestone for a round that already has been processed From c2b93c97c5cf6ae55741fa874db8e8d52d84b0d5 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 23 Jul 2019 14:22:22 +0200 Subject: [PATCH 109/223] genesis time --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 134cb70f..d9f68bb3 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -899,7 +899,7 @@ public interface Defaults { String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; int MS_DELAY = 0; int MS_MIN_DELAY = 5; - long GENESIS_TIME = 1563285823554L; + long GENESIS_TIME = 1563883196073L; int ROUND_DURATION = 5000; //Snapshot From d69807dfe6768ce92d063809f2facec7f06e28be Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 23 Jul 2019 14:41:18 +0200 Subject: [PATCH 110/223] if no prev round, take NULL_HASH --- .../helix/hlx/controllers/RoundViewModel.java | 56 +++++++++++-------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 6b2439a5..483d6eb4 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -242,19 +242,23 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr // idx = n: milestone merkle root in trunk if (transaction.getCurrentIndex() == transaction.lastIndex()) { // add previous milestones to non analyzed transactions - Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); - //System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - //System.out.println("merkleRoot: " + transaction.getTrunkTransactionHash().hexString()); - //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); - if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("trunk (prev. milestones): "); - if (prevMilestones.isEmpty()){ + RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); + if (prevMilestone == null) { + if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { trunk.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH.hexString()); - } else { - trunk.addAll(prevMilestones); - prevMilestones.forEach(c -> System.out.println(c.hexString())); + } + } else { + Set prevMilestones = prevMilestone.getHashes(); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + System.out.println("trunk (prev. milestones): "); + if (prevMilestones.isEmpty()) { + trunk.add(Hash.NULL_HASH); + System.out.println(Hash.NULL_HASH.hexString()); + } else { + trunk.addAll(prevMilestones); + prevMilestones.forEach(c -> System.out.println(c.hexString())); + } } } } @@ -291,19 +295,23 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t } else { // add previous milestones to non analyzed transactions - Set prevMilestones = RoundViewModel.get(tangle, round-1).getHashes(); //todo null pointer exception - //System.out.println("round: " + RoundViewModel.get(tangle, round-1).toString()); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); - //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("branch (prev. milestones): "); - if (prevMilestones.isEmpty()){ + RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); + if (prevMilestone == null) { + if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { branch.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH.hexString()); - } else { - branch.addAll(prevMilestones); - prevMilestones.forEach(c -> System.out.println(c.hexString())); + } + } else { + Set prevMilestones = prevMilestone.getHashes(); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + System.out.println("branch (prev. milestones): "); + if (prevMilestones.isEmpty()) { + branch.add(Hash.NULL_HASH); + System.out.println(Hash.NULL_HASH.hexString()); + } else { + branch.addAll(prevMilestones); + prevMilestones.forEach(c -> System.out.println(c.hexString())); + } } } } From 8e2bd406ee8e4922165796394ac6d12f01c40460 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 23 Jul 2019 16:30:07 +0200 Subject: [PATCH 111/223] update status of approver --- .../java/net/helix/hlx/controllers/RoundViewModel.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 483d6eb4..1ca4755d 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -393,13 +393,14 @@ public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { public static void updateApprovees(Tangle tangle, TransactionValidator transactionValidator, List milestoneBundle, Hash milestone) throws Exception{ Set confirmedTips = getTipSet(tangle, milestone); + TransactionViewModel lastTx = milestoneBundle.get(milestoneBundle.size() - 1); // last transaction references tips for (Hash tip : confirmedTips){ - ApproveeViewModel approvee = new ApproveeViewModel(tip); - approvee.addHash(milestoneBundle.get(milestoneBundle.size() - 1).getHash()); - approvee.store(tangle); - transactionValidator.updateStatus(TransactionViewModel.fromHash(tangle, tip)); + ApproveeViewModel approve = new ApproveeViewModel(tip); + approve.addHash(lastTx.getHash()); + approve.store(tangle); } + transactionValidator.updateStatus(TransactionViewModel.fromHash(tangle, lastTx.getHash())); } /** From 5eefff8d456df068a8f4d41cf853d02439cee54e Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 23 Jul 2019 16:30:59 +0200 Subject: [PATCH 112/223] set milestone before update approvees --- .../hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java | 1 - .../helix/hlx/service/milestone/impl/MilestoneServiceImpl.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 31069287..52df7ea9 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -314,7 +314,6 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw milestoneSolidifier.add(transaction.getHash(), roundIndex); } - transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); break; case INCOMPLETE: diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 7fec7f6a..74dabf4e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -208,6 +208,8 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { + transactionViewModel.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); + //update tip status of approved tips RoundViewModel.updateApprovees(tangle, transactionValidator, bundleTransactionViewModels, transactionViewModel.getHash()); From 9736c8cb74de7b362d8a6acdd390ebeaf2f71259 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 24 Jul 2019 16:51:15 +0200 Subject: [PATCH 113/223] rename milestonePublisher, add updateKeyfile() --- src/main/java/net/helix/hlx/HLX.java | 10 +++---- .../{MSS.java => MilestonePublisher.java} | 26 ++++++++++++++++--- 2 files changed, 28 insertions(+), 8 deletions(-) rename src/main/java/net/helix/hlx/service/milestone/{MSS.java => MilestonePublisher.java} (71%) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 9a6c5de8..e98e4e1b 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -8,7 +8,7 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.service.API; import net.helix.hlx.service.Spammer; -import net.helix.hlx.service.milestone.MSS; +import net.helix.hlx.service.milestone.MilestonePublisher; import net.helix.hlx.service.restserver.resteasy.RestEasy; import net.helix.hlx.utils.HelixIOUtils; import org.apache.commons.lang3.ArrayUtils; @@ -94,7 +94,7 @@ private static class HLXLauncher { public static Helix helix; public static API api; public static XI XI; - public static MSS mss; + public static MilestonePublisher milestonePublisher; public static Spammer spammer; /** @@ -122,7 +122,7 @@ public static void main(String [] args) throws Exception { helix.snapshotProvider, helix.ledgerService, helix.node, helix.tipsSelector, helix.tipsViewModel, helix.transactionValidator, helix.latestMilestoneTracker, helix.graph); - mss = new MSS(config, api); + milestonePublisher = new MilestonePublisher(config, api); spammer = new Spammer(config, api); shutdownHook(); @@ -137,7 +137,7 @@ public static void main(String [] args) throws Exception { throw e; } if(config.getMsDelay() > 0) { - mss.startScheduledExecutorService(); + milestonePublisher.startScheduledExecutorService(); } if(config.getSpamDelay() > 0) { spammer.startScheduledExecutorService(); @@ -152,7 +152,7 @@ private static void shutdownHook() { Runtime.getRuntime().addShutdownHook(new Thread(() -> { log.info("Shutting down Helix node, please hold tight..."); try { - mss.shutdown(); + milestonePublisher.shutdown(); XI.shutdown(); api.shutDown(); helix.shutdown(); diff --git a/src/main/java/net/helix/hlx/service/milestone/MSS.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java similarity index 71% rename from src/main/java/net/helix/hlx/service/milestone/MSS.java rename to src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 2735c203..b93e08c0 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MSS.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -1,18 +1,25 @@ package net.helix.hlx.service.milestone; import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.model.Hash; import net.helix.hlx.service.API; import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.security.SecureRandom; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.Random; -public class MSS { +public class MilestonePublisher { - private static final Logger log = LoggerFactory.getLogger(MSS.class); + private static final Logger log = LoggerFactory.getLogger(MilestonePublisher.class); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private HelixConfig config; private API api; @@ -22,8 +29,9 @@ public class MSS { private int delay; private int mwm; private Boolean sign; + private int pubkeyDepth; - public MSS(HelixConfig configuration, API api) { + public MilestonePublisher(HelixConfig configuration, API api) { this.config = configuration; this.api = api; this.delay = this.config.getMsDelay(); @@ -32,12 +40,24 @@ public MSS(HelixConfig configuration, API api) { this.message = StringUtils.repeat('0', 1024); this.address = "6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"; this.sign = !this.config.isDontValidateTestnetMilestoneSig(); + this.pubkeyDepth = config.getNumberOfKeysInMilestone(); if(this.delay < minDelay) { this.delay = minDelay; } } + private void updateKeyfile(int keyfileIndex) throws IOException { + Random rnd = new SecureRandom(); + byte[] seed = new byte[32]; + rnd.nextBytes(seed); + int numberOfKeys = (int) Math.pow(2,pubkeyDepth); + List> merkleTree = Merkle.buildMerkleKeyTree(Hex.toHexString(seed), pubkeyDepth, numberOfKeys * keyfileIndex, numberOfKeys); + Merkle.createKeyfile(merkleTree, seed, pubkeyDepth, "Nominee.key"); + this.address = merkleTree.get(merkleTree.size()-1).get(0).toString(); + // todo send new application + } + private int getRound(long time) { return (int) (time - config.getGenesisTime()) / config.getRoundDuration(); } From 7c5a628629f21189ab6c5055cf52e9d1f6fc225c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 26 Jul 2019 20:10:39 +0200 Subject: [PATCH 114/223] get seed from keyfile --- .../service/milestone/MilestonePublisher.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index b93e08c0..c51aaeac 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -9,6 +9,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.security.SecureRandom; import java.util.List; @@ -47,13 +50,20 @@ public MilestonePublisher(HelixConfig configuration, API api) { } } + public static String getSeed() throws IOException { + StringBuilder seedBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(new File("./src/main/resources/Nominee.key")))) { + String[] fields = br.readLine().split(" "); + seedBuilder.append(fields[1]); + } + return seedBuilder.toString(); + } + private void updateKeyfile(int keyfileIndex) throws IOException { - Random rnd = new SecureRandom(); - byte[] seed = new byte[32]; - rnd.nextBytes(seed); + String seed = getSeed(); int numberOfKeys = (int) Math.pow(2,pubkeyDepth); - List> merkleTree = Merkle.buildMerkleKeyTree(Hex.toHexString(seed), pubkeyDepth, numberOfKeys * keyfileIndex, numberOfKeys); - Merkle.createKeyfile(merkleTree, seed, pubkeyDepth, "Nominee.key"); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, numberOfKeys * keyfileIndex, numberOfKeys); + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, "Nominee.key"); this.address = merkleTree.get(merkleTree.size()-1).get(0).toString(); // todo send new application } From 8107b02757da8f9dbfb7c02463f82f1023701c46 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 29 Jul 2019 13:24:22 +0200 Subject: [PATCH 115/223] publish candidate tx (application) with new address --- .../net/helix/hlx/conf/BaseHelixConfig.java | 2 +- src/main/java/net/helix/hlx/service/API.java | 87 ++++++++++++++++++- .../service/milestone/MilestonePublisher.java | 29 +++++-- 3 files changed, 109 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 1439c13f..5cdf0dfb 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -943,7 +943,7 @@ public interface Defaults { String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; int MS_DELAY = 0; int MS_MIN_DELAY = 5; - long GENESIS_TIME = 1563968260276L; + long GENESIS_TIME = 1564398684743L; int ROUND_DURATION = 5000; //Snapshot diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index f6d2aec3..123e63b3 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1486,7 +1486,7 @@ private void attachStoreAndBroadcast(final String address, final String message, broadcastTransactionsStatement(powResult); } - public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int incrementIndex) throws Exception { + public boolean storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int incrementIndex) throws Exception { // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) List confirmedTips = new LinkedList<>(); @@ -1521,6 +1521,12 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); long nextIndex = currentRoundIndex + incrementIndex; + int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); + if (nextIndex > maxKeyIndex) { + log.info("Keyfile has expired! A new Keyfile will be generated, which pauses the MilestonePublisher until the process is finished."); + return false; + } + // get number of transactions needed for tips long n = (long) (confirmedTips.size()/16) + 1; @@ -1572,7 +1578,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri // Get merkle path and store in signatureMessageFragment of Sibling Transaction StringBuilder seedBuilder = new StringBuilder(); byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); - String seed = seedBuilder.toString(), coordinatorAddress = Hex.toHexString(merkleTree[merkleTree.length - 1][0]); + String seed = seedBuilder.toString(); // create merkle path from keyfile byte[] merklePath = Merkle.getMerklePath(merkleTree, (int) nextIndex); System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); @@ -1630,6 +1636,83 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); + return true; + } + + + public void sendApplication(final String address, final int minWeightMagnitude, boolean sign) throws Exception { + + // contain a signature that signs the siblings and thereby ensures the integrity. + byte[] txApplication = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Hex.decode(address), 0, txApplication, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(Serializer.serialize(1L), 0, txApplication, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txApplication, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + + // siblings for merkle tree. + byte[] txSibling = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + + // curator recipient. + byte[] txCurator = new byte[TransactionViewModel.SIZE]; + System.arraycopy(configuration.getTrusteeAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(Serializer.serialize(1L), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txCurator, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + + // calculate bundle hash + Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); + + byte[] applicationEssence = Arrays.copyOfRange(txApplication, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(applicationEssence, 0, applicationEssence.length); + byte[] siblingEssence = Arrays.copyOfRange(txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(siblingEssence, 0, siblingEssence.length); + byte[] curatorEssence = Arrays.copyOfRange(txCurator, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(curatorEssence, 0, curatorEssence.length); + + byte[] bundleHash = new byte[32]; + sponge.squeeze(bundleHash, 0, bundleHash.length); + System.arraycopy(bundleHash, 0, txApplication, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + System.arraycopy(bundleHash, 0, txCurator, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + + if (sign) { + // Get merkle path and store in signatureMessageFragment of Sibling Transaction + StringBuilder seedBuilder = new StringBuilder(); + byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); + String seed = seedBuilder.toString(); + // create merkle path from keyfile + byte[] merklePath = Merkle.getMerklePath(merkleTree, 0); + System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); + + + // sign bundle hash and store signature in Milestone Transaction + byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); + byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), 0); + final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); + byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); + byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); + byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); + System.arraycopy(signature, 0, txApplication, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + } + + // get branch and trunk + List txToApprove = new ArrayList<>(); + if(RoundViewModel.latest(tangle) == null) { + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); + } else { + txToApprove = getTransactionToApproveTips(3, Optional.empty()); + } + + // attach, broadcast and store + List transactions = new ArrayList<>(); + transactions.add(Hex.toHexString(txCurator)); + transactions.add(Hex.toHexString(txSibling)); + transactions.add(Hex.toHexString(txApplication)); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); + storeTransactionsStatement(powResult); + broadcastTransactionsStatement(powResult); } // diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index c51aaeac..2f5f1b8d 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -33,6 +33,9 @@ public class MilestonePublisher { private int mwm; private Boolean sign; private int pubkeyDepth; + private int keyfileIndex; + + public boolean active; public MilestonePublisher(HelixConfig configuration, API api) { this.config = configuration; @@ -44,6 +47,9 @@ public MilestonePublisher(HelixConfig configuration, API api) { this.address = "6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"; this.sign = !this.config.isDontValidateTestnetMilestoneSig(); this.pubkeyDepth = config.getNumberOfKeysInMilestone(); + this.keyfileIndex = 1; + + this.active = true; if(this.delay < minDelay) { this.delay = minDelay; @@ -59,13 +65,18 @@ public static String getSeed() throws IOException { return seedBuilder.toString(); } - private void updateKeyfile(int keyfileIndex) throws IOException { + private void updateKeyfile() throws Exception { String seed = getSeed(); int numberOfKeys = (int) Math.pow(2,pubkeyDepth); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, numberOfKeys * keyfileIndex, numberOfKeys); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, numberOfKeys * this.keyfileIndex, numberOfKeys); Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, "Nominee.key"); this.address = merkleTree.get(merkleTree.size()-1).get(0).toString(); - // todo send new application + sendApplication(); + //this.active = true; + } + + private void sendApplication() throws Exception { + this.api.sendApplication(this.address, this.mwm, this.sign); } private int getRound(long time) { @@ -86,9 +97,15 @@ public void startScheduledExecutorService() { } private void publishMilestone() throws Exception { - log.info("Publishing next Milestone..."); - this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, 0); - //this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, 2); + if (this.active) { + log.info("Publishing next Milestone..."); + boolean success = this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, 0); + if (!success) { + this.active = false; + this.keyfileIndex += 1; + updateKeyfile(); + } + } } private Runnable getRunnablePublishMilestone() { From e2ac2d61ccc9c79ff20a17a628e04d1a6257ea3a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 29 Jul 2019 15:11:53 +0200 Subject: [PATCH 116/223] integrate curator --- .../service/curator/CandidateSolidifier.java | 31 ++ .../hlx/service/curator/CandidateTracker.java | 49 +++ .../service/curator/CandidateValidity.java | 10 + .../hlx/service/curator/CuratorException.java | 38 ++ .../hlx/service/curator/CuratorService.java | 73 ++++ .../curator/impl/CandidateSolidifierImpl.java | 369 ++++++++++++++++++ .../curator/impl/CandidateTrackerImpl.java | 339 ++++++++++++++++ .../curator/impl/CuratorServiceImpl.java | 184 +++++++++ 8 files changed, 1093 insertions(+) create mode 100755 src/main/java/net/helix/hlx/service/curator/CandidateSolidifier.java create mode 100755 src/main/java/net/helix/hlx/service/curator/CandidateTracker.java create mode 100755 src/main/java/net/helix/hlx/service/curator/CandidateValidity.java create mode 100755 src/main/java/net/helix/hlx/service/curator/CuratorException.java create mode 100755 src/main/java/net/helix/hlx/service/curator/CuratorService.java create mode 100755 src/main/java/net/helix/hlx/service/curator/impl/CandidateSolidifierImpl.java create mode 100755 src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java create mode 100755 src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java diff --git a/src/main/java/net/helix/hlx/service/curator/CandidateSolidifier.java b/src/main/java/net/helix/hlx/service/curator/CandidateSolidifier.java new file mode 100755 index 00000000..a2d2dc17 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/CandidateSolidifier.java @@ -0,0 +1,31 @@ +package net.helix.hlx.service.curator; + +import net.helix.hlx.model.Hash; + +/** + * This interface defines the contract for a manager that tries to solidify unsolid candidates by incorporating a + * background worker that periodically checks the solidity of the candidates and issues transaction requests for the + * missing transactions until the candidates become solid. + */ +public interface CandidateSolidifier { + /** + * This method allows us to add new candidates to the solidifier that will consequently be solidified. + * + * @param candidateHash Hash of the candidate that shall be solidified + * @param roundIndex index corresponding to the round that the candidate submit the application + */ + void add(Hash candidateHash, int roundIndex); + + /** + * This method starts the background worker that asynchronously solidifies the candidates. + */ + void start(); + + /** + * This method shuts down the background worker that asynchronously solidifies the candidates. + */ + void shutdown(); + +} + + diff --git a/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java new file mode 100755 index 00000000..5d742024 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java @@ -0,0 +1,49 @@ +package net.helix.hlx.service.curator; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; + +public interface CandidateTracker { + + /** + * curator address.
+ */ + Hash curatorAddress = HashFactory.ADDRESS.create("c2eb2d5297f4e70f3e40e3d7aa3f5c1d7405264aeb72232d06776605d8b61211"); + + /** + * Analyzes the given transaction to determine if it is a valid candidate application.
+ *
+ * If the transaction that was analyzed represents a candidate application, we check...
+ * + * @param transaction the transaction that shall be examined + * @return {@code true} if the milestone could be processed and {@code false} if the bundle is not complete, yet + * @throws CuratorException if anything unexpected happens while trying to analyze the milestone candidate + */ + boolean processCandidate(TransactionViewModel transaction) throws CuratorException; + + /** + * Does the same as {@link #processCandidate(TransactionViewModel)} but automatically retrieves the + * transaction belonging to the passed in hash.
+ * + * @param transactionHash the hash of the transaction that shall be examined + * @return {@code true} if the milestone could be processed and {@code false} if the bundle is not complete, yet + * @throws CuratorException if anything unexpected happens while trying to analyze the milestone candidate + */ + boolean processCandidate(Hash transactionHash) throws CuratorException; + + /** + * @param candidateAddress address of the candidate to add to the {@code nominationQueue} + */ + void addToNomineeQueue(Hash candidateAddress); + + /** + * This method starts the background worker that... + */ + void start(); + + /** + * This method stops the background worker that....
+ */ + void shutdown(); +} diff --git a/src/main/java/net/helix/hlx/service/curator/CandidateValidity.java b/src/main/java/net/helix/hlx/service/curator/CandidateValidity.java new file mode 100755 index 00000000..519ac465 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/CandidateValidity.java @@ -0,0 +1,10 @@ +package net.helix.hlx.service.curator; + +/** + * Validity states of candidate transactions that are used to express their "relevance" for the curator. + */ +public enum CandidateValidity { + VALID, + INVALID, + INCOMPLETE +} diff --git a/src/main/java/net/helix/hlx/service/curator/CuratorException.java b/src/main/java/net/helix/hlx/service/curator/CuratorException.java new file mode 100755 index 00000000..5d15b2b9 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/CuratorException.java @@ -0,0 +1,38 @@ +package net.helix.hlx.service.curator; + +/** + * This class is used to wrap exceptions that are specific to the curator logic. + * + * It allows us to distinct between the different kinds of errors that can happen during the execution of the code. + */ +public class CuratorException extends Exception { + /** + * Constructor of the exception which allows us to provide a specific error message and the cause of the error. + * + * @param message reason why this error occurred + * @param cause wrapped exception that caused this error + */ + public CuratorException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of the exception which allows us to provide a specific error message without having an underlying + * cause. + * + * @param message reason why this error occurred + */ + public CuratorException(String message) { + super(message); + } + + /** + * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a + * specific reason. + * + * @param cause wrapped exception that caused this error + */ + public CuratorException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/net/helix/hlx/service/curator/CuratorService.java b/src/main/java/net/helix/hlx/service/curator/CuratorService.java new file mode 100755 index 00000000..126399ab --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/CuratorService.java @@ -0,0 +1,73 @@ +package net.helix.hlx.service.curator; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.Hash; + +import java.util.Set; + +public interface CuratorService { + + /** + *

+ * Analyzes the given transaction to determine if it is a valid candidate transaction. + *

+ *

+ * It first checks if all transactions that belong to the candidate bundle are known already + * + * + * (and only then verifies + * the signature to analyze if the given candidate transaction was really issued by a valid candidate) <- might not be needed. + *

+ * + * @param transactionViewModel transaction that shall be analyzed + * @param roundIndex round index of the transaction + * @return validity status of the transaction regarding its role as a nominee application + * @throws CuratorException if anything unexpected goes wrong while validating the candidate transaction + */ + CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, int roundIndex) throws CuratorException; + + /** + *

+ * Checks if the given transaction was confirmed by the candidate with the given index (or any of its + * predecessors). + *

+ *

+ * We determine if the transaction was confirmed by examining its {@code snapshotIndex} value. For this method to + * work we require that the previous candidates have been processed already (which is enforced by the {@link + * net.helix.hlx.service.curator.CandidateTracker} which applies the candidates in the order that they + * are issued by the coordinator). + *

+ * + * @param transaction the transaction that shall be examined + * @param roundIndex the round index that we want to check against + * @return {@code true} if the candidate has been nominated and {@code false} otherwise + */ + boolean isCandidateConfirmed(TransactionViewModel transaction, int roundIndex); + + /** + * Does the same as {@link #isCandidateConfirmed(TransactionViewModel, int)} but defaults to the latest solid + * round index for the {@code roundIndex} which means that the candidate has been nominated and is eligible to participate for as full node. + * + * @param transaction the transaction that shall be examined + * @return {@code true} if the transaction belongs to the candidate and {@code false} otherwise + */ + boolean isCandidateConfirmed(TransactionViewModel transaction); + + /** + * interval = initial_candidate_tx[timestamp] + now + * uptime = (interval – downtime ) / interval + * + * @param address address of the candidate whom's uptime since application is computed + * @return relative uptime (since application or since in {@code nominationCandidates} + */ + double getCandidateRelativeUptime(Hash address); + + /** + * @param candidateAddress address of the candidate whom's reputation is computed + * @param seenCandidateTransactions all transactions of seen candidates + * @param testRep testing reputation + * @return relative reputation of a candidate (since application or since in {@code nominationCandidates} + */ + double getCandidateNormalizedWeight(Hash candidateAddress, Set seenCandidateTransactions, Double testRep); + double getCandidateNormalizedWeight(Hash candidateAddress, Set seenCandidateTransactions); +} diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateSolidifierImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateSolidifierImpl.java new file mode 100755 index 00000000..163f8bc7 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateSolidifierImpl.java @@ -0,0 +1,369 @@ +package net.helix.hlx.service.curator.impl; + +import net.helix.hlx.TransactionValidator; +import net.helix.hlx.model.Hash; +import net.helix.hlx.service.curator.CandidateSolidifier; +import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.utils.log.interval.IntervalLogger; +import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; +import net.helix.hlx.utils.thread.SilentScheduledExecutorService; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + *

+ * This class implements the basic contract of the {@link CandidateSolidifier} interface. + *

+ *

+ * It manages a map of unsolid candidates to collect all candidates that have to be solidified. It then periodically + * issues checkSolidity calls on the earliest candidates to solidify them. + *

+ *

+ * To save resources and make the call a little bit more efficient, we cache the earliest candidates in a separate map, + * so the relatively expensive task of having to search for the next earliest candidates in the pool only has to be + * performed after a candidates has become solid or irrelevant for our node. + *

+ */ +public class CandidateSolidifierImpl implements CandidateSolidifier { + /** + * Defines the amount of candidates that we "simultaneously" try to solidify in one pass. + */ + private static final int SOLIDIFICATION_QUEUE_SIZE = 2; + + /** + * Defines the interval in which solidity checks are issued (in milliseconds). + */ + private static final int SOLIDIFICATION_INTERVAL = 5000; + + /** + *

+ * Defines the maximum amount of transactions that are allowed to get processed while trying to solidify a + * candidate. + *

+ *

+ * Note: We want to find the next previous candidate and not get stuck somewhere at the end of the tangle with a + * long running {@link TransactionValidator#checkSolidity(Hash, boolean)} call. + *

+ */ + private static final int SOLIDIFICATION_TRANSACTIONS_LIMIT = 50000; + + /** + * Logger for this class allowing us to dump debug and status messages. + */ + private static final IntervalLogger log = new IntervalLogger(CandidateSolidifier.class); + + /** + * Holds the snapshot provider which gives us access to the relevant snapshots. + */ + private SnapshotProvider snapshotProvider; + + /** + * Holds a reference to the TransactionValidator which allows us to issue solidity checks. + */ + private TransactionValidator transactionValidator; + + /** + * Holds a reference to the manager of the background worker. + */ + private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Candidate Solidifier", log.delegate()); + + /** + *

+ * Holds the candidates that were newly added, but not examined yet. + *

+ *

+ * Note: This is used to be able to add candidates to the solidifier without having to synchronize the access to the + * underlying Maps. + *

+ */ + private final Map newlyAddedCandidates = new ConcurrentHashMap<>(); + + /** + * Holds all unsolid candidates that shall be solidified (the transaction hash mapped to its candidate index). + */ + private final Map unsolidCandidatesPool = new ConcurrentHashMap<>(); + + /** + * Holds the candidates that are actively trying to be solidified by the background {@link Thread} (acts as a + * Queue). + */ + private final Map candidatesToSolidify = new HashMap<>(); + + /** + *

+ * Holds and entry that represents the youngest candidate in the {@link #candidatesToSolidify} Map. + *

+ *

+ * Note: It is used to check if new candidates that are being added, are older that the currently processed ones and + * should replace them in the queue (we solidify from oldest to youngest). + *

+ */ + private Map.Entry youngestCandidateInQueue = null; + + /** + *

+ * This method initializes the instance and registers its dependencies. + *

+ *

+ * It stores the passed in values in their corresponding private properties. + *

+ *

+ * Note: Instead of handing over the dependencies in the constructor, we register them lazy. This allows us to have + * circular dependencies because the instantiation is separated from the dependency injection. To reduce the + * amount of code that is necessary to correctly instantiate this class, we return the instance itself which + * allows us to still instantiate, initialize and assign in one line - see Example: + *

+ * {@code candidateSolidifier = new CandidateSolidifierImpl().init(...);} + * + * @param snapshotProvider snapshot provider which gives us access to the relevant snapshots + * @param transactionValidator TransactionValidator instance that is used by the node + * @return the initialized instance itself to allow chaining + */ + public CandidateSolidifierImpl init(SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { + this.snapshotProvider = snapshotProvider; + this.transactionValidator = transactionValidator; + + return this; + } + + /** + * {@inheritDoc} + * + *

+ * Since this method might be called from a performance critical context, we simply add the candidate to a temporary + * pool, that gets examined later by the background process. This doesn't just speed up the addition of new jobs but + * also prevents us from having to synchronize the access to the underlying maps. + *

+ */ + @Override + public void add(Hash candidateHash, int roundIndex) { + if (!unsolidCandidatesPool.containsKey(candidateHash) && !newlyAddedCandidates.containsKey(candidateHash) && + roundIndex > snapshotProvider.getInitialSnapshot().getIndex()) { + + newlyAddedCandidates.put(candidateHash, roundIndex); + } + } + + @Override + public void start() { + executorService.silentScheduleWithFixedDelay(this::candidateSolidificationThread, 0, SOLIDIFICATION_INTERVAL, + TimeUnit.MILLISECONDS); + } + + @Override + public void shutdown() { + executorService.shutdownNow(); + } + + /** + *

+ * This method takes an entry from the {@link #unsolidCandidatesPool} and adds it to the + * {@link #candidatesToSolidify} queue. + *

+ *

+ * It first checks if the given candidate is already part of the queue and then tries to add it. If the queue is not + * full yet, the addition to the queue is relatively cheap, because the {@link #youngestCandidateInQueue} marker can + * be updated without iterating over all entries. If the queue reached its capacity already, we replace the entry + * marked by the {@link #youngestCandidateInQueue} marker and update the marker by recalculating it using + * {@link #determineYoungestCandidateInQueue()}. + *

+ * + * @param candidateEntry entry from the {@link #unsolidCandidatesPool} that shall get added to the queue + */ + private void addToSolidificationQueue(Map.Entry candidateEntry) { + if (candidatesToSolidify.containsKey(candidateEntry.getKey())) { + return; + } + + if (candidatesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE) { + candidatesToSolidify.put(candidateEntry.getKey(), candidateEntry.getValue()); + + if (youngestCandidateInQueue == null || candidateEntry.getValue() > youngestCandidateInQueue.getValue()) { + youngestCandidateInQueue = candidateEntry; + } + } else if (candidateEntry.getValue() < youngestCandidateInQueue.getValue()) { + candidatesToSolidify.remove(youngestCandidateInQueue.getKey()); + candidatesToSolidify.put(candidateEntry.getKey(), candidateEntry.getValue()); + + determineYoungestCandidateInQueue(); + } + } + + /** + *

+ * This method contains the logic for the candidate solidification, that gets executed in a separate + * {@link Thread}. + *

+ *

+ * It executes the necessary steps periodically while waiting a short time to give the nodes the ability to + * answer to the issued transaction requests. + *

+ */ + private void candidateSolidificationThread() { + processNewlyAddedCandidates(); + processSolidificationQueue(); + refillSolidificationQueue(); + } + + /** + *

+ * This method processes the newly added candidates. + *

+ *

+ * We process them lazy to decrease the synchronization requirements and speed up the addition of candidates from + * outside {@link Thread}s. + *

+ *

+ * It iterates over the candidates and adds them to the pool. If they are older than the + * {@link #youngestCandidateInQueue}, we add the to the solidification queue. + *

+ */ + private void processNewlyAddedCandidates() { + for (Iterator> iterator = newlyAddedCandidates.entrySet().iterator(); + !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + + Map.Entry currentEntry = iterator.next(); + + unsolidCandidatesPool.put(currentEntry.getKey(), currentEntry.getValue()); + + if (youngestCandidateInQueue == null || currentEntry.getValue() < youngestCandidateInQueue.getValue()) { + addToSolidificationQueue(currentEntry); + } + + iterator.remove(); + } + } + + /** + *

+ * This method contains the logic for processing the {@link #candidatesToSolidify}. + *

+ *

+ * It iterates through the queue and checks if the corresponding candidates are still relevant for our node, or if + * they could be successfully solidified. If the candidates become solid or irrelevant, we remove them from the + * pool and the queue and reset the {@link #youngestCandidateInQueue} marker (if necessary). + *

+ */ + private void processSolidificationQueue() { + for (Iterator> iterator = candidatesToSolidify.entrySet().iterator(); + !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + + Map.Entry currentEntry = iterator.next(); + + if (currentEntry.getValue() <= snapshotProvider.getInitialSnapshot().getIndex() || isSolid(currentEntry)) { + unsolidCandidatesPool.remove(currentEntry.getKey()); + iterator.remove(); + + if (youngestCandidateInQueue != null && + currentEntry.getKey().equals(youngestCandidateInQueue.getKey())) { + + youngestCandidateInQueue = null; + } + } + } + } + + /** + *

+ * This method takes care of adding new candidates from the pool to the solidification queue, and filling it up + * again after it was processed / emptied before. + *

+ *

+ * It first updates the {@link #youngestCandidateInQueue} marker and then just adds new candidates as long as there + * is still space in the {@link #candidatesToSolidify} queue. + *

+ */ + private void refillSolidificationQueue() { + if(youngestCandidateInQueue == null && !candidatesToSolidify.isEmpty()) { + determineYoungestCandidateInQueue(); + } + + Map.Entry nextSolidificationCandidate; + while (!Thread.currentThread().isInterrupted() && candidatesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE && + (nextSolidificationCandidate = getNextSolidificationCandidate()) != null) { + + addToSolidificationQueue(nextSolidificationCandidate); + } + } + + /** + *

+ * This method determines the youngest candidate in the solidification queue. + *

+ *

+ * It iterates over all candidates in the Queue and keeps track of the youngest one found (the one with the highest + * candidate index). + *

+ */ + private void determineYoungestCandidateInQueue() { + youngestCandidateInQueue = null; + for (Map.Entry currentEntry : candidatesToSolidify.entrySet()) { + if (youngestCandidateInQueue == null || currentEntry.getValue() > youngestCandidateInQueue.getValue()) { + youngestCandidateInQueue = currentEntry; + } + } + } + + /** + *

+ * This method returns the earliest seen Candidate from the unsolid candidates pool, that is not part of the + * {@link #candidatesToSolidify} queue yet. + *

+ *

+ * It simply iterates over all candidates in the pool and looks for the one with the lowest index, that is not + * getting actively solidified, yet. + *

+ * + * @return the Map.Entry holding the earliest candidate or null if the pool does not contain any new candidates. + */ + private Map.Entry getNextSolidificationCandidate() { + Map.Entry nextSolidificationCandidate = null; + for (Map.Entry candidateEntry : unsolidCandidatesPool.entrySet()) { + if (!candidatesToSolidify.containsKey(candidateEntry.getKey()) && (nextSolidificationCandidate == null || + candidateEntry.getValue() < nextSolidificationCandidate.getValue())) { + + nextSolidificationCandidate = candidateEntry; + } + } + + return nextSolidificationCandidate; + } + + /** + *

+ * This method performs the actual solidity check on the selected candidate. + *

+ *

+ * It first dumps a log message to keep the node operator informed about the progress of solidification, and then + * issues the {@link TransactionValidator#checkSolidity(Hash, boolean, int)} call that starts the solidification + * process. + *

+ *

+ * We limit the amount of transactions that may be processed during the solidity check, since we want to solidify + * from the oldest candidate to the newest one and not "block" the solidification with a very recent candidate that + * needs to traverse huge chunks of the tangle. The main goal of this is to give the solidification just enough + * "resources" to discover the previous candidate while at the same time allowing fast solidity checks. + *

+ * + * @param currentEntry candidate entry that shall be checked + * @return true if the given candidate is solid or false otherwise + */ + private boolean isSolid(Map.Entry currentEntry) { + if (unsolidCandidatesPool.size() > 1) { + log.info("Solidifying candidate #" + currentEntry.getValue() + + " [" + candidatesToSolidify.size() + " / " + unsolidCandidatesPool.size() + "]"); + } + + try { + return transactionValidator.checkSolidity(currentEntry.getKey(), true, + SOLIDIFICATION_TRANSACTIONS_LIMIT); + } catch (Exception e) { + log.error("Error while solidifying candidate #" + currentEntry.getValue(), e); + + return false; + } + } +} diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java new file mode 100755 index 00000000..85ae90b9 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -0,0 +1,339 @@ +package net.helix.hlx.service.curator.impl; + +import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.controllers.AddressViewModel; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.Hash; +import net.helix.hlx.service.curator.CandidateSolidifier; +import net.helix.hlx.service.curator.CuratorService; +import net.helix.hlx.service.milestone.MilestoneSolidifier; +import net.helix.hlx.service.curator.CandidateTracker; +import net.helix.hlx.service.curator.CuratorException; +import net.helix.hlx.storage.Tangle; +import net.helix.hlx.utils.log.interval.IntervalLogger; +import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; +import net.helix.hlx.utils.thread.SilentScheduledExecutorService; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * This class implements the CandidateTracker interface and is responsible for searching for applicants, + * processing and validating their requests and adding / removing candidates to the set of nominees. + * + * The basic flow is: + * A search for candidate_bundle(s).
+ * B approve validity and solidity of the application_bundle.
+ * C validate candidate's registration data.
+ * D (update candidate's reputation).
+ * E sample candidate's from queue pseudo-randomly with reputation as a soft bias.
+ */ + +public class CandidateTrackerImpl implements CandidateTracker { + + /** + * Holds the amount of milestone candidates that will be analyzed per iteration of the background worker.
+ */ + private static final int MAX_CANDIDATES_TO_ANALYZE = 5; + + /** + * Holds the time (in milliseconds) between iterations of the background worker.
+ */ + private static final int RESCAN_INTERVAL = 1000; + + /** + * Holds the logger of this class (a rate limited logger that doesn't spam the CLI output).
+ */ + private static final IntervalLogger log = new IntervalLogger(CandidateTrackerImpl.class); + + /** + * Holds a reference to the manager of the background worker.
+ */ + private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Candidate Tracker", log.delegate()); + + /** + * Holds the Tangle object which acts as a database interface.
+ */ + private Tangle tangle; + /** + * Holds the HelixConfig object which acts as a config.
+ */ + private HelixConfig config; + + /** + * Service class containing the business logic of the curator package. + */ + private CuratorService curatorService; + + /** + * Solidifies candidate transactions + */ + private CandidateSolidifier candidateSolidifier; + + /** + * A set that allows us to keep track of the candidates that have been seen and added to the {@link + * #candidatesToAnalyze} already.
+ */ + private final Set seenCandidates = new HashSet<>(); + + /** + * Maps candidateAddress hash to weight
+ */ + private Map candidatesToNominate = new HashMap<>(); + + /** + * A list of candidates that still have to be analyzed.
+ */ + private final Deque candidatesToAnalyze = new ArrayDeque<>(); + + /** + * Initial nomination probability of a candidate
+ */ + private Double initialNomProb = 0.0; + + /** + * A flag that allows us to detect if the background worker is in its first iteration (for different log + * handling).
+ */ + private boolean firstRun = true; + + /** + * Flag which indicates if this tracker has finished its initial scan of all old milestone candidates.
+ */ + private boolean initialized = false; + + /** + * Hash of the candidate transaction
+ */ + private Hash candidateAddress; + + /** + * Initialize currentRoundIndex
+ */ + private int currentRoundIndex = 0; + + /** + * Initialize the starting round that the candidate is assigned to.
+ */ + private int startRound = 0; + + /** + * This method initializes the instance and registers its dependencies.
+ *
+ * Note: Instead of handing over the dependencies in the constructor, we register them lazy. This allows us to have + * circular dependencies because the instantiation is separated from the dependency injection. To reduce the + * amount of code that is necessary to correctly instantiate this class, we return the instance itself which + * allows us to still instantiate, initialize and assign in one line - see Example:
+ *
+ * {@code candidateTracker = new candidateTrackerImpl().init(...);} + * + * @param tangle Tangle object which acts as a database interface + * @param config configuration object which allows us to determine the important config parameters of the node + * @return the initialized instance itself to allow chaining + */ + public CandidateTrackerImpl init(Tangle tangle, CandidateSolidifier candidateSolidifier, HelixConfig config, CuratorService curatorService) { + + this.tangle = tangle; + this.config = config; + this.curatorService = curatorService; + this.candidateSolidifier = candidateSolidifier; + + return this; + } + + /** + * This method contains the logic for scanning for new candidates that gets executed in a background + * worker.
+ *
+ * It first collects all new candidates that need to be analyzed, then analyzes them and finally checks if + * the initialization is complete. In addition to this scanning logic it also issues regular log messages about the + * progress of the scanning.
+ * + * A search for candidate_bundle(s) {@link #collectNewCandidates} + * B approve validity and solidity of the candidate_bundle.{@link #processCandidate}
+ * + */ + private void candidateTrackerThread() { + try { + logProgress(); + collectNewCandidates(); // A + + // additional log message on the first run to indicate how many application candidates we have in total + if (firstRun) { + firstRun = false; + logProgress(); + } + analyzeCandidates(); // B + checkIfInitializationComplete(); + } catch (CuratorException e) { + log.error("error while analyzing the applying candidates", e); + } + } + + /** + * This method collects the new candidates that have not been "seen" before, by collecting them in the {@link + * #candidatesToAnalyze} queue.
+ *
+ * We simply request all transaction that are originating from the coordinator address and treat them as potential + * milestone candidates.
+ * + * @throws CuratorException if anything unexpected happens while collecting the new milestone candidates + */ + //@VisibleForTesting + private void collectNewCandidates() throws CuratorException { + try { + for (Hash hash : AddressViewModel.load(tangle, this.curatorAddress).getHashes()) { + if (Thread.currentThread().isInterrupted()) { + return; + } + if (seenCandidates.add(hash)) { + candidatesToAnalyze.addFirst(hash); + } + } + } catch (Exception e) { + throw new CuratorException("failed to collect the new candidates", e); + } + } + + /** + * This method analyzes the candidates by working through the {@link #candidatesToAnalyze} + * queue.
+ *
+ * We only process {@link #MAX_CANDIDATES_TO_ANALYZE} at a time, to give the caller the option to terminate early + * and pick up new milestones as fast as possible without being stuck with analyzing the old ones for too + * long.
+ * + * @throws CuratorException if anything unexpected happens while analyzing the milestone candidates + */ + private void analyzeCandidates() throws CuratorException { + int toAnalyze = Math.min(candidatesToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); + for (int i = 0; i < toAnalyze; i++) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Hash candidateTransactionHash = candidatesToAnalyze.pollFirst(); + if(!processCandidate(candidateTransactionHash)) { + seenCandidates.remove(candidateTransactionHash); + } + } + } + + @Override + public boolean processCandidate(Hash transactionHash) throws CuratorException { + try { + return processCandidate(TransactionViewModel.fromHash(tangle, transactionHash)); + } catch (Exception e) { + throw new CuratorException("unexpected error while analyzing the transaction " + transactionHash, e); + } + } + + /** + * {@inheritDoc} + *
+ * If we detect a candidate that is either {@code INCOMPLETE} or not solid, yet we hand it over to the + * {@link MilestoneSolidifier} that takes care of requesting the missing parts of the milestone bundle.
+ */ + @Override + public boolean processCandidate(TransactionViewModel transaction) throws CuratorException { + try { + if (curatorAddress.equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { + switch (curatorService.validateCandidate(transaction, currentRoundIndex)) { + case VALID: + if (startRound > currentRoundIndex) { + addToNomineeQueue(transaction.getAddressHash()); + } + if (!transaction.isSolid()) { + candidateSolidifier.add(transaction.getHash(), currentRoundIndex); + } + // not yet implemented isCandidate + //transaction.isCandidate(tangle, snapshotProvider.getInitialSnapshot(), true); + break; + + case INCOMPLETE: + return false; + + case INVALID: + // do not re-analyze anymore + return true; + + default: + // we can consider the candidate processed and move on w/o farther action + } + } + + return true; + } catch (Exception e) { + throw new CuratorException("unexpected error while analyzing the transaction: " + transaction.getHash(), e); + } + } + + /** + * {@inheritDoc} + * Adds a candidate to the {@code candidatesToNominate} + * A corresponding initial probability based on reputation (work and time) is added, but isn't meaningful until the reputation update. + *

+ * In addition to setting the internal properties, we also issue a log message and publish the change to the ZeroMQ + * message processor so external receivers get informed about this change. + *

+ * + * @param candidateAddress + * + */ + public void addToNomineeQueue(Hash candidateAddress) { + Double w = (curatorService.getCandidateNormalizedWeight(candidateAddress, this.seenCandidates)); + this.candidatesToNominate.put(candidateAddress, w); + + tangle.publish("nac %d %d", candidateAddress, startRound); //nac = newly added candidate + log.delegate().info("New candidate {} added for", candidateAddress); + } + + /** + * This method emits a log message about the scanning progress.
+ *
+ * It only emits a log message if we have more than one {@link #candidatesToAnalyze}, which means that the + * very first call to this method in the "first run" on {@link #candidateTrackerThread()} will not produce any + * output (which is the reason why we call this method a second time after we have collected all the + * candidates in the "first run").
+ */ + private void logProgress() { + if (candidatesToAnalyze.size() > 1) { + log.info("Processing full node candidates (" + candidatesToAnalyze.size() + " remaining) ..."); + } + } + + /** + * This method checks if the initialization is complete.
+ *
+ * It simply checks if the {@link #initialized} flag is not set yet and there are no more {@link + * #candidatesToAnalyze}. If the initialization was complete, we issue a log message and set the + * corresponding flag to {@code true}.
+ */ + private void checkIfInitializationComplete() { + if (!initialized && candidatesToAnalyze.size() == 0) { + initialized = true; + + log.info("Processing candidates ... [DONE]").triggerOutput(true); + } + } + + /** + * {@inheritDoc} + *
+ * We repeatedly call {@link #candidateTrackerThread()} to search for new application bundles in the database. + * This is a bit inefficient and should at some point maybe be replaced with a check on transaction arrival, but + * this would required adjustments in the whole way IRI (and Helix-1.0) handles transactions and is therefore postponed for + * now.
+ */ + @Override + public void start() { + executorService.silentScheduleWithFixedDelay(this::candidateTrackerThread, 0, RESCAN_INTERVAL, + TimeUnit.MILLISECONDS); + } + + @Override + public void shutdown() { + executorService.shutdownNow(); + } +} diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java new file mode 100755 index 00000000..28f4d673 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java @@ -0,0 +1,184 @@ +package net.helix.hlx.service.curator.impl; + +import net.helix.hlx.BundleValidator; +import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.Hash; +import net.helix.hlx.service.curator.CandidateValidity; +import net.helix.hlx.service.curator.CuratorException; +import net.helix.hlx.service.curator.CuratorService; +import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.service.snapshot.SnapshotService; +import net.helix.hlx.storage.Tangle; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.SecureRandom; +import java.util.*; + + +import static net.helix.hlx.service.curator.CandidateValidity.*; + +public class CuratorServiceImpl implements CuratorService { + /** + * Holds the logger of this class. + */ + private final static Logger log = LoggerFactory.getLogger(CuratorServiceImpl.class); + + /** + * Holds the tangle object which acts as a database interface. + */ + private Tangle tangle; + + /** + * Holds the snapshot provider which gives us access to the relevant snapshots. + */ + private SnapshotProvider snapshotProvider; + + /** + * Holds a reference to the service instance of the snapshot package that allows us to roll back ledger states. + */ + private SnapshotService snapshotService; + + /** + * Configurations for milestone + */ + private HelixConfig config; + + public CuratorServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotService snapshotService, HelixConfig config) { + + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.snapshotService = snapshotService; + this.config = config; + + return this; + } + + @Override + public CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, int roundIndex) throws CuratorException { + + // getCandidates not yet implemented + // TransactionViewModel existingCandidate = TransactionViewModel.getCandidates(tangle, roundIndex); + TransactionViewModel existingCurator = null; + + //log.debug("Max round index set to: {}", config.getMaxRoundIndex()); + if (roundIndex < 0 || roundIndex >= 0x200000) { + return INVALID; + } + + try { + if(existingCurator != null){ + return existingCurator.getHash().equals(transactionViewModel.getHash()) ? VALID : INVALID; + } + + final List> bundleTransactions = BundleValidator.validate(tangle, + snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); + + if (bundleTransactions.isEmpty()) { + return INCOMPLETE; + } else { + for (final List bundleTransactionViewModels : bundleTransactions) { + final TransactionViewModel tail = bundleTransactionViewModels.get(0); + if (tail.getHash().equals(transactionViewModel.getHash())) { + // the signed transaction - which references the confirmed transactions and contains + // the Merkle tree siblings. + + if (isCandidateBundleStructureValid(bundleTransactionViewModels, 1)) { + + boolean skipValidation = true; //config.isTestnet() && config.isDontValidateTestnetMilestoneSig(); // should be config.isDontValidateCandidates() + if (skipValidation /*|| candidateTracker.getSeenCandidatesMerkleRoot(transactionViewModel.getHash).equals(HashFactory.ADDRESS.create(merkleRoot)*/) { + // not yet implemented + // verify signature corresponds to merkle root of candidate, if so save the merkle root in + return VALID; + } else { + return INVALID; + } + } + } + } + } + } catch (Exception e) { + throw new CuratorException("error while checking candidate status of " + transactionViewModel.getHash(), e); + } + return INVALID; + } + + /** + *

+ * This method is a utility method that checks if the transactions belonging to the potential candidate bundle has + * a valid structure (used during the validation of candidates). + *

+ *

+ * It first checks if the bundle has enough transactions to conform to the given {@code securityLevel} and then + * verifies that the {@code branchTransactionsHash}es are pointing to the {@code trunkTransactionHash} of the head + * transactions. + *

+ * + * @param bundleTransactions all transactions belonging to the milestone + * @param securityLevel the security level used for the signature + * @return {@code true} if the basic structure is valid and {@code false} otherwise + */ + private boolean isCandidateBundleStructureValid(List bundleTransactions, int securityLevel) { + if (bundleTransactions.size() <= securityLevel) { + return false; + } + + Hash headTransactionHash = bundleTransactions.get(securityLevel).getTrunkTransactionHash(); + return bundleTransactions.stream() + .limit(securityLevel) + .map(TransactionViewModel::getBranchTransactionHash) + .allMatch(branchTransactionHash -> branchTransactionHash.equals(headTransactionHash)); + } + @Override + public double getCandidateRelativeUptime(Hash address) { + return 0; + } + + @Override + public double getCandidateNormalizedWeight(Hash candidateAddress, Set seenCandidateTransactions, Double testWeight) { + + TransactionViewModel txvm; + double powWeight = 0; + //double uptimeWeight = getCandidateRelativeUptime(candidateAddress); (beta * uptimeWeight) // wt: + getCandidateRelativeUptime(h) + double w = 0; // own weight + double wt = 0; // total weight + double alpha = 0.55; // weight factor for pow + double beta = 0.02; // weight factor for relative uptime + + if(!seenCandidateTransactions.isEmpty()) { + for (Hash h : seenCandidateTransactions) { + try { + txvm = TransactionViewModel.fromHash(tangle, h); + if (txvm.getAddressHash().equals(candidateAddress)) { + powWeight += h.leadingZeros(); + } + } catch (Exception e) { + log.error("unexpected error while calculating own reputation", e); + } + } + w = testWeight != null ? Math.exp(alpha * testWeight) : Math.exp(alpha * powWeight); + wt = (seenCandidateTransactions.stream().mapToDouble(h -> Math.exp(alpha * (h.leadingZeros() ))).sum()); + } + return wt != 0 ? w/wt : 0; + } + + @Override + public double getCandidateNormalizedWeight(Hash candidateAddress, Set seenCandidateTransactions) { + return getCandidateNormalizedWeight(candidateAddress, seenCandidateTransactions, null); + } + + public void nominate(Map candidatesToNominate) { + } + + @Override + public boolean isCandidateConfirmed(TransactionViewModel transaction) { + return true; // not yet implemented + } + @Override + public boolean isCandidateConfirmed(TransactionViewModel transaction, int roundIndex) { + return true; // not yet implemented + } + +} From 5bbd49543ae2afc2ac84ad0fed698923203d326a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 29 Jul 2019 15:39:34 +0200 Subject: [PATCH 117/223] keyIndex is independant of roundIndex --- src/main/java/net/helix/hlx/crypto/Merkle.java | 8 +++++--- src/main/java/net/helix/hlx/service/API.java | 17 +++++------------ .../service/milestone/MilestonePublisher.java | 16 +++++++++++----- .../milestone/impl/MilestoneServiceImpl.java | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index 6a0815c3..cbd9b541 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -1,5 +1,6 @@ package net.helix.hlx.crypto; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; @@ -54,7 +55,7 @@ public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, List keys = new ArrayList<>(1 << pubkeyDepth); for (int i = 0; i < pubkeyCount; i++) { int idx = firstIndex + i; - keys.set(idx, HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, 1))); + keys.add(HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, 1))); } return buildMerkleTree(keys); } @@ -94,9 +95,10 @@ public static List> buildMerkleTree(List leaves){ return merkleTree; } - public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int index, int securityLevel, int depth) { + public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); + int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) //TODO: check if its okay here to use bundle hash instead of tx hash @@ -117,7 +119,7 @@ public static boolean validateMerkleSignature(List bundleT //validate Merkle path byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, - merkleTx.getSignature(), 0, index, depth); + merkleTx.getSignature(), 0, keyIndex, depth); return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); } diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 123e63b3..6a73b8f6 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1486,7 +1486,7 @@ private void attachStoreAndBroadcast(final String address, final String message, broadcastTransactionsStatement(powResult); } - public boolean storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int incrementIndex) throws Exception { + public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) List confirmedTips = new LinkedList<>(); @@ -1519,13 +1519,6 @@ public boolean storeAndBroadcastMilestoneStatement(final String address, final S // get round int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); - long nextIndex = currentRoundIndex + incrementIndex; - - int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); - if (nextIndex > maxKeyIndex) { - log.info("Keyfile has expired! A new Keyfile will be generated, which pauses the MilestonePublisher until the process is finished."); - return false; - } // get number of transactions needed for tips long n = (long) (confirmedTips.size()/16) + 1; @@ -1535,13 +1528,14 @@ public boolean storeAndBroadcastMilestoneStatement(final String address, final S System.arraycopy(Hex.decode(address), 0, txMilestone, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); System.arraycopy(Serializer.serialize(1L + n), 0, txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txMilestone, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize(nextIndex), 0, txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + System.arraycopy(Serializer.serialize((long) currentRoundIndex), 0, txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); // siblings for merkle tree. byte[] txSibling = new byte[TransactionViewModel.SIZE]; System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); System.arraycopy(Serializer.serialize(1L + n), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize((long) keyIndex), 0, txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); // list of confirming tips List txTips = new ArrayList<>(); @@ -1580,13 +1574,13 @@ public boolean storeAndBroadcastMilestoneStatement(final String address, final S byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); String seed = seedBuilder.toString(); // create merkle path from keyfile - byte[] merklePath = Merkle.getMerklePath(merkleTree, (int) nextIndex); + byte[] merklePath = Merkle.getMerklePath(merkleTree, keyIndex); System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); // sign bundle hash and store signature in Milestone Transaction byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); - byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), (int) nextIndex); + byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); @@ -1636,7 +1630,6 @@ public boolean storeAndBroadcastMilestoneStatement(final String address, final S List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); - return true; } diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 2f5f1b8d..b4f09949 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -34,6 +34,8 @@ public class MilestonePublisher { private Boolean sign; private int pubkeyDepth; private int keyfileIndex; + private int maxKeyIndex; + private int currentKeyIndex; public boolean active; @@ -48,6 +50,8 @@ public MilestonePublisher(HelixConfig configuration, API api) { this.sign = !this.config.isDontValidateTestnetMilestoneSig(); this.pubkeyDepth = config.getNumberOfKeysInMilestone(); this.keyfileIndex = 1; + this.maxKeyIndex = (int) Math.pow(2,this.pubkeyDepth); + this.currentKeyIndex = 0; this.active = true; @@ -67,12 +71,11 @@ public static String getSeed() throws IOException { private void updateKeyfile() throws Exception { String seed = getSeed(); - int numberOfKeys = (int) Math.pow(2,pubkeyDepth); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, numberOfKeys * this.keyfileIndex, numberOfKeys); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, this.maxKeyIndex * this.keyfileIndex, this.maxKeyIndex); Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, "Nominee.key"); this.address = merkleTree.get(merkleTree.size()-1).get(0).toString(); sendApplication(); - //this.active = true; + this.active = true; } private void sendApplication() throws Exception { @@ -99,8 +102,11 @@ public void startScheduledExecutorService() { private void publishMilestone() throws Exception { if (this.active) { log.info("Publishing next Milestone..."); - boolean success = this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, 0); - if (!success) { + if (currentKeyIndex < maxKeyIndex * this.keyfileIndex) { + this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, this.currentKeyIndex); + this.currentKeyIndex += 1; + } else { + log.info("Keyfile has expired! A new Keyfile will be generated, which pauses the MilestonePublisher until the process is finished."); this.active = false; this.keyfileIndex += 1; updateKeyfile(); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index ff2bee01..71081084 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -187,7 +187,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getNumberOfKeysInMilestone()); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { From 2556417ed87714eda15f98605510c62d7beaef27 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:07:46 +0200 Subject: [PATCH 118/223] process and validate candidate transactions, get nominees --- .../hlx/service/curator/CandidateTracker.java | 6 +- .../curator/impl/CandidateTrackerImpl.java | 111 +++++++++++++----- 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java index 5d742024..fe25dbd4 100755 --- a/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java +++ b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java @@ -4,12 +4,14 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; +import java.util.Set; + public interface CandidateTracker { /** * curator address.
*/ - Hash curatorAddress = HashFactory.ADDRESS.create("c2eb2d5297f4e70f3e40e3d7aa3f5c1d7405264aeb72232d06776605d8b61211"); + //Hash curatorAddress = HashFactory.ADDRESS.create("c2eb2d5297f4e70f3e40e3d7aa3f5c1d7405264aeb72232d06776605d8b61211"); /** * Analyzes the given transaction to determine if it is a valid candidate application.
@@ -37,6 +39,8 @@ public interface CandidateTracker { */ void addToNomineeQueue(Hash candidateAddress); + Set getNominees(); + /** * This method starts the background worker that... */ diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index 85ae90b9..11b2fefd 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -1,14 +1,16 @@ package net.helix.hlx.service.curator.impl; +import net.helix.hlx.BundleValidator; import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.AddressViewModel; +import net.helix.hlx.controllers.BundleViewModel; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; -import net.helix.hlx.service.curator.CandidateSolidifier; -import net.helix.hlx.service.curator.CuratorService; +import net.helix.hlx.service.curator.*; import net.helix.hlx.service.milestone.MilestoneSolidifier; -import net.helix.hlx.service.curator.CandidateTracker; -import net.helix.hlx.service.curator.CuratorException; +import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; @@ -17,6 +19,10 @@ import java.util.*; import java.util.concurrent.TimeUnit; +import static net.helix.hlx.service.curator.CandidateValidity.INCOMPLETE; +import static net.helix.hlx.service.curator.CandidateValidity.INVALID; +import static net.helix.hlx.service.curator.CandidateValidity.VALID; + /** * This class implements the CandidateTracker interface and is responsible for searching for applicants, * processing and validating their requests and adding / removing candidates to the set of nominees. @@ -66,6 +72,8 @@ public class CandidateTrackerImpl implements CandidateTracker { */ private CuratorService curatorService; + private SnapshotProvider snapshotProvider; + /** * Solidifies candidate transactions */ @@ -80,7 +88,7 @@ public class CandidateTrackerImpl implements CandidateTracker { /** * Maps candidateAddress hash to weight
*/ - private Map candidatesToNominate = new HashMap<>(); + private Set nominees = new HashSet<>(); /** * A list of candidates that still have to be analyzed.
@@ -103,20 +111,6 @@ public class CandidateTrackerImpl implements CandidateTracker { */ private boolean initialized = false; - /** - * Hash of the candidate transaction
- */ - private Hash candidateAddress; - - /** - * Initialize currentRoundIndex
- */ - private int currentRoundIndex = 0; - - /** - * Initialize the starting round that the candidate is assigned to.
- */ - private int startRound = 0; /** * This method initializes the instance and registers its dependencies.
@@ -132,12 +126,11 @@ public class CandidateTrackerImpl implements CandidateTracker { * @param config configuration object which allows us to determine the important config parameters of the node * @return the initialized instance itself to allow chaining */ - public CandidateTrackerImpl init(Tangle tangle, CandidateSolidifier candidateSolidifier, HelixConfig config, CuratorService curatorService) { + public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, HelixConfig config) { this.tangle = tangle; this.config = config; - this.curatorService = curatorService; - this.candidateSolidifier = candidateSolidifier; + this.snapshotProvider = snapshotProvider; return this; } @@ -183,7 +176,7 @@ private void candidateTrackerThread() { //@VisibleForTesting private void collectNewCandidates() throws CuratorException { try { - for (Hash hash : AddressViewModel.load(tangle, this.curatorAddress).getHashes()) { + for (Hash hash : AddressViewModel.load(tangle, this.config.getTrusteeAddress()).getHashes()) { if (Thread.currentThread().isInterrupted()) { return; } @@ -238,15 +231,28 @@ public boolean processCandidate(Hash transactionHash) throws CuratorException { @Override public boolean processCandidate(TransactionViewModel transaction) throws CuratorException { try { - if (curatorAddress.equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { - switch (curatorService.validateCandidate(transaction, currentRoundIndex)) { + if (this.config.getTrusteeAddress().equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == transaction.lastIndex()) { + System.out.println("Process Candidate"); + System.out.println("Hash: " + transaction.getHash()); + System.out.println("Address: " + transaction.getAddressHash()); + // get tail + BundleViewModel bundle = BundleViewModel.load(tangle, transaction.getBundleHash()); + TransactionViewModel tail = null; + for (Hash bundleTx : bundle.getHashes()) { + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, bundleTx); + if (tx.getCurrentIndex() == 0) { + tail = tx; + } + } + switch (validateCandidate(tail, SpongeFactory.Mode.S256, 1)) { case VALID: - if (startRound > currentRoundIndex) { - addToNomineeQueue(transaction.getAddressHash()); + if (!nominees.contains(tail.getAddressHash())) { + addToNomineeQueue(tail.getAddressHash()); } - if (!transaction.isSolid()) { + + /*if (!transaction.isSolid()) { candidateSolidifier.add(transaction.getHash(), currentRoundIndex); - } + }*/ // not yet implemented isCandidate //transaction.isCandidate(tangle, snapshotProvider.getInitialSnapshot(), true); break; @@ -269,6 +275,41 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator } } + public CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel) throws CuratorException { + + try { + + final List> bundleTransactions = BundleValidator.validate(tangle, + snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); + + if (bundleTransactions.isEmpty()) { + return INCOMPLETE; + } else { + for (final List bundleTransactionViewModels : bundleTransactions) { + final TransactionViewModel tail = bundleTransactionViewModels.get(0); + if (tail.getHash().equals(transactionViewModel.getHash())) { + + //todo implement when sure how bundle structure has to look like + //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { + + Hash senderAddress = tail.getAddressHash(); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getNumberOfKeysInMilestone()); + System.out.println("valid signature (candidate): " + validSignature); + + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { + return VALID; + } else { + return INVALID; + } + } + } + } + } catch (Exception e) { + throw new CuratorException("error while checking candidate status of " + transactionViewModel.getHash(), e); + } + return INVALID; +} + /** * {@inheritDoc} * Adds a candidate to the {@code candidatesToNominate} @@ -281,14 +322,20 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator * @param candidateAddress * */ + @Override public void addToNomineeQueue(Hash candidateAddress) { - Double w = (curatorService.getCandidateNormalizedWeight(candidateAddress, this.seenCandidates)); - this.candidatesToNominate.put(candidateAddress, w); + //Double w = (curatorService.getCandidateNormalizedWeight(candidateAddress, this.seenCandidates)); + this.nominees.add(candidateAddress); - tangle.publish("nac %d %d", candidateAddress, startRound); //nac = newly added candidate + tangle.publish("nac %d", candidateAddress); //nac = newly added candidate log.delegate().info("New candidate {} added for", candidateAddress); } + @Override + public Set getNominees(){ + return this.nominees; + } + /** * This method emits a log message about the scanning progress.
*
From 892dcf1b93ed68083075b7f45ab5dd786b72ea09 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:09:22 +0200 Subject: [PATCH 119/223] implement NomineePublisher --- .../curator/impl/NomineePublisher.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java new file mode 100644 index 00000000..96cc200d --- /dev/null +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -0,0 +1,61 @@ +package net.helix.hlx.service.curator.impl; + +import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.service.API; +import net.helix.hlx.service.milestone.MilestonePublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class NomineePublisher { + private static final Logger log = LoggerFactory.getLogger(MilestonePublisher.class); + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + private HelixConfig config; + private API api; + + private int delay; + private int mwm; + private Boolean sign; + private int currentKeyIndex; + private int startRoundDelay; + + public NomineePublisher(HelixConfig configuration, API api) { + this.config = configuration; + this.api = api; + this.delay = 30000; + this.mwm = 2; + this.sign = !this.config.isDontValidateTestnetMilestoneSig(); + this.currentKeyIndex = 0; + this.startRoundDelay = 2; + } + + public void startScheduledExecutorService() { + log.info("NomineePublisher scheduledExecutorService started."); + log.info("Update Nominees every: " + delay / 1000 + "s."); + this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnableUpdateNominees(), this.delay, this.delay, TimeUnit.MILLISECONDS); + } + + private void UpdateNominees() throws Exception { + log.info("Publishing new Nominees..."); + this.api.updateNominees(this.startRoundDelay, this.mwm, this.sign, this.currentKeyIndex); + this.currentKeyIndex += 1; + } + + private Runnable getRunnableUpdateNominees() { + return () -> { + try { + UpdateNominees(); + } catch (Exception e) { + e.printStackTrace(); + } + }; + } + + public void shutdown() { + log.info("Shutting down NomineePublisher Thread"); + scheduledExecutorService.shutdown(); + } +} From 9ec6bb4d7eb4fb05a5518e2eabf2f30f9e1d7aaa Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:12:50 +0200 Subject: [PATCH 120/223] send transaction to update nominees, take correct keyindex, add CandidateTracker to API --- src/main/java/net/helix/hlx/service/API.java | 134 +++++++++++++++++-- 1 file changed, 122 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 6a73b8f6..ea2b3934 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -17,6 +17,7 @@ import net.helix.hlx.network.Neighbor; import net.helix.hlx.network.Node; import net.helix.hlx.network.TransactionRequester; +import net.helix.hlx.service.curator.CandidateTracker; import net.helix.hlx.service.dto.*; import net.helix.hlx.service.ledger.LedgerService; import net.helix.hlx.service.milestone.LatestMilestoneTracker; @@ -105,6 +106,7 @@ public class API { private final TipsViewModel tipsViewModel; private final TransactionValidator transactionValidator; private final LatestMilestoneTracker latestMilestoneTracker; + private final CandidateTracker candidateTracker; private final Graphstream graph; private final int maxFindTxs; @@ -148,7 +150,7 @@ public API(HelixConfig configuration, XI XI, TransactionRequester transactionReq SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, SnapshotProvider snapshotProvider, LedgerService ledgerService, Node node, TipSelector tipsSelector, TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - LatestMilestoneTracker latestMilestoneTracker, Graphstream graph) { + LatestMilestoneTracker latestMilestoneTracker, CandidateTracker candidateTracker, Graphstream graph) { this.configuration = configuration; this.XI = XI; @@ -163,6 +165,7 @@ public API(HelixConfig configuration, XI XI, TransactionRequester transactionReq this.tipsViewModel = tipsViewModel; this.transactionValidator = transactionValidator; this.latestMilestoneTracker = latestMilestoneTracker; + this.candidateTracker = candidateTracker; this.graph = graph; maxFindTxs = configuration.getMaxFindTransactions(); @@ -1519,6 +1522,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri // get round int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); + int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); // get number of transactions needed for tips long n = (long) (confirmedTips.size()/16) + 1; @@ -1535,7 +1539,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); System.arraycopy(Serializer.serialize(1L + n), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) keyIndex), 0, txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + System.arraycopy(Serializer.serialize((long) keyIndex % maxKeyIndex), 0, txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); // list of confirming tips List txTips = new ArrayList<>(); @@ -1574,7 +1578,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); String seed = seedBuilder.toString(); // create merkle path from keyfile - byte[] merklePath = Merkle.getMerklePath(merkleTree, keyIndex); + byte[] merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); @@ -1608,16 +1612,15 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri if (previousRound == null){ txToApprove.add(Hash.NULL_HASH); } else { - System.out.println("Previous Round: " + previousRound.toString()); List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones //txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); - System.out.println("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); + log.info("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); } //branch List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips - System.out.println("Branch (Tips): " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); + log.info("Branch (Tips): " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); } // attach, broadcast and store @@ -1633,24 +1636,26 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri } - public void sendApplication(final String address, final int minWeightMagnitude, boolean sign) throws Exception { + public void sendApplication(final String address, final int minWeightMagnitude, boolean sign, int keyIndex) throws Exception { + System.out.println("Send Application"); // contain a signature that signs the siblings and thereby ensures the integrity. byte[] txApplication = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, txApplication, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(1L), 0, txApplication, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(2L), 0, txApplication, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txApplication, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); // siblings for merkle tree. byte[] txSibling = new byte[TransactionViewModel.SIZE]; System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(2L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); // curator recipient. byte[] txCurator = new byte[TransactionViewModel.SIZE]; System.arraycopy(configuration.getTrusteeAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(1L), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(2L), 0, txCurator, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(2L), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txCurator, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); // calculate bundle hash @@ -1675,13 +1680,13 @@ public void sendApplication(final String address, final int minWeightMagnitude, byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); String seed = seedBuilder.toString(); // create merkle path from keyfile - byte[] merklePath = Merkle.getMerklePath(merkleTree, 0); + byte[] merklePath = Merkle.getMerklePath(merkleTree, 0) ; System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); // sign bundle hash and store signature in Milestone Transaction byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); - byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), 0); + byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); @@ -1708,6 +1713,111 @@ public void sendApplication(final String address, final int minWeightMagnitude, broadcastTransactionsStatement(powResult); } + + public void updateNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { + + System.out.println("Update Nominees"); + List nominees = new ArrayList<>(candidateTracker.getNominees()); + + // get round + int startRoundIndex = latestMilestoneTracker.getCurrentRoundIndex() + startRoundDelay; + + // get number of transactions needed for tips + long n = (long) (nominees.size()/16) + 1; + + // contain a signature that signs the siblings and thereby ensures the integrity. + byte[] txCurator = new byte[TransactionViewModel.SIZE]; + System.arraycopy(configuration.getTrusteeAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txCurator, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize((long) startRoundIndex), 0, txCurator, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + + // siblings for merkle tree. + byte[] txSibling = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize((long) keyIndex), 0, txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + + // list of nominee addresses + List txNominees = new ArrayList<>(); + for (long i = 2L; i <= (1L + n); i++) { + byte[] tx = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(i), 0, tx, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, tx, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, tx, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + txNominees.add(tx); + } + + + // calculate bundle hash + Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); + + byte[] curatorEssence = Arrays.copyOfRange(txCurator, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(curatorEssence, 0, curatorEssence.length); + byte[] siblingEssence = Arrays.copyOfRange(txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(siblingEssence, 0, siblingEssence.length); + for (byte[] tx : txNominees) { + byte[] nomineeEssence = Arrays.copyOfRange(tx, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(nomineeEssence, 0, nomineeEssence.length); + } + + byte[] bundleHash = new byte[32]; + sponge.squeeze(bundleHash, 0, bundleHash.length); + System.arraycopy(bundleHash, 0, txCurator, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + for (byte[] tx : txNominees) { + System.arraycopy(bundleHash, 0, tx, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + } + + if (sign) { + // Get merkle path and store in signatureMessageFragment of Sibling Transaction + StringBuilder seedBuilder = new StringBuilder(); + byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Coordinator.key"), seedBuilder); + String seed = seedBuilder.toString(); + // create merkle path from keyfile + //int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); + byte[] merklePath = Merkle.getMerklePath(merkleTree, keyIndex); + System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); + + + // sign bundle hash and store signature in Milestone Transaction + byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); + byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); + final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); + byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); + byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); + byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); + System.arraycopy(signature, 0, txCurator, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + } + + // write confirmed tips into signature message fragment of txTips + for (int i=0; i txToApprove = new ArrayList<>(); + if(RoundViewModel.latest(tangle) == null) { + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); + } else { + txToApprove = getTransactionToApproveTips(3, Optional.empty()); + } + + // attach, broadcast and store + List transactions = new ArrayList<>(); + for (int i = txNominees.size()-1; i >= 0; i--) { + transactions.add(Hex.toHexString(txNominees.get(i))); + } + transactions.add(Hex.toHexString(txSibling)); + transactions.add(Hex.toHexString(txCurator)); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); + storeTransactionsStatement(powResult); + broadcastTransactionsStatement(powResult); + } + // // FUNCTIONAL COMMAND ROUTES // From ab65914f6195b316eba17084ad4e81722b442800 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:17:47 +0200 Subject: [PATCH 121/223] validate nominee tx with merkle path depth 15, fix getNomineeAddresses --- .../milestone/impl/NomineeTrackerImpl.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java index 05f65b7d..12338a8b 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java @@ -111,7 +111,8 @@ public MilestoneValidity validateNominees(TransactionViewModel transactionViewMo // validate signature Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, roundIndex, securityLevel, config.getNumberOfKeysInMilestone()); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, 15); + System.out.println("valid signature (nominee): " + validSignature); if (Curator_Address.equals(senderAddress) && validSignature) { return VALID; @@ -136,6 +137,9 @@ public boolean processNominees(Hash transactionHash) throws Exception { int roundIndex = RoundViewModel.getRoundIndex(transaction); + System.out.println("Process Nominee"); + System.out.println("Hash: " + transaction.getHash() + ", start round: " + roundIndex); + // if the trustee transaction is older than our ledger start point: we already processed it in the past if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { return true; @@ -144,9 +148,13 @@ public boolean processNominees(Hash transactionHash) throws Exception { // validate switch (validateNominees(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { case VALID: + System.out.println("round index: " + roundIndex); + System.out.println("start round: " + startRound); if (roundIndex > startRound) { - latestNominees = getNomineeAddresses(transaction.getHash()); + latestNominees.addAll(getNomineeAddresses(transaction.getHash())); startRound = roundIndex; + System.out.println("New nominees:"); + latestNominees.forEach(n -> System.out.println(n)); } if (!transaction.isSolid()) { @@ -186,11 +194,13 @@ public Set getNomineeAddresses(Hash transaction) throws Exception{ // security+1 - n: tx with validator addresses if ((tx.getCurrentIndex() > security)) { + System.out.println("Get Nominees"); for (int i = 0; i < TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE / Hash.SIZE_IN_BYTES; i++) { Hash address = HashFactory.ADDRESS.create(tx.getSignature(), i * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); + Hash null_address = HashFactory.ADDRESS.create("0000000000000000000000000000000000000000000000000000000000000000"); // TODO: Do we send all validators or only adding and removing ones ? - if (address.equals(Hash.NULL_HASH)){ - break; + if (address.equals(null_address)){ + return validators; } validators.add(address); } From 0946ad4b14f74210b2ed685a1d7b22e904871bd5 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:22:53 +0200 Subject: [PATCH 122/223] update nominees when start round is reached, store round of latestNomineeUpdate --- .../impl/LatestMilestoneTrackerImpl.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 17af72e3..087b0fbf 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -90,6 +90,8 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { private int roundPause; + private int latestNomineeUpdate; + /** * Holds the round index of the latest round that we have seen / processed.
*/ @@ -160,6 +162,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP genesisTime = config.getGenesisTime(); roundDuration = config.getRoundDuration(); roundPause = 1000; //ms + latestNomineeUpdate = 0; setCurrentNominees(nomineeTracker.getLatestNominees()); @@ -192,6 +195,7 @@ public void setCurrentNominees(Set nominees) { //tangle.publish("lv %d %d", getCurrentRoundIndex(), nominees); log.delegate().info("Validators of round #{}: {}", getCurrentRoundIndex(), nominees); this.currentNominees = nominees; + this.latestNomineeUpdate = getCurrentRoundIndex(); } @Override @@ -245,19 +249,13 @@ public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneE @Override public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { try { - //System.out.println("Process Milestone"); - //System.out.println("Hash: " + transaction.getHash().hexString() + ", round: " + milestoneService.getRoundIndex(transaction)); - //System.out.println("Current Round: " + getCurrentRoundIndex()); + System.out.println("Process Milestone"); + System.out.println("Hash: " + transaction.getHash() + ", round: " + RoundViewModel.getRoundIndex(transaction)); + System.out.println("Current Round: " + getCurrentRoundIndex()); int roundIndex = RoundViewModel.getRoundIndex(transaction); int currentRound = getCurrentRoundIndex(); - // todo what happens if this method isn't called for that round and it passes the start round (do we need <= here?) - if (nomineeTracker.getStartRound() == currentRound) { - setCurrentNominees(nomineeTracker.getLatestNominees()); - allNominees.addAll(nomineeTracker.getLatestNominees()); - } - Set nominees = currentNominees; if (roundIndex != currentRound) { nominees = nomineeTracker.getNomineesOfRound(roundIndex); @@ -437,6 +435,12 @@ private void logProgress() { */ private void collectNewMilestoneCandidates() throws MilestoneException { try { + // update nominees + System.out.println("Update Nominees, startRound: " + nomineeTracker.getStartRound() + ", currentRound: " + getCurrentRoundIndex()); + if (nomineeTracker.getStartRound() == getCurrentRoundIndex() && latestNomineeUpdate < getCurrentRoundIndex()) { + setCurrentNominees(nomineeTracker.getLatestNominees()); + allNominees.addAll(nomineeTracker.getLatestNominees()); + } for (Hash address : allNominees) { for (Hash hash : AddressViewModel.load(tangle, address).getHashes()) { if (Thread.currentThread().isInterrupted()) { From d046bb5058044c4e49e296611ac7cd3e31ca24f6 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:25:00 +0200 Subject: [PATCH 123/223] integrate nomineeTracker, candidateTracker and nomineePublisher --- src/main/java/net/helix/hlx/HLX.java | 8 +++++++- src/main/java/net/helix/hlx/Helix.java | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index e98e4e1b..fd821416 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -9,6 +9,8 @@ import net.helix.hlx.service.API; import net.helix.hlx.service.Spammer; import net.helix.hlx.service.milestone.MilestonePublisher; +import net.helix.hlx.service.curator.impl.NomineePublisher; +import net.helix.hlx.service.milestone.NomineeTracker; import net.helix.hlx.service.restserver.resteasy.RestEasy; import net.helix.hlx.utils.HelixIOUtils; import org.apache.commons.lang3.ArrayUtils; @@ -95,6 +97,7 @@ private static class HLXLauncher { public static API api; public static XI XI; public static MilestonePublisher milestonePublisher; + public static NomineePublisher nomineePublisher; public static Spammer spammer; /** @@ -121,8 +124,9 @@ public static void main(String [] args) throws Exception { helix.spentAddressesService, helix.tangle, helix.bundleValidator, helix.snapshotProvider, helix.ledgerService, helix.node, helix.tipsSelector, helix.tipsViewModel, helix.transactionValidator, - helix.latestMilestoneTracker, helix.graph); + helix.latestMilestoneTracker, helix.candidateTracker, helix.graph); milestonePublisher = new MilestonePublisher(config, api); + nomineePublisher = new NomineePublisher(config, api); spammer = new Spammer(config, api); shutdownHook(); @@ -142,6 +146,7 @@ public static void main(String [] args) throws Exception { if(config.getSpamDelay() > 0) { spammer.startScheduledExecutorService(); } + nomineePublisher.startScheduledExecutorService(); } /** @@ -153,6 +158,7 @@ private static void shutdownHook() { log.info("Shutting down Helix node, please hold tight..."); try { milestonePublisher.shutdown(); + nomineePublisher.shutdown(); XI.shutdown(); api.shutDown(); helix.shutdown(); diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index d0d38a42..a982841d 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -11,6 +11,8 @@ import net.helix.hlx.network.replicator.Replicator; import net.helix.hlx.service.Graphstream; import net.helix.hlx.service.TipsSolidifier; +import net.helix.hlx.service.curator.CandidateTracker; +import net.helix.hlx.service.curator.impl.CandidateTrackerImpl; import net.helix.hlx.service.ledger.impl.LedgerServiceImpl; import net.helix.hlx.service.milestone.impl.*; import net.helix.hlx.service.snapshot.SnapshotException; @@ -81,6 +83,7 @@ public class Helix { public final MilestoneServiceImpl milestoneService; public final LatestMilestoneTrackerImpl latestMilestoneTracker; public final NomineeTrackerImpl nomineeTracker; + public final CandidateTrackerImpl candidateTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; public final SeenMilestonesRetrieverImpl seenMilestonesRetriever; public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); @@ -127,6 +130,7 @@ public Helix(HelixConfig configuration) throws TransactionPruningException, Snap milestoneService = new MilestoneServiceImpl(); latestMilestoneTracker = new LatestMilestoneTrackerImpl(); nomineeTracker = new NomineeTrackerImpl(); + candidateTracker = new CandidateTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); @@ -183,6 +187,8 @@ public void init() throws Exception { latestMilestoneTracker.start(); latestSolidMilestoneTracker.start(); + nomineeTracker.start(); + candidateTracker.start(); seenMilestonesRetriever.start(); milestoneSolidifier.start(); transactionRequesterWorker.start(); @@ -213,6 +219,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx } milestoneService.init(tangle, snapshotProvider, snapshotService, transactionValidator, configuration); nomineeTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); + candidateTracker.init(tangle, snapshotProvider, configuration); latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, nomineeTracker, configuration); latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService, latestMilestoneTracker, nomineeTracker); From 94a8320b86e1aa23111972f09c0a4ce95b5e435c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:26:23 +0200 Subject: [PATCH 124/223] add nomineeTracker to API tests --- src/test/java/net/helix/hlx/service/APICallTest.java | 2 +- src/test/java/net/helix/hlx/service/APIIntegrationTest.java | 2 +- src/test/java/net/helix/hlx/service/APITest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/net/helix/hlx/service/APICallTest.java b/src/test/java/net/helix/hlx/service/APICallTest.java index fc6656ac..d0f96519 100644 --- a/src/test/java/net/helix/hlx/service/APICallTest.java +++ b/src/test/java/net/helix/hlx/service/APICallTest.java @@ -16,7 +16,7 @@ public class APICallTest { public void setup() { HelixConfig configuration = Mockito.mock(HelixConfig.class); api = new API(configuration, null, null, null, null, null, null, null, - null, null, null, null, null, null); + null, null, null, null, null, null, null); } @Test diff --git a/src/test/java/net/helix/hlx/service/APIIntegrationTest.java b/src/test/java/net/helix/hlx/service/APIIntegrationTest.java index 24467ef2..3c4495d6 100644 --- a/src/test/java/net/helix/hlx/service/APIIntegrationTest.java +++ b/src/test/java/net/helix/hlx/service/APIIntegrationTest.java @@ -97,7 +97,7 @@ public static void setup() throws Exception { helix.spentAddressesService, helix.tangle, helix.bundleValidator, helix.snapshotProvider, helix.ledgerService, helix.node, helix.tipsSelector, helix.tipsViewModel, helix.transactionValidator, - helix.latestMilestoneTracker, helix.graph); + helix.latestMilestoneTracker, helix.candidateTracker, helix.graph); //init try { diff --git a/src/test/java/net/helix/hlx/service/APITest.java b/src/test/java/net/helix/hlx/service/APITest.java index ad330c34..e004dfa9 100644 --- a/src/test/java/net/helix/hlx/service/APITest.java +++ b/src/test/java/net/helix/hlx/service/APITest.java @@ -47,7 +47,7 @@ public void whenStoreTransactionsStatementThenSetArrivalTimeToCurrentMillis() th API api = new API(config, null, null, null, null, null, snapshotProvider, null, null, null, null, - transactionValidator, null, null); + transactionValidator, null, null, null); api.storeTransactionsStatement(Collections.singletonList(txHex)); verify(transaction).setArrivalTime(longThat( From 9554ad41e006b815460077b1cbe36e66ef6a23f1 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:29:56 +0200 Subject: [PATCH 125/223] take correct keyIndex and filepath --- .../service/milestone/MilestonePublisher.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index b4f09949..37bafa5a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -49,9 +49,15 @@ public MilestonePublisher(HelixConfig configuration, API api) { this.address = "6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"; this.sign = !this.config.isDontValidateTestnetMilestoneSig(); this.pubkeyDepth = config.getNumberOfKeysInMilestone(); - this.keyfileIndex = 1; + this.keyfileIndex = 0; this.maxKeyIndex = (int) Math.pow(2,this.pubkeyDepth); - this.currentKeyIndex = 0; + this.currentKeyIndex = 1024; + + /*try { + updateKeyfile(); + } catch (Exception e) { + e.printStackTrace(); + }*/ this.active = true; @@ -72,14 +78,13 @@ public static String getSeed() throws IOException { private void updateKeyfile() throws Exception { String seed = getSeed(); List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, this.maxKeyIndex * this.keyfileIndex, this.maxKeyIndex); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, "Nominee.key"); + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, "./src/main/resources/Nominee.key"); this.address = merkleTree.get(merkleTree.size()-1).get(0).toString(); sendApplication(); - this.active = true; } private void sendApplication() throws Exception { - this.api.sendApplication(this.address, this.mwm, this.sign); + this.api.sendApplication(this.address, this.mwm, this.sign, this.maxKeyIndex * this.keyfileIndex); } private int getRound(long time) { @@ -102,14 +107,15 @@ public void startScheduledExecutorService() { private void publishMilestone() throws Exception { if (this.active) { log.info("Publishing next Milestone..."); - if (currentKeyIndex < maxKeyIndex * this.keyfileIndex) { + if (currentKeyIndex < maxKeyIndex * (this.keyfileIndex + 1)) { this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, this.currentKeyIndex); this.currentKeyIndex += 1; } else { - log.info("Keyfile has expired! A new Keyfile will be generated, which pauses the MilestonePublisher until the process is finished."); + log.info("Keyfile has expired! The MilestonePublisher is paused until a new keyfile is generated."); this.active = false; this.keyfileIndex += 1; updateKeyfile(); + this.active = true; } } } From 42cc48c7acd1014e915e22d967f635697167a185 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:30:41 +0200 Subject: [PATCH 126/223] genesis time --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 5cdf0dfb..6a487bd1 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -943,7 +943,7 @@ public interface Defaults { String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; int MS_DELAY = 0; int MS_MIN_DELAY = 5; - long GENESIS_TIME = 1564398684743L; + long GENESIS_TIME = 1564496322557L; int ROUND_DURATION = 5000; //Snapshot From c65f2bd46b7dc653bed3c57de8e60d849fc0e9dc Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 30 Jul 2019 17:31:01 +0200 Subject: [PATCH 127/223] logs --- .../java/net/helix/hlx/BundleValidator.java | 2 +- .../helix/hlx/controllers/RoundViewModel.java | 24 +++++++++---------- .../java/net/helix/hlx/crypto/Merkle.java | 11 ++++++++- .../java/net/helix/hlx/crypto/Winternitz.java | 6 ----- .../milestone/impl/MilestoneServiceImpl.java | 1 + 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/main/java/net/helix/hlx/BundleValidator.java b/src/main/java/net/helix/hlx/BundleValidator.java index 1fef634c..b428c1bc 100644 --- a/src/main/java/net/helix/hlx/BundleValidator.java +++ b/src/main/java/net/helix/hlx/BundleValidator.java @@ -119,7 +119,7 @@ public static List> validate(Tangle tangle, Snapshot } sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); //verify bundle hash is correct - //System.out.println("Bundle Hash: " + instanceTransactionViewModels.get(0).getBundleHash().hexString()); + //System.out.println("Bundle Hash: " + instanceTransactionViewModels.get(0).getBundleHash()); //System.out.println("recalculated Bundle Hash: " + Hex.toHexString(bundleHashBytes)); if (Arrays.equals(instanceTransactionViewModels.get(0).getBundleHash().bytes(), bundleHashBytes)) { //normalizing the bundle in preparation for signature verification diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 6a1bd916..f23773c1 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -238,7 +238,7 @@ public static int getRoundIndex(TransactionViewModel milestoneTransaction) { public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ Set trunk = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - System.out.println("Get Milestone Trunk: hash = " + transaction.getHash() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); + //System.out.println("Get Milestone Trunk: hash = " + transaction.getHash() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk if (transaction.getCurrentIndex() == transaction.lastIndex()) { // add previous milestones to non analyzed transactions @@ -251,13 +251,13 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr Set prevMilestones = prevMilestone.getHashes(); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { - System.out.println("trunk (prev. milestones): "); + //System.out.println("trunk (prev. milestones): "); if (prevMilestones.isEmpty()) { trunk.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH); + //System.out.println(Hash.NULL_HASH); } else { trunk.addAll(prevMilestones); - prevMilestones.forEach(c -> System.out.println(c)); + //prevMilestones.forEach(c -> System.out.println(c)); } } } @@ -265,7 +265,7 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr else { // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash trunk.add(transaction.getTrunkTransactionHash()); - System.out.println("trunk (bundle): " + transaction.getTrunkTransactionHash()); + //System.out.println("trunk (bundle): " + transaction.getTrunkTransactionHash()); } return trunk; @@ -274,7 +274,7 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ Set branch = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); - System.out.println("Get Milestone Branch: hash = " + transaction.getHash() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); + //System.out.println("Get Milestone Branch: hash = " + transaction.getHash() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk and tips merkle root in branch if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root @@ -283,13 +283,13 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - System.out.println("branch (tips): "); + //System.out.println("branch (tips): "); if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH); + //System.out.println(Hash.NULL_HASH); } else { branch.addAll(confirmedTips); - confirmedTips.forEach(c -> System.out.println(c)); + //confirmedTips.forEach(c -> System.out.println(c)); } } } @@ -304,13 +304,13 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t Set prevMilestones = prevMilestone.getHashes(); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { - System.out.println("branch (prev. milestones): "); + //System.out.println("branch (prev. milestones): "); if (prevMilestones.isEmpty()) { branch.add(Hash.NULL_HASH); - System.out.println(Hash.NULL_HASH); + //System.out.println(Hash.NULL_HASH); } else { branch.addAll(prevMilestones); - prevMilestones.forEach(c -> System.out.println(c)); + //prevMilestones.forEach(c -> System.out.println(c)); } } } diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index cbd9b541..12b83a8c 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -97,9 +97,13 @@ public static List> buildMerkleTree(List leaves){ public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { + //System.out.println("Validate Merkle Signature"); final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex + //System.out.println("Address: " + validationAddress); + //System.out.println("Keyindex: " + keyIndex); + //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) //TODO: check if its okay here to use bundle hash instead of tx hash byte[] bundleHash = new byte[Sha3.HASH_LENGTH]; @@ -117,10 +121,15 @@ public static boolean validateMerkleSignature(List bundleT byte[] digests = bb.array(); byte[] address = Winternitz.address(mode, digests); + //System.out.println("Public Key: " + Hex.toHexString(address)); + //validate Merkle path + //System.out.println("Merkle Path: " + Hex.toHexString(merkleTx.getSignature())); byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, merkleTx.getSignature(), 0, keyIndex, depth); + //System.out.println("Recalculated Address: " + HashFactory.ADDRESS.create(merkleRoot)); + return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); } @@ -151,7 +160,7 @@ public static void createKeyfile(List> merkleTree, byte[] seed, int p bw.newLine(); writeKeys(bw, merkleTree.get(0)); for (int i = 1; i < merkleTree.size(); i++) { - writeKeys(bw, merkleTree.get(0)); + writeKeys(bw, merkleTree.get(i)); } } } diff --git a/src/main/java/net/helix/hlx/crypto/Winternitz.java b/src/main/java/net/helix/hlx/crypto/Winternitz.java index 80a2358a..11737e17 100644 --- a/src/main/java/net/helix/hlx/crypto/Winternitz.java +++ b/src/main/java/net/helix/hlx/crypto/Winternitz.java @@ -240,12 +240,6 @@ public static byte[] generateAddress (byte[] seed, int index, int security) { byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, seed, index); byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, security); byte[] digests = Winternitz.digests(SpongeFactory.Mode.S256, key); - if (index < 20) { - System.out.println(index); - System.out.println("Private Key: " + Hex.toHexString(key)); - System.out.println("Address: " + Hex.toHexString(Winternitz.address(SpongeFactory.Mode.S256, digests))); - System.out.println(); - } return Winternitz.address(SpongeFactory.Mode.S256, digests); } } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 71081084..c65bec67 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -188,6 +188,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getNumberOfKeysInMilestone()); + System.out.println("valid signature: " + validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { From c005f80d7324226329e60bbc1424e45a9cdee89a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 31 Jul 2019 16:24:32 +0200 Subject: [PATCH 128/223] rename logger --- .../net/helix/hlx/service/curator/impl/NomineePublisher.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 96cc200d..48183c6d 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -2,7 +2,6 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.service.API; -import net.helix.hlx.service.milestone.MilestonePublisher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,7 +10,7 @@ import java.util.concurrent.TimeUnit; public class NomineePublisher { - private static final Logger log = LoggerFactory.getLogger(MilestonePublisher.class); + private static final Logger log = LoggerFactory.getLogger(NomineePublisher.class); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private HelixConfig config; private API api; From 712a871cc77045f6f97915b6ca70aa5e4e8c58e8 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 31 Jul 2019 16:27:28 +0200 Subject: [PATCH 129/223] add nomineeTracker, activate MilestonePublisher when address is accepted as nominee --- src/main/java/net/helix/hlx/HLX.java | 2 +- .../service/milestone/MilestonePublisher.java | 47 ++++++++++++------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index fd821416..2ab1f4a2 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -125,7 +125,7 @@ public static void main(String [] args) throws Exception { helix.snapshotProvider, helix.ledgerService, helix.node, helix.tipsSelector, helix.tipsViewModel, helix.transactionValidator, helix.latestMilestoneTracker, helix.candidateTracker, helix.graph); - milestonePublisher = new MilestonePublisher(config, api); + milestonePublisher = new MilestonePublisher(config, api, helix.nomineeTracker); nomineePublisher = new NomineePublisher(config, api); spammer = new Spammer(config, api); shutdownHook(); diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 37bafa5a..d5ed6e8f 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -2,7 +2,9 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.model.AddressHash; import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.API; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; @@ -26,8 +28,9 @@ public class MilestonePublisher { private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private HelixConfig config; private API api; + NomineeTracker nomineeTracker; - private String address; + private Hash address; private String message; private int delay; private int mwm; @@ -36,34 +39,34 @@ public class MilestonePublisher { private int keyfileIndex; private int maxKeyIndex; private int currentKeyIndex; + private String seed; + private int startRound; public boolean active; - public MilestonePublisher(HelixConfig configuration, API api) { + public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nomineeTracker) { this.config = configuration; this.api = api; + this.nomineeTracker = nomineeTracker; this.delay = this.config.getMsDelay(); - int minDelay = this.config.getMinDelay(); this.mwm = this.config.getMwm(); this.message = StringUtils.repeat('0', 1024); - this.address = "6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"; + this.address = HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"); this.sign = !this.config.isDontValidateTestnetMilestoneSig(); this.pubkeyDepth = config.getNumberOfKeysInMilestone(); this.keyfileIndex = 0; this.maxKeyIndex = (int) Math.pow(2,this.pubkeyDepth); this.currentKeyIndex = 1024; + this.seed = "da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce"; // todo how to store seed secure + this.startRound = 0; /*try { - updateKeyfile(); + generateKeyfile(this.seed); } catch (Exception e) { e.printStackTrace(); }*/ this.active = true; - - if(this.delay < minDelay) { - this.delay = minDelay; - } } public static String getSeed() throws IOException { @@ -75,16 +78,15 @@ public static String getSeed() throws IOException { return seedBuilder.toString(); } - private void updateKeyfile() throws Exception { - String seed = getSeed(); + private void generateKeyfile(String seed) throws Exception { + log.info("Generating Keyfile (idx: " + this.keyfileIndex + ")"); List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, this.maxKeyIndex * this.keyfileIndex, this.maxKeyIndex); Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, "./src/main/resources/Nominee.key"); - this.address = merkleTree.get(merkleTree.size()-1).get(0).toString(); - sendApplication(); + this.address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } private void sendApplication() throws Exception { - this.api.sendApplication(this.address, this.mwm, this.sign, this.maxKeyIndex * this.keyfileIndex); + this.api.sendApplication(this.address.toString(), this.mwm, this.sign, this.maxKeyIndex * this.keyfileIndex); } private int getRound(long time) { @@ -105,17 +107,26 @@ public void startScheduledExecutorService() { } private void publishMilestone() throws Exception { + if (!this.active) { + if (this.startRound < getRound(System.currentTimeMillis()) && nomineeTracker.getLatestNominees().contains(this.address)) { + this.startRound = nomineeTracker.getStartRound(); + log.info("Address " + this.address + " is accepted from round #" + this.startRound); + } + if (this.startRound == getRound(System.currentTimeMillis())) { + this.active = true; + } + } if (this.active) { log.info("Publishing next Milestone..."); if (currentKeyIndex < maxKeyIndex * (this.keyfileIndex + 1)) { - this.api.storeAndBroadcastMilestoneStatement(this.address, this.message, this.mwm, this.sign, this.currentKeyIndex); + this.api.storeAndBroadcastMilestoneStatement(this.address.toString(), this.message, this.mwm, this.sign, this.currentKeyIndex); this.currentKeyIndex += 1; } else { - log.info("Keyfile has expired! The MilestonePublisher is paused until a new keyfile is generated."); + log.info("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network. This can take some time (<1m)."); this.active = false; this.keyfileIndex += 1; - updateKeyfile(); - this.active = true; + generateKeyfile(this.seed); + sendApplication(); } } } From 4cf16e080d4adff2009034814275351a98025912 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 1 Aug 2019 11:35:10 +0200 Subject: [PATCH 130/223] recover keyIndex and keyfileIndex when starting node --- .../service/milestone/MilestonePublisher.java | 122 +++++++++++------- 1 file changed, 72 insertions(+), 50 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index d5ed6e8f..662d1725 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -2,7 +2,6 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.crypto.Merkle; -import net.helix.hlx.model.AddressHash; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.API; @@ -11,20 +10,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.security.SecureRandom; +import java.io.*; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.Random; public class MilestonePublisher { private static final Logger log = LoggerFactory.getLogger(MilestonePublisher.class); + private static String keyfile = "./src/main/resources/Nominee.key"; private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private HelixConfig config; private API api; @@ -48,45 +43,47 @@ public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nom this.config = configuration; this.api = api; this.nomineeTracker = nomineeTracker; - this.delay = this.config.getMsDelay(); - this.mwm = this.config.getMwm(); - this.message = StringUtils.repeat('0', 1024); - this.address = HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"); - this.sign = !this.config.isDontValidateTestnetMilestoneSig(); - this.pubkeyDepth = config.getNumberOfKeysInMilestone(); - this.keyfileIndex = 0; - this.maxKeyIndex = (int) Math.pow(2,this.pubkeyDepth); - this.currentKeyIndex = 1024; - this.seed = "da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce"; // todo how to store seed secure - this.startRound = 0; - - /*try { - generateKeyfile(this.seed); - } catch (Exception e) { - e.printStackTrace(); - }*/ - this.active = true; + delay = config.getMsDelay(); + mwm = config.getMwm(); + message = StringUtils.repeat('0', 1024); + address = HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"); + sign = !config.isDontValidateTestnetMilestoneSig(); + pubkeyDepth = config.getNumberOfKeysInMilestone(); + keyfileIndex = 0; + maxKeyIndex = (int) Math.pow(2,pubkeyDepth); + currentKeyIndex = 0; + seed = "da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce"; // todo how to store seed secure + startRound = 0; + active = false; + } + + private void writeKeyIndex() throws IOException { + List> merkleTree = Merkle.readKeyfile(new File(keyfile)); + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); } - public static String getSeed() throws IOException { - StringBuilder seedBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(new File("./src/main/resources/Nominee.key")))) { + private void readKeyfileMetadata() throws IOException { + try (BufferedReader br = new BufferedReader(new FileReader(new File(keyfile)))) { String[] fields = br.readLine().split(" "); - seedBuilder.append(fields[1]); + pubkeyDepth = Integer.parseInt(fields[0]); + keyfileIndex = Integer.parseInt(fields[2]); + currentKeyIndex = Integer.parseInt(fields[3]); } - return seedBuilder.toString(); + List> merkleTree = Merkle.readKeyfile(new File(keyfile)); + address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } private void generateKeyfile(String seed) throws Exception { - log.info("Generating Keyfile (idx: " + this.keyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, this.maxKeyIndex * this.keyfileIndex, this.maxKeyIndex); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, "./src/main/resources/Nominee.key"); - this.address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); + log.info("Generating Keyfile (idx: " + keyfileIndex + ")"); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex); + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } private void sendApplication() throws Exception { - this.api.sendApplication(this.address.toString(), this.mwm, this.sign, this.maxKeyIndex * this.keyfileIndex); + log.info("Sending Application for Address " + address); + api.sendApplication(address.toString(), mwm, sign, maxKeyIndex * keyfileIndex); } private int getRound(long time) { @@ -99,33 +96,52 @@ private long getStartTime(int round) { public void startScheduledExecutorService() { log.info("MSS scheduledExecutorService started."); - log.info("Submitting Milestones every: " + config.getRoundDuration() + "s."); + try { + File f = new File(keyfile); + // read keyindex if keyfile exists, otherwise build new keyfile + if (f.isFile()) { + readKeyfileMetadata(); + } else { + generateKeyfile(seed); + } + // activate publisher if address is nominee, otherwise send application + if (nomineeTracker.getLatestNominees().contains(address)) { + log.info("Submitting Milestones every: " + (config.getRoundDuration() / 1000) + "s."); + active = true; + } else { + sendApplication(); + } + } catch (Exception e) { + e.printStackTrace(); + } + // get start time of next round int currentRound = getRound(System.currentTimeMillis()); long startTimeNextRound = getStartTime(currentRound + 1); log.info("Next Round starts in " + ((startTimeNextRound - System.currentTimeMillis()) / 1000) + "s."); - this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), config.getRoundDuration(), TimeUnit.MILLISECONDS); + scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), config.getRoundDuration(), TimeUnit.MILLISECONDS); } private void publishMilestone() throws Exception { - if (!this.active) { - if (this.startRound < getRound(System.currentTimeMillis()) && nomineeTracker.getLatestNominees().contains(this.address)) { - this.startRound = nomineeTracker.getStartRound(); - log.info("Address " + this.address + " is accepted from round #" + this.startRound); + if (!active) { + if (startRound < getRound(System.currentTimeMillis()) && nomineeTracker.getLatestNominees().contains(address)) { + startRound = nomineeTracker.getStartRound(); + log.info("Address " + address + " is accepted from round #" + startRound); } - if (this.startRound == getRound(System.currentTimeMillis())) { - this.active = true; + if (startRound == getRound(System.currentTimeMillis())) { + log.info("Submitting Milestones every: " + (config.getRoundDuration() / 1000) + "s."); + active = true; } } - if (this.active) { + if (active) { log.info("Publishing next Milestone..."); - if (currentKeyIndex < maxKeyIndex * (this.keyfileIndex + 1)) { - this.api.storeAndBroadcastMilestoneStatement(this.address.toString(), this.message, this.mwm, this.sign, this.currentKeyIndex); - this.currentKeyIndex += 1; + if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1)) { + api.storeAndBroadcastMilestoneStatement(address.toString(), message, mwm, sign, currentKeyIndex); + currentKeyIndex += 1; } else { log.info("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network. This can take some time (<1m)."); - this.active = false; - this.keyfileIndex += 1; - generateKeyfile(this.seed); + active = false; + keyfileIndex += 1; + generateKeyfile(seed); sendApplication(); } } @@ -143,6 +159,12 @@ private Runnable getRunnablePublishMilestone() { public void shutdown() { log.info("Shutting down MSS Thread"); + // store keyindex in keyfile + try { + writeKeyIndex(); + } catch (Exception e) { + e.printStackTrace(); + } scheduledExecutorService.shutdown(); } } From 2e2fc088ddb97715210a140e9daabeddd971beea Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 1 Aug 2019 11:37:25 +0200 Subject: [PATCH 131/223] store keyIndex and keyfileIndex in keyfile, move getSeed, tree and path representation as list --- .../java/net/helix/hlx/crypto/Merkle.java | 38 ++++++++++++------- src/main/java/net/helix/hlx/service/API.java | 37 +++++++++--------- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index 12b83a8c..851207c2 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -41,11 +41,11 @@ public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] return hash; } - public static byte[] getMerklePath(byte[][][] merkleTree, int keyIndex){ - byte[] merklePath = new byte[(merkleTree.length-1) * 32]; - for (int i = 0; i < merkleTree.length-1; i++) { - byte[] subkey = merkleTree[i][keyIndex ^ 1]; - System.arraycopy((subkey == null ? Hash.NULL_HASH.bytes() : subkey), 0, merklePath, i * 32, 32); + public static List getMerklePath(List> merkleTree, int keyIndex){ + List merklePath = new ArrayList<>((merkleTree.size()-1) * 32); + for (int i = 0; i < merkleTree.size()-1; i++) { + Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); + merklePath.add(subkey == null ? Hash.NULL_HASH : subkey); keyIndex /= 2; } return merklePath; @@ -133,30 +133,42 @@ public static boolean validateMerkleSignature(List bundleT return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); } - public static byte[][][] readKeyfile(File keyfile, StringBuilder seedBuilder) throws IOException { + public static List> readKeyfile(File keyfile) throws IOException { try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { String[] fields = br.readLine().split(" "); int depth = Integer.parseInt(fields[0]); - seedBuilder.append(fields[1]); - byte[][][] result = new byte[depth + 1][][]; + List> result = new ArrayList<>(depth + 1); for (int i = 0; i <= depth; i++) { - int col = 1 << (depth - i); - result[i] = new byte[1 << (depth - i)][32]; fields = br.readLine().split(" "); int leadingNulls = Integer.parseInt(fields[0]); + List row = new ArrayList<>(); + for (int j = 0; j < leadingNulls; j++) { + row.add(Hash.NULL_HASH); + } for (int j = 0; j < fields[1].length() / 64; j++) { - result[i][j + leadingNulls] = Hex.decode(fields[1].substring(j * 64, (j+1) * 64)); + row.add(HashFactory.TRANSACTION.create(fields[1].substring(j * 64, (j+1) * 64))); } + result.add(row); } return result; } } - public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, String filename) throws IOException { + public static String getSeed(File keyfile) throws IOException { + StringBuilder seedBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + seedBuilder.append(fields[1]); + } + return seedBuilder.toString(); + } + + + public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { // fill buffer try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { // write pubkey depth and seed into buffer - bw.write(pubkeyDepth + " " + Hex.toHexString(seed)); + bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); bw.newLine(); writeKeys(bw, merkleTree.get(0)); for (int i = 1; i < merkleTree.size(); i++) { diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index ea2b3934..2edfda43 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1574,12 +1574,13 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri if (sign) { // Get merkle path and store in signatureMessageFragment of Sibling Transaction - StringBuilder seedBuilder = new StringBuilder(); - byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); - String seed = seedBuilder.toString(); + File keyfile = new File("./src/main/resources/Nominee.key"); + List> merkleTree = Merkle.readKeyfile(keyfile); + String seed = Merkle.getSeed(keyfile); // create merkle path from keyfile - byte[] merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); - System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); + List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); // sign bundle hash and store signature in Milestone Transaction @@ -1638,7 +1639,6 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri public void sendApplication(final String address, final int minWeightMagnitude, boolean sign, int keyIndex) throws Exception { - System.out.println("Send Application"); // contain a signature that signs the siblings and thereby ensures the integrity. byte[] txApplication = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, txApplication, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); @@ -1676,12 +1676,14 @@ public void sendApplication(final String address, final int minWeightMagnitude, if (sign) { // Get merkle path and store in signatureMessageFragment of Sibling Transaction - StringBuilder seedBuilder = new StringBuilder(); - byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Nominee.key"), seedBuilder); - String seed = seedBuilder.toString(); + File keyfile = new File("./src/main/resources/Nominee.key"); + List> merkleTree = Merkle.readKeyfile(keyfile); + String seed = Merkle.getSeed(keyfile); // create merkle path from keyfile - byte[] merklePath = Merkle.getMerklePath(merkleTree, 0) ; - System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); + List merklePath = Merkle.getMerklePath(merkleTree, 0); + byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); + // sign bundle hash and store signature in Milestone Transaction @@ -1716,7 +1718,6 @@ public void sendApplication(final String address, final int minWeightMagnitude, public void updateNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { - System.out.println("Update Nominees"); List nominees = new ArrayList<>(candidateTracker.getNominees()); // get round @@ -1772,13 +1773,13 @@ public void updateNominees(int startRoundDelay, final int minWeightMagnitude, Bo if (sign) { // Get merkle path and store in signatureMessageFragment of Sibling Transaction - StringBuilder seedBuilder = new StringBuilder(); - byte[][][] merkleTree = Merkle.readKeyfile(new File("./src/main/resources/Coordinator.key"), seedBuilder); - String seed = seedBuilder.toString(); + File keyfile = new File("./src/main/resources/Coordinator.key"); + List> merkleTree = Merkle.readKeyfile(keyfile); + String seed = Merkle.getSeed(keyfile); // create merkle path from keyfile - //int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); - byte[] merklePath = Merkle.getMerklePath(merkleTree, keyIndex); - System.arraycopy(merklePath, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, merklePath.length); + List merklePath = Merkle.getMerklePath(merkleTree, keyIndex); + byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); // sign bundle hash and store signature in Milestone Transaction From 777a4d13dc668638622bbaa7707986a8ce586058 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 1 Aug 2019 11:37:50 +0200 Subject: [PATCH 132/223] logs --- .../net/helix/hlx/service/curator/impl/NomineePublisher.java | 2 +- .../hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 48183c6d..13efd2dc 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -38,7 +38,7 @@ public void startScheduledExecutorService() { } private void UpdateNominees() throws Exception { - log.info("Publishing new Nominees..."); + log.info("Publishing new Nominees ..."); this.api.updateNominees(this.startRoundDelay, this.mwm, this.sign, this.currentKeyIndex); this.currentKeyIndex += 1; } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 087b0fbf..6ca0532a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -436,7 +436,6 @@ private void logProgress() { private void collectNewMilestoneCandidates() throws MilestoneException { try { // update nominees - System.out.println("Update Nominees, startRound: " + nomineeTracker.getStartRound() + ", currentRound: " + getCurrentRoundIndex()); if (nomineeTracker.getStartRound() == getCurrentRoundIndex() && latestNomineeUpdate < getCurrentRoundIndex()) { setCurrentNominees(nomineeTracker.getLatestNominees()); allNominees.addAll(nomineeTracker.getLatestNominees()); From ed64189caf962448c0dbfeae4385fd10407df92c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 14:41:09 +0200 Subject: [PATCH 133/223] wait 20s before spamming --- src/main/java/net/helix/hlx/service/Spammer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/Spammer.java b/src/main/java/net/helix/hlx/service/Spammer.java index 0dde8b25..848edfa9 100644 --- a/src/main/java/net/helix/hlx/service/Spammer.java +++ b/src/main/java/net/helix/hlx/service/Spammer.java @@ -31,7 +31,7 @@ public Spammer(SpamConfig config, API api) { public void startScheduledExecutorService() { log.info("Spammer scheduledExecutorService started."); log.info("Submitting Tx every: " + this.delay + "ms."); - this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnableSendTx(), 10000, this.delay, TimeUnit.MILLISECONDS); + this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnableSendTx(), 20000, this.delay, TimeUnit.MILLISECONDS); } private void sendTx() throws Exception { From bb86b7d6c345977a41a03dadd0ccb4b8657e143b Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 14:45:10 +0200 Subject: [PATCH 134/223] delete addToNomineeQueue from interface --- .../java/net/helix/hlx/service/curator/CandidateTracker.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java index fe25dbd4..fcd6664e 100755 --- a/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java +++ b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java @@ -34,10 +34,6 @@ public interface CandidateTracker { */ boolean processCandidate(Hash transactionHash) throws CuratorException; - /** - * @param candidateAddress address of the candidate to add to the {@code nominationQueue} - */ - void addToNomineeQueue(Hash candidateAddress); Set getNominees(); From 4fa618959a6f088ba67d55965aeb95fb4f54a6d6 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 14:46:45 +0200 Subject: [PATCH 135/223] move initial nominees to CandidateTracker, enable removing nominees, logs --- .../curator/impl/CandidateTrackerImpl.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index 11b2fefd..1bebb047 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -4,10 +4,12 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.AddressViewModel; import net.helix.hlx.controllers.BundleViewModel; +import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.Merkle; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; +import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.curator.*; import net.helix.hlx.service.milestone.MilestoneSolidifier; import net.helix.hlx.service.snapshot.SnapshotProvider; @@ -132,6 +134,10 @@ public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide this.config = config; this.snapshotProvider = snapshotProvider; + nominees = new HashSet<>(); + nominees.add(HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db")); + nominees.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); + return this; } @@ -232,9 +238,7 @@ public boolean processCandidate(Hash transactionHash) throws CuratorException { public boolean processCandidate(TransactionViewModel transaction) throws CuratorException { try { if (this.config.getTrusteeAddress().equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == transaction.lastIndex()) { - System.out.println("Process Candidate"); - System.out.println("Hash: " + transaction.getHash()); - System.out.println("Address: " + transaction.getAddressHash()); + log.info("Process Candidate Transaction " + transaction.getHash()); // get tail BundleViewModel bundle = BundleViewModel.load(tangle, transaction.getBundleHash()); TransactionViewModel tail = null; @@ -246,9 +250,14 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator } switch (validateCandidate(tail, SpongeFactory.Mode.S256, 1)) { case VALID: - if (!nominees.contains(tail.getAddressHash())) { + log.info("Candidate Transaction " + transaction.getHash() + " is VALID, Address: " + tail.getAddressHash()); + log.info("Join/Leave: " + RoundViewModel.getRoundIndex(tail)); + if (RoundViewModel.getRoundIndex(tail) == 1) { addToNomineeQueue(tail.getAddressHash()); } + if (RoundViewModel.getRoundIndex(tail) == -1) { + removeFromNomineeQueue(tail.getAddressHash()); + } /*if (!transaction.isSolid()) { candidateSolidifier.add(transaction.getHash(), currentRoundIndex); @@ -258,10 +267,12 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator break; case INCOMPLETE: + log.info("Candidate Transaction " + transaction.getHash() + " is INCOMPLETE"); return false; case INVALID: // do not re-analyze anymore + log.info("Candidate Transaction " + transaction.getHash() + " is INVALID"); return true; default: @@ -283,6 +294,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); if (bundleTransactions.isEmpty()) { + System.out.println("bundle empty"); return INCOMPLETE; } else { for (final List bundleTransactionViewModels : bundleTransactions) { @@ -322,13 +334,20 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM * @param candidateAddress * */ - @Override - public void addToNomineeQueue(Hash candidateAddress) { + + private void addToNomineeQueue(Hash candidateAddress) { //Double w = (curatorService.getCandidateNormalizedWeight(candidateAddress, this.seenCandidates)); this.nominees.add(candidateAddress); tangle.publish("nac %d", candidateAddress); //nac = newly added candidate - log.delegate().info("New candidate {} added for", candidateAddress); + log.delegate().info("New candidate {} added", candidateAddress); + } + + private void removeFromNomineeQueue(Hash candidateAddress) { + this.nominees.remove(candidateAddress); + + tangle.publish("nrc %d", candidateAddress); //nac = newly removed candidate + log.delegate().info("Candidate {} removed", candidateAddress); } @Override From 210a0eef04ccf8b049e5b9f424147a299bc567af Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 14:50:18 +0200 Subject: [PATCH 136/223] remove old address when generating new keyfile --- .../service/milestone/MilestonePublisher.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 662d1725..b9e6c2c8 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -47,7 +47,6 @@ public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nom delay = config.getMsDelay(); mwm = config.getMwm(); message = StringUtils.repeat('0', 1024); - address = HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"); sign = !config.isDontValidateTestnetMilestoneSig(); pubkeyDepth = config.getNumberOfKeysInMilestone(); keyfileIndex = 0; @@ -81,9 +80,10 @@ private void generateKeyfile(String seed) throws Exception { address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } - private void sendApplication() throws Exception { - log.info("Sending Application for Address " + address); - api.sendApplication(address.toString(), mwm, sign, maxKeyIndex * keyfileIndex); + private void sendApplication(Hash address, boolean join) throws Exception { + log.info("Sending Application for Address " + address + ", " + (join ? "join" : "leave")); + api.sendApplication(address.toString(), mwm, sign, currentKeyIndex, join); + currentKeyIndex += 1; } private int getRound(long time) { @@ -104,13 +104,6 @@ public void startScheduledExecutorService() { } else { generateKeyfile(seed); } - // activate publisher if address is nominee, otherwise send application - if (nomineeTracker.getLatestNominees().contains(address)) { - log.info("Submitting Milestones every: " + (config.getRoundDuration() / 1000) + "s."); - active = true; - } else { - sendApplication(); - } } catch (Exception e) { e.printStackTrace(); } @@ -123,7 +116,7 @@ public void startScheduledExecutorService() { private void publishMilestone() throws Exception { if (!active) { - if (startRound < getRound(System.currentTimeMillis()) && nomineeTracker.getLatestNominees().contains(address)) { + if (startRound < getRound(System.currentTimeMillis()) && !nomineeTracker.getLatestNominees().isEmpty() && nomineeTracker.getLatestNominees().contains(address)) { startRound = nomineeTracker.getStartRound(); log.info("Address " + address + " is accepted from round #" + startRound); } @@ -134,15 +127,19 @@ private void publishMilestone() throws Exception { } if (active) { log.info("Publishing next Milestone..."); - if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1)) { + if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1) - 1) { api.storeAndBroadcastMilestoneStatement(address.toString(), message, mwm, sign, currentKeyIndex); currentKeyIndex += 1; } else { log.info("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network. This can take some time (<1m)."); active = false; + // remove old address + sendApplication(address, false); + // generate keyfile and add new address keyfileIndex += 1; + currentKeyIndex = maxKeyIndex * keyfileIndex; generateKeyfile(seed); - sendApplication(); + sendApplication(address, true); } } } From 8682cf1e18dc7fb71646949c62c65458d939949e Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 14:51:43 +0200 Subject: [PATCH 137/223] remove initial nominees, logs --- .../milestone/impl/LatestMilestoneTrackerImpl.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 6ca0532a..22c03925 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -154,7 +154,6 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP this.nomineeTracker = nomineeTracker; allNominees = new HashSet<>(); - allNominees.addAll(nomineeTracker.getLatestNominees()); System.out.println("Current Time: " + System.currentTimeMillis()); System.out.println("Genesis Time: " + config.getGenesisTime()); @@ -249,9 +248,7 @@ public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneE @Override public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { try { - System.out.println("Process Milestone"); - System.out.println("Hash: " + transaction.getHash() + ", round: " + RoundViewModel.getRoundIndex(transaction)); - System.out.println("Current Round: " + getCurrentRoundIndex()); + log.info("Process Milestone " + transaction.getHash() + ", round: " + RoundViewModel.getRoundIndex(transaction)); int roundIndex = RoundViewModel.getRoundIndex(transaction); int currentRound = getCurrentRoundIndex(); @@ -270,7 +267,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, nominees)) { case VALID: - System.out.println("VALID"); + log.info("Milestone " + transaction.getHash() + " is VALID"); // TODO: 1.review conditions, 2.is okay to store here? // before a milestone can be added to a round the following conditions have to be checked: // - index is bigger than initial snapshot (above) @@ -280,7 +277,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw // - index is bigger than snapshot index // - attachment timestamp is in correct time window for the index // - there doesn't already exist a milestone with the same address for that round - System.out.println("attachment timestamp round: " + getRound(transaction.getAttachmentTimestamp())); + //System.out.println("attachment timestamp round: " + getRound(transaction.getAttachmentTimestamp())); if (roundIndex == getRound(transaction.getAttachmentTimestamp()) && isRoundActive(transaction.getAttachmentTimestamp())) { RoundViewModel currentRoundViewModel; @@ -315,7 +312,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw break; case INCOMPLETE: - System.out.println("INCOMPLETE"); + log.info("Milestone " + transaction.getHash() + " is INCOMPLETE"); milestoneSolidifier.add(transaction.getHash(), roundIndex); transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); From 2b2f6b989b12679e2406ba20aba4b5fc30fa67b3 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 14:54:46 +0200 Subject: [PATCH 138/223] store latest nominee tx hash, check current idy = 0, replace all nominees --- .../hlx/service/milestone/NomineeTracker.java | 2 ++ .../milestone/impl/NomineeTrackerImpl.java | 36 ++++++++++--------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java b/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java index 540de21b..fcb83a43 100644 --- a/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java @@ -16,6 +16,8 @@ public interface NomineeTracker { Set getNomineeAddresses(Hash transaction) throws Exception; + Hash getLatestNomineeHash(); + void analyzeCuratorTransactions() throws Exception; void collectNewCuratorTransactions() throws Exception; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java index 12338a8b..7d7270a5 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java @@ -41,6 +41,7 @@ public class NomineeTrackerImpl implements NomineeTracker { private MilestoneService milestoneService; private MilestoneSolidifier milestoneSolidifier; private Set latestNominees; + private Hash latestNomineeHash; private int startRound; private final Set seenCuratorTransactions = new HashSet<>(); private final Deque curatorTransactionsToAnalyze = new ArrayDeque<>(); @@ -54,11 +55,8 @@ public NomineeTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; - latestNominees = new HashSet<>(); - latestNominees.add(HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db")); - latestNominees.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); - - startRound = 1; + startRound = 0; + latestNomineeHash = Hash.NULL_HASH; //bootstrapLatestNominees(); return this; @@ -69,6 +67,11 @@ public Set getLatestNominees() { return latestNominees; } + @Override + public Hash getLatestNomineeHash() { + return latestNomineeHash; + } + @Override public int getStartRound() {return startRound; } @@ -112,7 +115,7 @@ public MilestoneValidity validateNominees(TransactionViewModel transactionViewMo // validate signature Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, 15); - System.out.println("valid signature (nominee): " + validSignature); + //System.out.println("valid signature (nominee): " + validSignature); if (Curator_Address.equals(senderAddress) && validSignature) { return VALID; @@ -133,12 +136,11 @@ public MilestoneValidity validateNominees(TransactionViewModel transactionViewMo public boolean processNominees(Hash transactionHash) throws Exception { TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); try { - if (Curator_Address.equals(transaction.getAddressHash())) { + if (Curator_Address.equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { int roundIndex = RoundViewModel.getRoundIndex(transaction); - System.out.println("Process Nominee"); - System.out.println("Hash: " + transaction.getHash() + ", start round: " + roundIndex); + log.info("Process Nominee Transaction " + transaction.getHash() + ", start round: " + roundIndex); // if the trustee transaction is older than our ledger start point: we already processed it in the past if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { @@ -148,13 +150,15 @@ public boolean processNominees(Hash transactionHash) throws Exception { // validate switch (validateNominees(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { case VALID: - System.out.println("round index: " + roundIndex); - System.out.println("start round: " + startRound); + log.info("Nominee Transaction " + transaction.getHash() + " is VALID"); + //System.out.println("round index: " + roundIndex); + //System.out.println("start round: " + startRound); if (roundIndex > startRound) { - latestNominees.addAll(getNomineeAddresses(transaction.getHash())); + latestNominees = getNomineeAddresses(transaction.getHash()); + latestNomineeHash = transaction.getHash(); startRound = roundIndex; - System.out.println("New nominees:"); - latestNominees.forEach(n -> System.out.println(n)); + //System.out.println("New nominees:"); + //latestNominees.forEach(n -> System.out.println(n)); } if (!transaction.isSolid()) { @@ -164,8 +168,8 @@ public boolean processNominees(Hash transactionHash) throws Exception { break; case INCOMPLETE: + log.info("Nominee Transaction " + transaction.getHash() + " is INCOMPLETE"); milestoneSolidifier.add(transaction.getHash(), roundIndex); - return false; default: @@ -194,7 +198,7 @@ public Set getNomineeAddresses(Hash transaction) throws Exception{ // security+1 - n: tx with validator addresses if ((tx.getCurrentIndex() > security)) { - System.out.println("Get Nominees"); + //System.out.println("Get Nominees"); for (int i = 0; i < TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE / Hash.SIZE_IN_BYTES; i++) { Hash address = HashFactory.ADDRESS.create(tx.getSignature(), i * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); Hash null_address = HashFactory.ADDRESS.create("0000000000000000000000000000000000000000000000000000000000000000"); From 3aca21584e50ded33aa77bea17fbd000e0e0f080 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 14:59:40 +0200 Subject: [PATCH 139/223] reference initial nominee tx hash, store 1/-1 fro join/leave in tag, add nomineeTracker to API --- src/main/java/net/helix/hlx/service/API.java | 34 +++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 2edfda43..d4aae73b 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -21,6 +21,7 @@ import net.helix.hlx.service.dto.*; import net.helix.hlx.service.ledger.LedgerService; import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.NomineeTracker; import net.helix.hlx.service.restserver.RestConnector; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.service.spentaddresses.SpentAddressesService; @@ -107,6 +108,7 @@ public class API { private final TransactionValidator transactionValidator; private final LatestMilestoneTracker latestMilestoneTracker; private final CandidateTracker candidateTracker; + private final NomineeTracker nomineeTracker; private final Graphstream graph; private final int maxFindTxs; @@ -150,7 +152,7 @@ public API(HelixConfig configuration, XI XI, TransactionRequester transactionReq SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, SnapshotProvider snapshotProvider, LedgerService ledgerService, Node node, TipSelector tipsSelector, TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - LatestMilestoneTracker latestMilestoneTracker, CandidateTracker candidateTracker, Graphstream graph) { + LatestMilestoneTracker latestMilestoneTracker, CandidateTracker candidateTracker, NomineeTracker nomineeTracker, Graphstream graph) { this.configuration = configuration; this.XI = XI; @@ -166,6 +168,7 @@ public API(HelixConfig configuration, XI XI, TransactionRequester transactionReq this.transactionValidator = transactionValidator; this.latestMilestoneTracker = latestMilestoneTracker; this.candidateTracker = candidateTracker; + this.nomineeTracker = nomineeTracker; this.graph = graph; maxFindTxs = configuration.getMaxFindTransactions(); @@ -623,7 +626,7 @@ public void storeTransactionsStatement(final List txString) throws Excep } transactionViewModel.updateSender("local"); transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); - System.out.println("published tx: " + transactionViewModel.getHash()); + //System.out.println("published tx: " + transactionViewModel.getHash()); } if (graph != null) { @@ -1494,13 +1497,13 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) List confirmedTips = new LinkedList<>(); - System.out.println("Tips (" + tipsViewModel.getTips().size() + ")"); + //System.out.println("Tips (" + tipsViewModel.getTips().size() + ")"); snapshotProvider.getLatestSnapshot().lockRead(); try { WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); for (Hash transaction : tipsViewModel.getTips()) { TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); - System.out.println("Tip: " + transaction); + //System.out.println("Tip: " + transaction); //System.out.println("bundle valid: " + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size()); //System.out.println("type: " + txVM.getType()); //System.out.println("index: " + txVM.getCurrentIndex()); @@ -1511,7 +1514,7 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri txVM.isSolid() && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { if (walkValidator.isValid(transaction)) { - System.out.println("(selected)"); + //System.out.println("(selected)"); confirmedTips.add(transaction); } } @@ -1603,8 +1606,8 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri List txToApprove = new ArrayList<>(); //System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); if(RoundViewModel.latest(tangle) == null) { - txToApprove.add(Hash.NULL_HASH); - txToApprove.add(Hash.NULL_HASH); + txToApprove.add(nomineeTracker.getLatestNomineeHash()); // approove initial curator tx + txToApprove.add(nomineeTracker.getLatestNomineeHash()); } else { // trunk // todo what happens if there is no entry for the previous round ? @@ -1637,19 +1640,24 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri } - public void sendApplication(final String address, final int minWeightMagnitude, boolean sign, int keyIndex) throws Exception { + public void sendApplication(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, boolean join) throws Exception { + + int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); // contain a signature that signs the siblings and thereby ensures the integrity. byte[] txApplication = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, txApplication, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); System.arraycopy(Serializer.serialize(2L), 0, txApplication, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txApplication, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + // add tag = 1 if node wants to join and tag = -1 if node wants to leave + System.arraycopy(Serializer.serialize(join ? 1L : -1L), 0, txApplication, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); // siblings for merkle tree. byte[] txSibling = new byte[TransactionViewModel.SIZE]; System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); System.arraycopy(Serializer.serialize(2L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize((long) keyIndex % maxKeyIndex), 0, txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); // curator recipient. byte[] txCurator = new byte[TransactionViewModel.SIZE]; @@ -1680,7 +1688,7 @@ public void sendApplication(final String address, final int minWeightMagnitude, List> merkleTree = Merkle.readKeyfile(keyfile); String seed = Merkle.getSeed(keyfile); // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, 0); + List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); @@ -1697,13 +1705,7 @@ public void sendApplication(final String address, final int minWeightMagnitude, } // get branch and trunk - List txToApprove = new ArrayList<>(); - if(RoundViewModel.latest(tangle) == null) { - txToApprove.add(Hash.NULL_HASH); - txToApprove.add(Hash.NULL_HASH); - } else { - txToApprove = getTransactionToApproveTips(3, Optional.empty()); - } + List txToApprove = getTransactionToApproveTips(3, Optional.empty()); // attach, broadcast and store List transactions = new ArrayList<>(); From 2aec63f5b083e715ed390ad26792b681832a7144 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 15:00:21 +0200 Subject: [PATCH 140/223] add nomineeTracker to API tests --- src/test/java/net/helix/hlx/service/APICallTest.java | 2 +- src/test/java/net/helix/hlx/service/APIIntegrationTest.java | 2 +- src/test/java/net/helix/hlx/service/APITest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/net/helix/hlx/service/APICallTest.java b/src/test/java/net/helix/hlx/service/APICallTest.java index d0f96519..2ef59059 100644 --- a/src/test/java/net/helix/hlx/service/APICallTest.java +++ b/src/test/java/net/helix/hlx/service/APICallTest.java @@ -16,7 +16,7 @@ public class APICallTest { public void setup() { HelixConfig configuration = Mockito.mock(HelixConfig.class); api = new API(configuration, null, null, null, null, null, null, null, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null, null); } @Test diff --git a/src/test/java/net/helix/hlx/service/APIIntegrationTest.java b/src/test/java/net/helix/hlx/service/APIIntegrationTest.java index 3c4495d6..7d1ab953 100644 --- a/src/test/java/net/helix/hlx/service/APIIntegrationTest.java +++ b/src/test/java/net/helix/hlx/service/APIIntegrationTest.java @@ -97,7 +97,7 @@ public static void setup() throws Exception { helix.spentAddressesService, helix.tangle, helix.bundleValidator, helix.snapshotProvider, helix.ledgerService, helix.node, helix.tipsSelector, helix.tipsViewModel, helix.transactionValidator, - helix.latestMilestoneTracker, helix.candidateTracker, helix.graph); + helix.latestMilestoneTracker, helix.candidateTracker, helix.nomineeTracker, helix.graph); //init try { diff --git a/src/test/java/net/helix/hlx/service/APITest.java b/src/test/java/net/helix/hlx/service/APITest.java index e004dfa9..e0d8e31d 100644 --- a/src/test/java/net/helix/hlx/service/APITest.java +++ b/src/test/java/net/helix/hlx/service/APITest.java @@ -47,7 +47,7 @@ public void whenStoreTransactionsStatementThenSetArrivalTimeToCurrentMillis() th API api = new API(config, null, null, null, null, null, snapshotProvider, null, null, null, null, - transactionValidator, null, null, null); + transactionValidator, null, null, null, null); api.storeTransactionsStatement(Collections.singletonList(txHex)); verify(transaction).setArrivalTime(longThat( From a8a2bd32965d1639a4f7497a1c9e7be994de722a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 15:01:06 +0200 Subject: [PATCH 141/223] add nomineeTracker to API --- src/main/java/net/helix/hlx/HLX.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 2ab1f4a2..426cb442 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -124,7 +124,7 @@ public static void main(String [] args) throws Exception { helix.spentAddressesService, helix.tangle, helix.bundleValidator, helix.snapshotProvider, helix.ledgerService, helix.node, helix.tipsSelector, helix.tipsViewModel, helix.transactionValidator, - helix.latestMilestoneTracker, helix.candidateTracker, helix.graph); + helix.latestMilestoneTracker, helix.candidateTracker, helix.nomineeTracker, helix.graph); milestonePublisher = new MilestonePublisher(config, api, helix.nomineeTracker); nomineePublisher = new NomineePublisher(config, api); spammer = new Spammer(config, api); From 6fccf1fdb5adecda9f0345f13c213f66de902fca Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 15:02:19 +0200 Subject: [PATCH 142/223] delete this --- .../service/curator/impl/NomineePublisher.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 13efd2dc..b19d94ec 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -24,23 +24,23 @@ public class NomineePublisher { public NomineePublisher(HelixConfig configuration, API api) { this.config = configuration; this.api = api; - this.delay = 30000; - this.mwm = 2; - this.sign = !this.config.isDontValidateTestnetMilestoneSig(); - this.currentKeyIndex = 0; - this.startRoundDelay = 2; + delay = 30000; + mwm = 2; + sign = !config.isDontValidateTestnetMilestoneSig(); + currentKeyIndex = 0; + startRoundDelay = 2; } public void startScheduledExecutorService() { log.info("NomineePublisher scheduledExecutorService started."); log.info("Update Nominees every: " + delay / 1000 + "s."); - this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnableUpdateNominees(), this.delay, this.delay, TimeUnit.MILLISECONDS); + scheduledExecutorService.scheduleWithFixedDelay(getRunnableUpdateNominees(), 0, delay, TimeUnit.MILLISECONDS); } private void UpdateNominees() throws Exception { log.info("Publishing new Nominees ..."); - this.api.updateNominees(this.startRoundDelay, this.mwm, this.sign, this.currentKeyIndex); - this.currentKeyIndex += 1; + api.updateNominees(startRoundDelay, mwm, sign, currentKeyIndex); + currentKeyIndex += 1; } private Runnable getRunnableUpdateNominees() { From ffdbb6b8d266a206114292fbc6674fc03ae0ff64 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 15:03:23 +0200 Subject: [PATCH 143/223] logs --- .../net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java | 1 - .../hlx/service/milestone/impl/MilestoneServiceImpl.java | 4 ++-- .../helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index e48ce8bf..11373250 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -105,7 +105,6 @@ public void restoreLedgerState() throws LedgerException { @Override public boolean applyMilestoneToLedger(RoundViewModel round) throws LedgerException { - System.out.println("Apply Round " + round.index() + " to ledger"); if (graph != null) { for (Hash milestoneHash : round.getHashes()) { try { diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index c65bec67..693bdd1f 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -169,7 +169,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM RoundViewModel round = RoundViewModel.get(tangle, roundIndex); if (round != null && round.getHashes().contains(transactionViewModel.getHash())) { // Already validated. - System.out.println("Already validated!"); + log.info("Milestone " + transactionViewModel.getHash() + " was already validated!"); return VALID; } @@ -188,7 +188,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getNumberOfKeysInMilestone()); - System.out.println("valid signature: " + validSignature); + //System.out.println("valid signature: " + validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 261b84e4..66fa6037 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -157,7 +157,7 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna snapshot.setHash(merkleTree.get(merkleTree.size() - 1).get(0)); //System.out.println("Milestones :"); //lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); - System.out.println("Snapshot Hash: " + snapshot.getHash()); + log.info("Apply Round " + lastAppliedRound.index() + " to ledger, Snapshot Hash: " + snapshot.getHash()); //System.out.println("Snapshot:"); //snapshot.getBalances().forEach((a,b) -> System.out.println("Address: " + a.hexString() + ", " + b)); From c56901e4deb3b95ec9848a2e0501ccf9fea9feaa Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 2 Aug 2019 19:15:37 +0200 Subject: [PATCH 144/223] add config parameters and flags for nominee and curator --- src/main/java/net/helix/hlx/HLX.java | 6 +- src/main/java/net/helix/hlx/Helix.java | 4 +- .../net/helix/hlx/conf/BaseHelixConfig.java | 117 ++++++++++++------ .../net/helix/hlx/conf/ConsensusConfig.java | 2 +- .../net/helix/hlx/conf/CuratorConfig.java | 43 +++++++ .../net/helix/hlx/conf/MilestoneConfig.java | 28 +++-- .../net/helix/hlx/conf/TestnetConfig.java | 13 -- src/main/java/net/helix/hlx/service/API.java | 6 +- .../curator/impl/CandidateTrackerImpl.java | 10 +- .../curator/impl/NomineePublisher.java | 8 +- .../service/milestone/MilestonePublisher.java | 4 +- .../milestone/impl/MilestoneServiceImpl.java | 2 +- .../milestone/impl/NomineeTrackerImpl.java | 9 +- .../java/net/helix/hlx/conf/ConfigTest.java | 4 +- 14 files changed, 168 insertions(+), 88 deletions(-) create mode 100644 src/main/java/net/helix/hlx/conf/CuratorConfig.java diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 426cb442..0e0d917d 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -140,13 +140,15 @@ public static void main(String [] args) throws Exception { log.error("Exception during Helix node initialisation: ", e); throw e; } - if(config.getMsDelay() > 0) { + if(config.getNomineeEnabled()) { milestonePublisher.startScheduledExecutorService(); } if(config.getSpamDelay() > 0) { spammer.startScheduledExecutorService(); } - nomineePublisher.startScheduledExecutorService(); + if(config.getCuratorEnabled()) { + nomineePublisher.startScheduledExecutorService(); + } } /** diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index a982841d..60b78b4c 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -188,7 +188,9 @@ public void init() throws Exception { latestMilestoneTracker.start(); latestSolidMilestoneTracker.start(); nomineeTracker.start(); - candidateTracker.start(); + if (configuration.getCuratorEnabled()) { + candidateTracker.start(); + } seenMilestonesRetriever.start(); milestoneSolidifier.start(); transactionRequesterWorker.start(); diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 6a487bd1..aa9ef548 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -15,9 +15,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; /* Note: the fields in this class are being deserialized from Jackson so they must follow Java Bean convention. @@ -117,12 +115,19 @@ public abstract class BaseHelixConfig implements HelixConfig { protected String saveLogBasePath = Defaults.SAVELOG_BASE_PATH; protected String saveLogXMLFile = Defaults.SAVELOG_XML_FILE; + //Curator + protected boolean curatorEnabled = Defaults.CURATOR_ENABLED; + protected Hash curatorAddress = Defaults.CURATOR_ADDRESS; + protected int updateNomineeDelay = Defaults.UPDATE_NOMINEE_DELAY; + protected int startRoundDelay = Defaults.START_ROUND_DELAY; + protected int curatorKeyDepth = Defaults.CURATOR_KEY_DEPTH; + //Milestone - protected int msDelay = Defaults.MS_DELAY; - protected int minDelay = Defaults.MS_MIN_DELAY; - protected Hash cooAddress = HashFactory.ADDRESS.create(Defaults.COORDINATOR_ADDRESS); + protected boolean nomineeEnabled = Defaults.NOMINEE_ENABLED; + protected Set initialNominees = Defaults.INITIAL_NOMINEES; protected long genesisTime = Defaults.GENESIS_TIME; protected int roundDuration = Defaults.ROUND_DURATION; + protected int milestoneKeyDepth = Defaults.MILESTONE_KEY_DEPTH; //Spammer protected int spamDelay = Defaults.SPAM_DELAY; @@ -747,16 +752,6 @@ protected void setCacheSizeBytes(int cacheSizeBytes) { this.cacheSizeBytes = cacheSizeBytes; } - @Override - public Hash getTrusteeAddress() { - return cooAddress; - } - - @Override - public boolean isDontValidateTestnetMilestoneSig() { - return false; - } - @Override public int getMaxDepth() { return maxDepth; @@ -795,43 +790,70 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { this.maxAnalyzedTransactions = maxAnalyzedTransactions; } + // Curator @Override - public int getPowThreads() { - return powThreads; - } + public boolean getCuratorEnabled() {return curatorEnabled; } + @JsonProperty + @Parameter(names = {"--curator"}, description = CuratorConfig.Descriptions.CURATOR_ENABLED, arity = 1) + protected void setCuratorEnabled(boolean curatorEnabled) { this.curatorEnabled = curatorEnabled; } + @Override + public Hash getCuratorAddress() { return curatorAddress; } + + @Override + public boolean isDontValidateTestnetCuratorSig() { return false; } + + @Override + public int getUpdateNomineeDelay() {return updateNomineeDelay; } @JsonProperty - @Parameter(names = "--pow-threads", description = PoWConfig.Descriptions.POW_THREADS) - protected void setPowThreads(int powThreads) { - this.powThreads = powThreads; - } + @Parameter(names = {"--update-nominee"}, description = CuratorConfig.Descriptions.UPDATE_NOMINEE_DELAY) + protected void setUpdateNomineeDelay(int updateNomineeDelay) { this.updateNomineeDelay = updateNomineeDelay; } @Override - public int getMsDelay() { - return msDelay; - } + public int getStartRoundDelay() {return startRoundDelay; } @JsonProperty - @Parameter(names = {"--ms-delay", "-m"}, description = MilestoneConfig.Descriptions.MS_DELAY) - protected void setMsDelay(int delay) { this.msDelay = delay; } + @Parameter(names = {"--start-nominee"}, description = CuratorConfig.Descriptions.START_ROUND_DELAY) + protected void setStartRoundDelay(int startRoundDelay) { this.startRoundDelay = startRoundDelay; } @Override - public int getMinDelay() { - return minDelay; - } + public int getCuratorKeyDepth() {return curatorKeyDepth; } + + // Milestone + @Override + public boolean getNomineeEnabled() {return nomineeEnabled; } @JsonProperty - @Parameter(names = {"--min-delay"}, description = MilestoneConfig.Descriptions.MS_MIN_DELAY) - protected void setMinDelay(int minDelay) { this.minDelay = minDelay; } + @Parameter(names = {"--nominee"}, description = MilestoneConfig.Descriptions.NOMINEE_ENABLED, arity = 1) + protected void setNomineeEnabled(boolean nomineeEnabled) { this.nomineeEnabled = nomineeEnabled; } + + @Override + public Set getInitialNominees() {return initialNominees; } + + @Override + public boolean isDontValidateTestnetMilestoneSig() { + return false; + } @Override public long getGenesisTime() { return genesisTime; } + @JsonProperty + @Parameter(names = {"--genesis"}, description = MilestoneConfig.Descriptions.GENESIS_TIME) + protected void setGenesisTime(int genesisTime) { this.genesisTime = genesisTime; } @Override public int getRoundDuration() { return roundDuration; } + @JsonProperty + @Parameter(names = {"--round"}, description = MilestoneConfig.Descriptions.ROUND_DURATION) + protected void setRoundDuration(int roundDuration) { this.roundDuration = roundDuration; } + + @Override + public int getMilestoneKeyDepth() {return milestoneKeyDepth; } + + // POW @Override public boolean isPoWDisabled() { return powDisabled; @@ -840,6 +862,16 @@ public boolean isPoWDisabled() { @Parameter(names = {"--pow-disabled"}, description = APIConfig.Descriptions.IS_POW_DISABLED) protected void setPowDisabled(boolean powDisabled) { this.powDisabled = powDisabled; } + @Override + public int getPowThreads() { + return powThreads; + } + @JsonProperty + @Parameter(names = "--pow-threads", description = PoWConfig.Descriptions.POW_THREADS) + protected void setPowThreads(int powThreads) { + this.powThreads = powThreads; + } + @Override public boolean isSaveLogEnabled() { return saveLogEnabled; @@ -864,6 +896,7 @@ public String getSaveLogXMLFile() { @Parameter(names = {"--savelog-xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE) protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; } + // Spam @Override public int getSpamDelay() { return spamDelay; @@ -939,12 +972,22 @@ public interface Defaults { //PoW int POW_THREADS = 8; + //Curator + boolean CURATOR_ENABLED = false; + Hash CURATOR_ADDRESS = HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"); + int UPDATE_NOMINEE_DELAY = 30000; + int START_ROUND_DELAY = 5; + int CURATOR_KEY_DEPTH = 15; + //Milestone - String COORDINATOR_ADDRESS = "2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"; - int MS_DELAY = 0; - int MS_MIN_DELAY = 5; - long GENESIS_TIME = 1564496322557L; + boolean NOMINEE_ENABLED = false; + Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( + HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"), + HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") + )); + long GENESIS_TIME = 1564763402905L; int ROUND_DURATION = 5000; + int MILESTONE_KEY_DEPTH = 10; //Snapshot boolean LOCAL_SNAPSHOTS_ENABLED = true; diff --git a/src/main/java/net/helix/hlx/conf/ConsensusConfig.java b/src/main/java/net/helix/hlx/conf/ConsensusConfig.java index 13ac8119..b84ce093 100644 --- a/src/main/java/net/helix/hlx/conf/ConsensusConfig.java +++ b/src/main/java/net/helix/hlx/conf/ConsensusConfig.java @@ -6,5 +6,5 @@ * @implNote It currently extends two other interfaces. This has been done due to lack of separation of concerns in * the current code base and will be changed in the future */ -public interface ConsensusConfig extends SnapshotConfig, MilestoneConfig { +public interface ConsensusConfig extends SnapshotConfig, MilestoneConfig, CuratorConfig { } diff --git a/src/main/java/net/helix/hlx/conf/CuratorConfig.java b/src/main/java/net/helix/hlx/conf/CuratorConfig.java new file mode 100644 index 00000000..c82ff3e0 --- /dev/null +++ b/src/main/java/net/helix/hlx/conf/CuratorConfig.java @@ -0,0 +1,43 @@ +package net.helix.hlx.conf; + +import net.helix.hlx.model.Hash; + +/** + * Configs that should be used for tracking candidate applications and publishing nominees + */ +public interface CuratorConfig extends Config { + /** + * @return {@value Descriptions#CURATOR_ENABLED} + */ + boolean getCuratorEnabled(); + /** + * @return Descriptions#CURATOR_ADDRESS + */ + Hash getCuratorAddress(); + /** + * @return {@value Descriptions#DONT_VALIDATE_TESTNET_CURATOR_SIG} + */ + boolean isDontValidateTestnetCuratorSig(); + /** + * @return {@value Descriptions#UPDATE_NOMINEE_DELAY} + */ + int getUpdateNomineeDelay(); + /** + * @return {@value Descriptions#START_ROUND_DELAY} + */ + int getStartRoundDelay(); + /** + * @return {@value Descriptions#CURATOR_KEY_DEPTH} + */ + int getCuratorKeyDepth(); + + interface Descriptions { + String CURATOR_ENABLED = "Flag that determines if the node is a Curator."; + String CURATOR_ADDRESS = "The address of the node that publishes nominees"; + String DONT_VALIDATE_TESTNET_CURATOR_SIG = "Disable curator validation on testnet"; + String UPDATE_NOMINEE_DELAY = "The desired delay for updating nominees in seconds."; + String START_ROUND_DELAY = "The number of rounds between nominees are published and the round they start to operate."; + String CURATOR_KEY_DEPTH = "Depth of the merkle tree nominee transactions are signed with."; + } +} + diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index 4229395e..7a4e9aff 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -9,32 +9,36 @@ */ public interface MilestoneConfig extends Config { /** - * @return Descriptions#VALIDATOR_ADDRESSES + * @return {@value Descriptions#NOMINEE_ENABLED} */ - Hash getTrusteeAddress(); + boolean getNomineeEnabled(); + /** + * @return Descriptions#INITIAL_NOMINEES + */ + Set getInitialNominees(); /** * @return {@value Descriptions#DONT_VALIDATE_TESTNET_MILESTONE_SIG} */ boolean isDontValidateTestnetMilestoneSig(); /** - * @return {@value Descriptions#MS_DELAY} + * @return {@value Descriptions#GENESIS_TIME} */ - int getMsDelay(); + long getGenesisTime(); /** - * @return {@value Descriptions#MS_MIN_DELAY} + * @return {@value Descriptions#ROUND_DURATION} */ - int getMinDelay(); - - long getGenesisTime(); - int getRoundDuration(); + /** + * @return {@value Descriptions#MILESTONE_KEY_DEPTH} + */ + int getMilestoneKeyDepth(); interface Descriptions { - String VALIDATOR_ADDRESSES = "The addresses of nodes that are allowed to publish milestones"; + String NOMINEE_ENABLED = "Flag that determines if the node is a Nominee."; + String INITIAL_NOMINEES = "The addresses of nominees the network starts with"; String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable coordinator validation on testnet"; - String MS_DELAY = "The desired milestone delay in seconds."; - String MS_MIN_DELAY = "The minimum delay between publishing milestones."; String GENESIS_TIME = "Time when the ledger started."; String ROUND_DURATION = "Duration of a round in milli secounds."; + String MILESTONE_KEY_DEPTH = "Depth of the merkle tree the milestones are signed with."; } } diff --git a/src/main/java/net/helix/hlx/conf/TestnetConfig.java b/src/main/java/net/helix/hlx/conf/TestnetConfig.java index 225f90ad..2c8acefc 100644 --- a/src/main/java/net/helix/hlx/conf/TestnetConfig.java +++ b/src/main/java/net/helix/hlx/conf/TestnetConfig.java @@ -11,7 +11,6 @@ public class TestnetConfig extends BaseHelixConfig { - protected Hash cooAddress = Defaults.COORDINATOR_ADDRESS; protected boolean dontValidateTestnetMilestoneSig = Defaults.DONT_VALIDATE_MILESTONE_SIG; protected String snapshotFile = Defaults.SNAPSHOT_FILE; protected String snapshotSignatureFile = Defaults.SNAPSHOT_SIG; @@ -34,17 +33,6 @@ public boolean isTestnet() { return true; } - @Override - public Hash getTrusteeAddress() { - return cooAddress; - } - - @JsonProperty - @Parameter(names = "--testnet-coordinator", description = MilestoneConfig.Descriptions.VALIDATOR_ADDRESSES) - protected void setCoordinator(Hash coordinator) { - this.cooAddress = coordinator; - } - @Override public boolean isDontValidateTestnetMilestoneSig() { return dontValidateTestnetMilestoneSig; @@ -163,7 +151,6 @@ public void setDbLogPath(String dbLogPath) { } public interface Defaults { - Hash COORDINATOR_ADDRESS = HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"); boolean DONT_VALIDATE_MILESTONE_SIG = false; String LOCAL_SNAPSHOTS_BASE_PATH = "testnet"; String SNAPSHOT_FILE = "/snapshotTestnet.txt"; diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index d4aae73b..47955e37 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -673,7 +673,7 @@ private AbstractResponse getNodeInfoStatement() throws Exception { tipsViewModel.size(), transactionRequester.numberOfTransactionsToRequest(), features, - configuration.getTrusteeAddress().toString()); + configuration.getCuratorAddress().toString()); } /** @@ -1661,7 +1661,7 @@ public void sendApplication(final String address, final int minWeightMagnitude, // curator recipient. byte[] txCurator = new byte[TransactionViewModel.SIZE]; - System.arraycopy(configuration.getTrusteeAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(configuration.getCuratorAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); System.arraycopy(Serializer.serialize(2L), 0, txCurator, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); System.arraycopy(Serializer.serialize(2L), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txCurator, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); @@ -1730,7 +1730,7 @@ public void updateNominees(int startRoundDelay, final int minWeightMagnitude, Bo // contain a signature that signs the siblings and thereby ensures the integrity. byte[] txCurator = new byte[TransactionViewModel.SIZE]; - System.arraycopy(configuration.getTrusteeAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(configuration.getCuratorAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); System.arraycopy(Serializer.serialize(1L + n), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txCurator, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); System.arraycopy(Serializer.serialize((long) startRoundIndex), 0, txCurator, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index 1bebb047..293dec96 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -134,9 +134,7 @@ public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide this.config = config; this.snapshotProvider = snapshotProvider; - nominees = new HashSet<>(); - nominees.add(HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db")); - nominees.add(HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033")); + nominees = config.getInitialNominees(); return this; } @@ -182,7 +180,7 @@ private void candidateTrackerThread() { //@VisibleForTesting private void collectNewCandidates() throws CuratorException { try { - for (Hash hash : AddressViewModel.load(tangle, this.config.getTrusteeAddress()).getHashes()) { + for (Hash hash : AddressViewModel.load(tangle, this.config.getCuratorAddress()).getHashes()) { if (Thread.currentThread().isInterrupted()) { return; } @@ -237,7 +235,7 @@ public boolean processCandidate(Hash transactionHash) throws CuratorException { @Override public boolean processCandidate(TransactionViewModel transaction) throws CuratorException { try { - if (this.config.getTrusteeAddress().equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == transaction.lastIndex()) { + if (this.config.getCuratorAddress().equals(transaction.getAddressHash()) && transaction.getCurrentIndex() == transaction.lastIndex()) { log.info("Process Candidate Transaction " + transaction.getHash()); // get tail BundleViewModel bundle = BundleViewModel.load(tangle, transaction.getBundleHash()); @@ -305,7 +303,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getNumberOfKeysInMilestone()); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); System.out.println("valid signature (candidate): " + validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index b19d94ec..3c85b967 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -24,11 +24,11 @@ public class NomineePublisher { public NomineePublisher(HelixConfig configuration, API api) { this.config = configuration; this.api = api; - delay = 30000; - mwm = 2; - sign = !config.isDontValidateTestnetMilestoneSig(); + delay = config.getUpdateNomineeDelay(); + mwm = config.getMwm(); + sign = !config.isDontValidateTestnetCuratorSig(); currentKeyIndex = 0; - startRoundDelay = 2; + startRoundDelay = config.getStartRoundDelay(); } public void startScheduledExecutorService() { diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index b9e6c2c8..1c6f2671 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -44,7 +44,7 @@ public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nom this.api = api; this.nomineeTracker = nomineeTracker; - delay = config.getMsDelay(); + delay = config.getRoundDuration(); mwm = config.getMwm(); message = StringUtils.repeat('0', 1024); sign = !config.isDontValidateTestnetMilestoneSig(); @@ -111,7 +111,7 @@ public void startScheduledExecutorService() { int currentRound = getRound(System.currentTimeMillis()); long startTimeNextRound = getStartTime(currentRound + 1); log.info("Next Round starts in " + ((startTimeNextRound - System.currentTimeMillis()) / 1000) + "s."); - scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), config.getRoundDuration(), TimeUnit.MILLISECONDS); + scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), delay, TimeUnit.MILLISECONDS); } private void publishMilestone() throws Exception { diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 693bdd1f..17baa49a 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -187,7 +187,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getNumberOfKeysInMilestone()); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); //System.out.println("valid signature: " + validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java index 7d7270a5..9df1b5b6 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java @@ -50,12 +50,13 @@ public NomineeTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, this.tangle = tangle; this.config = config; - this.Curator_Address = config.getTrusteeAddress(); + this.Curator_Address = config.getCuratorAddress(); this.snapshotProvider = snapshotProvider; this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; + this.latestNominees = config.getInitialNominees(); - startRound = 0; + startRound = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration() + 2; latestNomineeHash = Hash.NULL_HASH; //bootstrapLatestNominees(); @@ -114,10 +115,10 @@ public MilestoneValidity validateNominees(TransactionViewModel transactionViewMo // validate signature Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, 15); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getCuratorKeyDepth()); //System.out.println("valid signature (nominee): " + validSignature); - if (Curator_Address.equals(senderAddress) && validSignature) { + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (Curator_Address.equals(senderAddress) && validSignature)) { return VALID; } else { return INVALID; diff --git a/src/test/java/net/helix/hlx/conf/ConfigTest.java b/src/test/java/net/helix/hlx/conf/ConfigTest.java index 592ec9a1..0d4c8159 100644 --- a/src/test/java/net/helix/hlx/conf/ConfigTest.java +++ b/src/test/java/net/helix/hlx/conf/ConfigTest.java @@ -104,7 +104,7 @@ public void argsParsingMainnetTest() { Assert.assertEquals("db path", "/db", helixConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, helixConfig.isZmqEnabled()); Assert.assertNotEquals("mwm", 4, helixConfig.getMwm()); - Assert.assertNotEquals("coo", helixConfig.getTrusteeAddress(), "TTTTTTTTT"); + Assert.assertNotEquals("coo", helixConfig.getCuratorAddress(), "TTTTTTTTT"); Assert.assertEquals("--testnet-no-coo-validation", false, helixConfig.isDontValidateTestnetMilestoneSig()); } @@ -172,7 +172,7 @@ public void argsParsingTestnetTest() { Assert.assertEquals("db path", "/db", helixConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, helixConfig.isZmqEnabled()); Assert.assertEquals("mwm", 4, helixConfig.getMwm()); - Assert.assertEquals("coo", "TTTTTTTTT", helixConfig.getTrusteeAddress()); + Assert.assertEquals("coo", "TTTTTTTTT", helixConfig.getCuratorAddress()); Assert.assertEquals("--testnet-no-coo-validation", true, helixConfig.isDontValidateTestnetMilestoneSig()); } From 53806209256780e07e02c81cf37894eaaffbc946 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 6 Aug 2019 12:23:13 +0200 Subject: [PATCH 145/223] bump version --- README.md | 34 +++++++++++++++------------- pom.xml | 2 +- src/main/java/net/helix/hlx/HLX.java | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7838614c..f325ca09 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,12 @@ [![license][4]][5] [![build][6]][7] [![grade][8]][9] [![coverage][10]][11] [![discord][14]][15] # Helix-1.0 -This is the 1.0 implementation of the Helix Protocol based on [**IRI**](https://github.com/iotaledger/iri/). -- **Latest release:** 0.5.8 pre-release +A Quorum based Tangle implementation forked from [**IRI**](https://github.com/iotaledger/iri/). +- **Latest release:** 0.6.0 pre-release - **License:** GPLv3 +Special thanks to all of the [IOTA Contributors](https://github.com/iotaledger/iri/graphs/contributors)! + ## Developers - Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us. @@ -71,20 +73,20 @@ A client interested in real time state updates and notifications could use any d Currently the following topics are covered: -| Topic | Description | Tested? | -| ----- | ----- | ----- | -| `dns` | Neighbor related info | ✓ | -| `hmr` | Hit/miss ration | ✖ | -| `antn` | Added non-tethered neighbors (testnet only) | ✖ | -| `rntn` | Refused non-tethered neighbors | ✖ | -| `rtl` | for transactions randomly removed from the request list | ✖ | -| `lmi` | Latest solid milestone index | ✓ | -| `lmhs` | Latest solid milestone hash | ✓ | -| `sn` | Uses solid milestone's child measurement to publish newly confirmed tx. | ✓ | -| `tx` | Newly seen transactions | ✓ | -| `ct5s2m` | Confirmed transactions older than 5s and younger than 2m | ✓ | -| `t5s2m` | total transactions older than 5s and younger than 2m | ✓ | -| `
` | Watching all traffic on a specified address | ✓ | +| Topic | Description | +| ----- | ----- | +| `dns` | Neighbor related info | +| `hmr` | Hit/miss ration | +| `antn` | Added non-tethered neighbors (testnet only) | +| `rntn` | Refused non-tethered neighbors | +| `rtl` | for transactions randomly removed from the request list | +| `lmi` | Latest solid milestone index | +| `lmhs` | Latest solid milestone hash | +| `sn` | Uses solid milestone's child measurement to publish newly confirmed tx. | +| `tx` | Newly seen transactions | +| `ct5s2m` | Confirmed transactions older than 5s and younger than 2m | +| `t5s2m` | total transactions older than 5s and younger than 2m | +| `
` | Watching all traffic on a specified address | diff --git a/pom.xml b/pom.xml index 9493824a..d0c141d6 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.helix helix - 0.5.8 + 0.6.0 Helix Helix-1.0 diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 0e0d917d..4be31183 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -46,7 +46,7 @@ public class HLX { public static final String MAINNET_NAME = "HLX"; public static final String TESTNET_NAME = "HLX Testnet"; - public static final String VERSION = "0.5.8"; + public static final String VERSION = "0.6.0"; /** * The entry point of the helix sandbox. From c8b027710a55841ecd22310cf662e930842ba35e Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 6 Aug 2019 13:25:46 +0200 Subject: [PATCH 146/223] Added todos and optimized verbosity --- src/main/java/net/helix/hlx/HLX.java | 2 +- .../net/helix/hlx/conf/BaseHelixConfig.java | 2 +- .../net/helix/hlx/crypto/GreedyMiner.java | 6 ++--- src/main/java/net/helix/hlx/service/API.java | 13 ++++------ .../service/milestone/MilestonePublisher.java | 24 +++++++++---------- .../impl/LatestMilestoneTrackerImpl.java | 15 +++++------- .../impl/LatestSolidMilestoneTrackerImpl.java | 6 ++--- .../milestone/impl/MilestoneServiceImpl.java | 13 ++++------ .../impl/SeenMilestonesRetrieverImpl.java | 8 ------- .../snapshot/impl/SnapshotServiceImpl.java | 2 +- 10 files changed, 36 insertions(+), 55 deletions(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 4be31183..3c7ff5e8 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -68,7 +68,7 @@ public static void main(String[] args) throws Exception { private static void configureLogging() { HelixIOUtils.saveLogs(); String config = System.getProperty("logback.configurationFile"); - String level = System.getProperty("logging-level", "info").toUpperCase(); + String level = System.getProperty("logging-level", "debug").toUpperCase(); switch (level) { case "OFF": case "ERROR": diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index aa9ef548..fbb8e2dd 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -985,7 +985,7 @@ public interface Defaults { HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"), HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") )); - long GENESIS_TIME = 1564763402905L; + long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) int ROUND_DURATION = 5000; int MILESTONE_KEY_DEPTH = 10; diff --git a/src/main/java/net/helix/hlx/crypto/GreedyMiner.java b/src/main/java/net/helix/hlx/crypto/GreedyMiner.java index baa6f546..79ea9978 100644 --- a/src/main/java/net/helix/hlx/crypto/GreedyMiner.java +++ b/src/main/java/net/helix/hlx/crypto/GreedyMiner.java @@ -79,7 +79,7 @@ public synchronized boolean mine(byte[] txBytes, int difficulty, int threadCount cancel(); } } - log.debug("MINER_STATE: " + state); + //log.debug("MINER_STATE: {}", state); return state.get() == COMPLETED; } @@ -112,8 +112,8 @@ private Runnable getMiner(byte[] txBytes, byte[] target, int offset, int step) { if (state.compareAndSet(RUNNING, COMPLETED)) { System.arraycopy(result, TransactionViewModel.NONCE_OFFSET, txBytes, TransactionViewModel.NONCE_OFFSET, TransactionViewModel.NONCE_SIZE); - log.debug("TX_HASH: " + Hex.toHexString(hash)); - log.debug("NONCE : " + Hex.toHexString(ByteBuffer.allocate(Long.BYTES).putLong(nonce).array())); + //log.debug("TX_HASH: {}", Hex.toHexString(hash)); + //log.debug("NONCE : {}", Hex.toHexString(ByteBuffer.allocate(Long.BYTES).putLong(nonce).array())); } } } diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 47955e37..41279f85 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1497,18 +1497,11 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) List confirmedTips = new LinkedList<>(); - //System.out.println("Tips (" + tipsViewModel.getTips().size() + ")"); snapshotProvider.getLatestSnapshot().lockRead(); try { WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); for (Hash transaction : tipsViewModel.getTips()) { TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); - //System.out.println("Tip: " + transaction); - //System.out.println("bundle valid: " + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size()); - //System.out.println("type: " + txVM.getType()); - //System.out.println("index: " + txVM.getCurrentIndex()); - //System.out.println("solid: " + txVM.isSolid()); - //System.out.println("walker valid: " + walkValidator.isValid(transaction)); if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && txVM.getCurrentIndex() == 0 && txVM.isSolid() && @@ -1530,6 +1523,8 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri // get number of transactions needed for tips long n = (long) (confirmedTips.size()/16) + 1; + + // todo remove duplicates, generating different bundle types may be added as an utility class // contain a signature that signs the siblings and thereby ensures the integrity. byte[] txMilestone = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, txMilestone, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); @@ -1619,12 +1614,12 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones //txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); - log.info("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); + log.debug("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); //todo should be removed after testing } //branch List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips - log.info("Branch (Tips): " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); + log.debug("Branch (Tips): " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); //todo should be removed after testing } // attach, broadcast and store diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 1c6f2671..454125e2 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -52,7 +52,7 @@ public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nom keyfileIndex = 0; maxKeyIndex = (int) Math.pow(2,pubkeyDepth); currentKeyIndex = 0; - seed = "da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce"; // todo how to store seed secure + seed = "da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce"; // todo seed should be stored in a hidden file for which only the user and this application have read permission startRound = 0; active = false; } @@ -74,15 +74,15 @@ private void readKeyfileMetadata() throws IOException { } private void generateKeyfile(String seed) throws Exception { - log.info("Generating Keyfile (idx: " + keyfileIndex + ")"); + log.debug("Generating Keyfile (idx: " + keyfileIndex + ")"); List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex); Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } - private void sendApplication(Hash address, boolean join) throws Exception { - log.info("Sending Application for Address " + address + ", " + (join ? "join" : "leave")); - api.sendApplication(address.toString(), mwm, sign, currentKeyIndex, join); + private void sendApplication(Hash identity, boolean join) throws Exception { + log.debug("Signing {} identity: {} ", (join ? "up" : "off"), identity); + api.sendApplication(identity.toString(), mwm, sign, currentKeyIndex, join); currentKeyIndex += 1; } @@ -95,10 +95,10 @@ private long getStartTime(int round) { } public void startScheduledExecutorService() { - log.info("MSS scheduledExecutorService started."); + log.info("MilestonePublisher started."); try { File f = new File(keyfile); - // read keyindex if keyfile exists, otherwise build new keyfile + // read keyIndex if key-file exists, otherwise build new key-file if (f.isFile()) { readKeyfileMetadata(); } else { @@ -110,7 +110,7 @@ public void startScheduledExecutorService() { // get start time of next round int currentRound = getRound(System.currentTimeMillis()); long startTimeNextRound = getStartTime(currentRound + 1); - log.info("Next Round starts in " + ((startTimeNextRound - System.currentTimeMillis()) / 1000) + "s."); + log.debug("Next round commencing in {}s.", (startTimeNextRound - System.currentTimeMillis()) / 1000); scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), delay, TimeUnit.MILLISECONDS); } @@ -118,20 +118,20 @@ private void publishMilestone() throws Exception { if (!active) { if (startRound < getRound(System.currentTimeMillis()) && !nomineeTracker.getLatestNominees().isEmpty() && nomineeTracker.getLatestNominees().contains(address)) { startRound = nomineeTracker.getStartRound(); - log.info("Address " + address + " is accepted from round #" + startRound); + log.debug("Legitimized nominee {} for round #{}", address, startRound); } if (startRound == getRound(System.currentTimeMillis())) { - log.info("Submitting Milestones every: " + (config.getRoundDuration() / 1000) + "s."); + log.debug("Submitting milestones every: " + (config.getRoundDuration() / 1000) + "s."); active = true; } } if (active) { - log.info("Publishing next Milestone..."); + log.debug("Publishing next Milestone..."); if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1) - 1) { api.storeAndBroadcastMilestoneStatement(address.toString(), message, mwm, sign, currentKeyIndex); currentKeyIndex += 1; } else { - log.info("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network. This can take some time (<1m)."); + log.debug("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network."); active = false; // remove old address sendApplication(address, false); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java index 22c03925..5bbaedd7 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java @@ -155,9 +155,6 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP allNominees = new HashSet<>(); - System.out.println("Current Time: " + System.currentTimeMillis()); - System.out.println("Genesis Time: " + config.getGenesisTime()); - System.out.println("Round Duration: " + config.getRoundDuration()); genesisTime = config.getGenesisTime(); roundDuration = config.getRoundDuration(); roundPause = 1000; //ms @@ -179,7 +176,7 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP @Override public void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numberOfMilestones, int numberOfNominees) { tangle.publish("lmi %s %d", milestoneHash, roundIndex); - log.delegate().info("New milestone {} ({}/{}) added to round #{}", milestoneHash, numberOfMilestones, numberOfNominees, roundIndex); + log.delegate().debug("New milestone {} ({}/{}) added to round #{}", milestoneHash, numberOfMilestones, numberOfNominees, roundIndex); } @Override @@ -192,7 +189,7 @@ public void setCurrentRoundIndex(int roundIndex) { @Override public void setCurrentNominees(Set nominees) { //tangle.publish("lv %d %d", getCurrentRoundIndex(), nominees); - log.delegate().info("Validators of round #{}: {}", getCurrentRoundIndex(), nominees); + log.delegate().debug("Validators of round #{}: {}", getCurrentRoundIndex(), nominees); this.currentNominees = nominees; this.latestNomineeUpdate = getCurrentRoundIndex(); } @@ -248,7 +245,7 @@ public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneE @Override public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { try { - log.info("Process Milestone " + transaction.getHash() + ", round: " + RoundViewModel.getRoundIndex(transaction)); + log.debug("Process Milestone " + transaction.getHash() + ", round: " + RoundViewModel.getRoundIndex(transaction)); int roundIndex = RoundViewModel.getRoundIndex(transaction); int currentRound = getCurrentRoundIndex(); @@ -267,7 +264,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, nominees)) { case VALID: - log.info("Milestone " + transaction.getHash() + " is VALID"); + log.debug("Milestone " + transaction.getHash() + " is VALID"); // TODO: 1.review conditions, 2.is okay to store here? // before a milestone can be added to a round the following conditions have to be checked: // - index is bigger than initial snapshot (above) @@ -312,7 +309,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw break; case INCOMPLETE: - log.info("Milestone " + transaction.getHash() + " is INCOMPLETE"); + log.debug("Milestone " + transaction.getHash() + " is INCOMPLETE"); milestoneSolidifier.add(transaction.getHash(), roundIndex); transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); @@ -417,7 +414,7 @@ private void latestMilestoneTrackerThread() { */ private void logProgress() { if (milestoneCandidatesToAnalyze.size() > 1) { - log.info("Processing milestone candidates (" + milestoneCandidatesToAnalyze.size() + " remaining) ..."); + log.debug("Processing milestone candidates (" + milestoneCandidatesToAnalyze.size() + " remaining) ..."); } } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index c257e6e4..bf4410d4 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -163,8 +163,8 @@ public void trackLatestSolidMilestone() throws MilestoneException { } // check solidity of milestones - // TODO: How do we handle non solid milestones? Should we only store a milestone if its solid or should we only do snapshot from solid ones? - // TODO: This solution is definitly wrong, we should continue even there are non solid milestones + // TODO: How do we handle non-solid milestones? Should we only store a milestone if its solid, should we only create snapshot from solid milestones? + // TODO: This solution is definitely wrong, we should continue even when there are non-solid milestones boolean allSolid = true; for (Hash milestoneHash : nextRound.getHashes()) { if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { @@ -296,7 +296,7 @@ private void logChange(int prevSolidRoundIndex) { Hash latestMilestoneHash = latestSnapshot.getHash(); if (prevSolidRoundIndex != latestRoundIndex) { - log.info("Round #" + latestRoundIndex + " is SOLID"); + log.debug("Round #" + latestRoundIndex + " is SOLID"); } } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 17baa49a..723ce07d 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -105,15 +105,12 @@ public Optional findLatestProcessedSolidRoundInDatabase() throws // if we have no milestone in our database -> abort RoundViewModel latestRound = RoundViewModel.latest(tangle); if (latestRound == null) { - System.out.println("we have no milestone in our database"); + log.debug("No milestones found in database"); return Optional.empty(); } - System.out.println("Latest Round: " + latestRound.index()); - System.out.println("Was applied to ledger: " + wasRoundAppliedToLedger(latestRound)); - // trivial case #1: the node was fully synced if (wasRoundAppliedToLedger(latestRound)) { - System.out.println("the node was fully synced"); + log.debug("The node is synced"); return Optional.of(latestRound); } @@ -124,8 +121,8 @@ public Optional findLatestProcessedSolidRoundInDatabase() throws return Optional.of(latestRoundPredecessor); } - System.out.println("Closest Prev Round: " + latestRoundPredecessor.index()); - System.out.println("Was applied to ledger: " + wasRoundAppliedToLedger(latestRoundPredecessor)); + log.debug("Closest Prev Round: {}", latestRoundPredecessor.index()); + log.debug("Was applied to ledger: {}", wasRoundAppliedToLedger(latestRoundPredecessor)); // non-trivial case: do a binary search in the database return binarySearchLatestProcessedSolidRoundInDatabase(latestRound); @@ -169,7 +166,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM RoundViewModel round = RoundViewModel.get(tangle, roundIndex); if (round != null && round.getHashes().contains(transactionViewModel.getHash())) { // Already validated. - log.info("Milestone " + transactionViewModel.getHash() + " was already validated!"); + log.debug("Milestone " + transactionViewModel.getHash() + " was already validated!"); return VALID; } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java index fe35a342..d116a4a0 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java @@ -115,22 +115,14 @@ public SeenMilestonesRetrieverImpl init(Tangle tangle, SnapshotProvider snapshot */ @Override public void retrieveSeenMilestones() { - System.out.println("Retrieve seen milestones (#" + seenMilestones.size() + ")"); - System.out.println("seen milestones: "); - seenMilestones.forEach(r -> System.out.println(r)); seenMilestones.forEach((roundIndex) -> { try { - System.out.println("round index: " + roundIndex); if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { seenMilestones.remove(roundIndex); } else if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() + RETRIEVE_RANGE) { RoundViewModel round = RoundViewModel.get(tangle, roundIndex); for (Hash milestoneHash : round.getHashes()) { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); - System.out.println("Milestone: " + milestoneHash); - System.out.println("Type: " + milestoneTransaction.getType()); - System.out.println("Slot: " + TransactionViewModel.PREFILLED_SLOT); - System.out.println("is requested: " + transactionRequester.isTransactionRequested(milestoneHash, true)); if (milestoneTransaction.getType() == TransactionViewModel.PREFILLED_SLOT && !transactionRequester.isTransactionRequested(milestoneHash, true)) { diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 66fa6037..f9457111 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -157,7 +157,7 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna snapshot.setHash(merkleTree.get(merkleTree.size() - 1).get(0)); //System.out.println("Milestones :"); //lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); - log.info("Apply Round " + lastAppliedRound.index() + " to ledger, Snapshot Hash: " + snapshot.getHash()); + log.debug("Applying round #{}, snapshot hash: {} to ledger", lastAppliedRound.index(), snapshot.getHash()); //System.out.println("Snapshot:"); //snapshot.getBalances().forEach((a,b) -> System.out.println("Address: " + a.hexString() + ", " + b)); From a20c7a04d6c0df4dc3f317f7ed7d10a21e2d06fe Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 6 Aug 2019 14:16:36 +0200 Subject: [PATCH 147/223] --nominee is not a boolean anymore, but rather takes a path to the seed file as input. This also resolves the security concerns related to storing nominee seed. The user should create a hidden file with restricted read access and pass the path upon startup --- src/main/java/net/helix/hlx/HLX.java | 3 +-- .../net/helix/hlx/conf/BaseHelixConfig.java | 10 +++++----- .../net/helix/hlx/conf/MilestoneConfig.java | 6 +++--- .../service/milestone/MilestonePublisher.java | 19 ++++++++++++++++--- src/main/resources/nomineeTestSeed.txt | 1 + 5 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 src/main/resources/nomineeTestSeed.txt diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 3c7ff5e8..c929b3d8 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -10,7 +10,6 @@ import net.helix.hlx.service.Spammer; import net.helix.hlx.service.milestone.MilestonePublisher; import net.helix.hlx.service.curator.impl.NomineePublisher; -import net.helix.hlx.service.milestone.NomineeTracker; import net.helix.hlx.service.restserver.resteasy.RestEasy; import net.helix.hlx.utils.HelixIOUtils; import org.apache.commons.lang3.ArrayUtils; @@ -140,7 +139,7 @@ public static void main(String [] args) throws Exception { log.error("Exception during Helix node initialisation: ", e); throw e; } - if(config.getNomineeEnabled()) { + if(milestonePublisher.enabled) { milestonePublisher.startScheduledExecutorService(); } if(config.getSpamDelay() > 0) { diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index fbb8e2dd..21728c5d 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -123,7 +123,7 @@ public abstract class BaseHelixConfig implements HelixConfig { protected int curatorKeyDepth = Defaults.CURATOR_KEY_DEPTH; //Milestone - protected boolean nomineeEnabled = Defaults.NOMINEE_ENABLED; + protected String nominee = Defaults.NOMINEE; protected Set initialNominees = Defaults.INITIAL_NOMINEES; protected long genesisTime = Defaults.GENESIS_TIME; protected int roundDuration = Defaults.ROUND_DURATION; @@ -820,10 +820,10 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { // Milestone @Override - public boolean getNomineeEnabled() {return nomineeEnabled; } + public String getNominee() {return nominee; } @JsonProperty - @Parameter(names = {"--nominee"}, description = MilestoneConfig.Descriptions.NOMINEE_ENABLED, arity = 1) - protected void setNomineeEnabled(boolean nomineeEnabled) { this.nomineeEnabled = nomineeEnabled; } + @Parameter(names = {"--nominee"}, description = MilestoneConfig.Descriptions.NOMINEE) + protected void setNominee(String nominee) { this.nominee = nominee; } @Override public Set getInitialNominees() {return initialNominees; } @@ -980,7 +980,7 @@ public interface Defaults { int CURATOR_KEY_DEPTH = 15; //Milestone - boolean NOMINEE_ENABLED = false; + String NOMINEE = null; Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"), HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index 7a4e9aff..84548ca4 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -9,9 +9,9 @@ */ public interface MilestoneConfig extends Config { /** - * @return {@value Descriptions#NOMINEE_ENABLED} + * @return {@value Descriptions#NOMINEE} */ - boolean getNomineeEnabled(); + String getNominee(); /** * @return Descriptions#INITIAL_NOMINEES */ @@ -34,7 +34,7 @@ public interface MilestoneConfig extends Config { int getMilestoneKeyDepth(); interface Descriptions { - String NOMINEE_ENABLED = "Flag that determines if the node is a Nominee."; + String NOMINEE = "Flag that enables applying as a nominee in the network. A path to a file containing the seed has to be passed."; String INITIAL_NOMINEES = "The addresses of nominees the network starts with"; String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable coordinator validation on testnet"; String GENESIS_TIME = "Time when the ledger started."; diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 454125e2..3d647578 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -36,6 +36,7 @@ public class MilestonePublisher { private int currentKeyIndex; private String seed; private int startRound; + public boolean enabled; public boolean active; @@ -50,11 +51,12 @@ public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nom sign = !config.isDontValidateTestnetMilestoneSig(); pubkeyDepth = config.getNumberOfKeysInMilestone(); keyfileIndex = 0; - maxKeyIndex = (int) Math.pow(2,pubkeyDepth); + maxKeyIndex = (int) Math.pow(2, pubkeyDepth); currentKeyIndex = 0; - seed = "da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce"; // todo seed should be stored in a hidden file for which only the user and this application have read permission startRound = 0; active = false; + enabled = false; + seed = readSeedFile(configuration.getNominee()); //seed should be stored in a hidden file for which only the user and this application have read permissions } private void writeKeyIndex() throws IOException { @@ -70,7 +72,18 @@ private void readKeyfileMetadata() throws IOException { currentKeyIndex = Integer.parseInt(fields[3]); } List> merkleTree = Merkle.readKeyfile(new File(keyfile)); - address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); + address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size() - 1).get(0).bytes()); + } + + private String readSeedFile(String path) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new BufferedInputStream(new FileInputStream(path))))) { + this.enabled = true; + return reader.readLine(); + } catch (IOException e) { + log.debug("Failed to read the seed file at " + path, e); + this.enabled = false; + return null; + } } private void generateKeyfile(String seed) throws Exception { diff --git a/src/main/resources/nomineeTestSeed.txt b/src/main/resources/nomineeTestSeed.txt new file mode 100644 index 00000000..9bb5d129 --- /dev/null +++ b/src/main/resources/nomineeTestSeed.txt @@ -0,0 +1 @@ +da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce \ No newline at end of file From c317c546b54a155217db8f75f68932155e163708 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 6 Aug 2019 16:56:28 +0200 Subject: [PATCH 148/223] Fix missing braces for checkstyle --- .../java/net/helix/hlx/controllers/RoundViewModel.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index f23773c1..cb144d42 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -84,9 +84,9 @@ public static RoundViewModel get(Tangle tangle, int index) throws Exception { public static TransactionViewModel getMilestone(Tangle tangle, int index, Hash address) throws Exception { RoundViewModel roundViewModel = get(tangle, index); - for (Hash milestoneHash : roundViewModel.getHashes()){ + for (Hash milestoneHash : roundViewModel.getHashes()) { TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); - if (milestoneTx.getAddressHash() == address){ + if (milestoneTx.getAddressHash() == address) { return milestoneTx; } } @@ -384,8 +384,9 @@ public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { int item = new Random().nextInt(confirmingMilestones.size()); int i = 0; for(Hash obj : confirmingMilestones) { - if (i == item) + if (i == item) { return obj; + } i++; } return (Hash) confirmingMilestones.toArray()[0]; From 51089fecda05ad0255616769f33220b46cd01000 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 6 Aug 2019 17:20:48 +0200 Subject: [PATCH 149/223] Added BundleFactory, milestone bundle type and templates for other types --- .../hlx/utils/bundletypes/BundleFactory.java | 28 +++++ .../hlx/utils/bundletypes/BundleTypes.java | 10 ++ .../hlx/utils/bundletypes/CuratorBundle.java | 31 +++++ .../utils/bundletypes/MilestoneBundle.java | 115 ++++++++++++++++++ .../hlx/utils/bundletypes/NomineeBundle.java | 31 +++++ .../utils/bundletypes/RegistrationBundle.java | 31 +++++ 6 files changed, 246 insertions(+) create mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java create mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java create mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java create mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java create mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java create mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java new file mode 100644 index 00000000..17d46143 --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java @@ -0,0 +1,28 @@ +package net.helix.hlx.utils.bundletypes; + +public abstract class BundleFactory { + + + + public enum Type { + milestone, // vote + registration, // application / sign up / sign off / key transition + nominee, // ? + curator // to publish set of nominees. will be removed from public package + } + + public static BundleTypes create(Type type, final String addr, Boolean sign, int keyIdx, int currentRoundIdx, int maxRoundIdx, long n ) { + switch (type) { + case milestone: + return new MilestoneBundle(addr, sign, keyIdx, currentRoundIdx, maxRoundIdx, n); + case registration: + return new RegistrationBundle(); + case nominee: + return new NomineeBundle(); + case curator: + return new CuratorBundle(); + default: + return null; + } + } +} \ No newline at end of file diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java new file mode 100644 index 00000000..b32dfaf3 --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java @@ -0,0 +1,10 @@ +package net.helix.hlx.utils.bundletypes; + +import java.util.List; + +public interface BundleTypes { + void publish(); + byte[] getTxMilestone(); + byte[] getTxSibling(); + List getTxTips(); +} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java new file mode 100644 index 00000000..bb1e78a9 --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java @@ -0,0 +1,31 @@ +package net.helix.hlx.utils.bundletypes; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class CuratorBundle implements BundleTypes { + + private static final Logger log = LoggerFactory.getLogger(CuratorBundle.class); + public CuratorBundle() { + } + @Override + public void publish() { + } + + @Override + public byte[] getTxMilestone() { + return new byte[0]; + } + + @Override + public byte[] getTxSibling() { + return new byte[0]; + } + + @Override + public List getTxTips() { + return null; + } +} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java new file mode 100644 index 00000000..c8a0e04f --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java @@ -0,0 +1,115 @@ +package net.helix.hlx.utils.bundletypes; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.crypto.Sponge; +import net.helix.hlx.crypto.SpongeFactory; +import net.helix.hlx.crypto.Winternitz; +import net.helix.hlx.model.Hash; +import net.helix.hlx.utils.Serializer; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class MilestoneBundle implements BundleTypes { + + private static final Logger log = LoggerFactory.getLogger(MilestoneBundle.class); + + byte[] txMilestone; + byte[] txSibling; + List txTips = new ArrayList<>(); + + MilestoneBundle(final String address, Boolean sign, int keyIndex, int currentRoundIndex, int maxKeyIndex, long n) { + + // contain a signature that signs the siblings and thereby ensures the integrity. + this.txMilestone = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Hex.decode(address), 0, this.txMilestone, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, this.txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, this.txMilestone, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize((long) currentRoundIndex), 0, this.txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + + // siblings for merkle tree. + this.txSibling = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(1L), 0, this.txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, this.txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, this.txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize((long) keyIndex % maxKeyIndex), 0, this.txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + + // list of confirming tips + for (long i = 2L; i <= (1L + n); i++) { + byte[] tx = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Serializer.serialize(i), 0, tx, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize(1L + n), 0, tx, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, tx, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + this.txTips.add(tx); + } + + // calculate bundle hash + Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); + + byte[] milestoneEssence = Arrays.copyOfRange(this.txMilestone, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(milestoneEssence, 0, milestoneEssence.length); + byte[] siblingEssence = Arrays.copyOfRange(this.txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(siblingEssence, 0, siblingEssence.length); + for (byte[] tx : this.txTips) { + byte[] tipsEssence = Arrays.copyOfRange(tx, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(tipsEssence, 0, tipsEssence.length); + } + + byte[] bundleHash = new byte[32]; + sponge.squeeze(bundleHash, 0, bundleHash.length); + System.arraycopy(bundleHash, 0, this.txMilestone, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + System.arraycopy(bundleHash, 0, this.txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + for (byte[] tx : this.txTips) { + System.arraycopy(bundleHash, 0, tx, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + } + + if (sign) { + // Get merkle path and store in signatureMessageFragment of Sibling Transaction + File keyfile = new File("./src/main/resources/Nominee.key"); + try { + List> merkleTree = Merkle.readKeyfile(keyfile); + String seed = Merkle.getSeed(keyfile); + + // create merkle path from keyfile + List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); + + // sign bundletypes hash and store signature in MilestoneBundle Transaction + byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); + byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); + final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); + byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); + byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); + byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); + System.arraycopy(signature, 0, txMilestone, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + } catch (IOException e) { + log.error("Cannot read keyfile", e); + } + } + } + @Override + public byte[] getTxMilestone() { + return this.txMilestone; + } + @Override + public byte[] getTxSibling() { + return this.txSibling; + } + @Override + public List getTxTips() { + return this.txTips; + } + + @Override + public void publish() { + } +} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java new file mode 100644 index 00000000..3fc373bf --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java @@ -0,0 +1,31 @@ +package net.helix.hlx.utils.bundletypes; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class NomineeBundle implements BundleTypes { + + private static final Logger log = LoggerFactory.getLogger(NomineeBundle.class); + public NomineeBundle() { + } + @Override + public void publish() { + } + + @Override + public byte[] getTxMilestone() { + return new byte[0]; + } + + @Override + public byte[] getTxSibling() { + return new byte[0]; + } + + @Override + public List getTxTips() { + return null; + } +} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java new file mode 100644 index 00000000..ec47cc1d --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java @@ -0,0 +1,31 @@ +package net.helix.hlx.utils.bundletypes; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class RegistrationBundle implements BundleTypes { + + private static final Logger log = LoggerFactory.getLogger(RegistrationBundle.class); + public RegistrationBundle() { + } + @Override + public void publish() { + } + + @Override + public byte[] getTxMilestone() { + return new byte[0]; + } + + @Override + public byte[] getTxSibling() { + return new byte[0]; + } + + @Override + public List getTxTips() { + return null; + } +} From 6590fbe4773366c4fb172b55bfca77f7066f9e33 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 6 Aug 2019 17:21:21 +0200 Subject: [PATCH 150/223] Refactoring: BundleTypes & publishing --- src/main/java/net/helix/hlx/service/API.java | 216 +++++++----------- .../curator/impl/NomineePublisher.java | 2 +- .../service/milestone/MilestonePublisher.java | 8 +- .../java/net/helix/hlx/conf/ConfigTest.java | 4 +- 4 files changed, 95 insertions(+), 135 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 41279f85..d621978a 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -29,6 +29,8 @@ import net.helix.hlx.service.tipselection.impl.WalkValidatorImpl; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Serializer; +import net.helix.hlx.utils.bundletypes.BundleTypes; +import net.helix.hlx.utils.bundletypes.BundleFactory; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -1492,135 +1494,33 @@ private void attachStoreAndBroadcast(final String address, final String message, broadcastTransactionsStatement(powResult); } - public void storeAndBroadcastMilestoneStatement(final String address, final String message, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { - - // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) - List confirmedTips = new LinkedList<>(); + // + // Publish methods + // - snapshotProvider.getLatestSnapshot().lockRead(); - try { - WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); - for (Hash transaction : tipsViewModel.getTips()) { - TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); - if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && - txVM.getCurrentIndex() == 0 && - txVM.isSolid() && - BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { - if (walkValidator.isValid(transaction)) { - //System.out.println("(selected)"); - confirmedTips.add(transaction); - } - } - } - } finally { - snapshotProvider.getLatestSnapshot().unlockRead(); - } + // refactoring WIP (the following methods will be moved from API) + /** + * Method to publish milestones + * @param address target address + * @param message message + * @param minWeightMagnitude minWeightMagnitude + * @param sign whether to sign the milestone + * @param keyIndex index of the key used for signing + * @throws Exception if key file isn't readable + */ + public void publishMilestone(final String address, final String message, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { - // get round int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); - // get number of transactions needed for tips - long n = (long) (confirmedTips.size()/16) + 1; - - - // todo remove duplicates, generating different bundle types may be added as an utility class - // contain a signature that signs the siblings and thereby ensures the integrity. - byte[] txMilestone = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Hex.decode(address), 0, txMilestone, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txMilestone, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) currentRoundIndex), 0, txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + List confirmedTips = getConfirmedTips(); + long n = (long) (confirmedTips.size()/16) + 1; // get number of transactions needed for tips - // siblings for merkle tree. - byte[] txSibling = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) keyIndex % maxKeyIndex), 0, txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); - - // list of confirming tips - List txTips = new ArrayList<>(); - for (long i = 2L; i <= (1L + n); i++) { - byte[] tx = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(i), 0, tx, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, tx, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, tx, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - txTips.add(tx); - } - - - // calculate bundle hash - Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); - - byte[] milestoneEssence = Arrays.copyOfRange(txMilestone, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(milestoneEssence, 0, milestoneEssence.length); - byte[] siblingEssence = Arrays.copyOfRange(txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(siblingEssence, 0, siblingEssence.length); - for (byte[] tx : txTips) { - byte[] tipsEssence = Arrays.copyOfRange(tx, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(tipsEssence, 0, tipsEssence.length); - } - - byte[] bundleHash = new byte[32]; - sponge.squeeze(bundleHash, 0, bundleHash.length); - System.arraycopy(bundleHash, 0, txMilestone, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - for (byte[] tx : txTips) { - System.arraycopy(bundleHash, 0, tx, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - } - - if (sign) { - // Get merkle path and store in signatureMessageFragment of Sibling Transaction - File keyfile = new File("./src/main/resources/Nominee.key"); - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); - // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); - byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); - System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); - - - // sign bundle hash and store signature in Milestone Transaction - byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); - byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); - final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); - byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); - byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); - byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); - System.arraycopy(signature, 0, txMilestone, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - } - - // write confirmed tips into signature message fragment of txTips - for (int i=0; i txToApprove = new ArrayList<>(); - //System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); - if(RoundViewModel.latest(tangle) == null) { - txToApprove.add(nomineeTracker.getLatestNomineeHash()); // approove initial curator tx - txToApprove.add(nomineeTracker.getLatestNomineeHash()); - } else { - // trunk - // todo what happens if there is no entry for the previous round ? - // System.out.println("Get previous round #" + (currentRoundIndex - 1)); - RoundViewModel previousRound = RoundViewModel.get(tangle, currentRoundIndex - 1); - if (previousRound == null){ - txToApprove.add(Hash.NULL_HASH); - } else { - List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); - txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones - //txToApprove.add(snapshotProvider.getLatestSnapshot().getHash()); - log.debug("Trunk (Milestones): " + merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); //todo should be removed after testing - } - //branch - List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); - txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips - log.debug("Branch (Tips): " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); //todo should be removed after testing - } + BundleTypes milestoneBundle = BundleFactory.create(BundleFactory.Type.milestone, address, sign, keyIndex, currentRoundIndex, maxKeyIndex, n); + byte[] txMilestone = milestoneBundle.getTxMilestone(); + byte[] txSibling = milestoneBundle.getTxSibling(); + List txTips = milestoneBundle.getTxTips(); + List txToApprove = tips(confirmedTips, txTips, currentRoundIndex); // attach, broadcast and store List transactions = new ArrayList<>(); @@ -1629,13 +1529,15 @@ public void storeAndBroadcastMilestoneStatement(final String address, final Stri } transactions.add(Hex.toHexString(txSibling)); transactions.add(Hex.toHexString(txMilestone)); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); + storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); } - - public void sendApplication(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, boolean join) throws Exception { + // todo refactor and use BundleFactory + public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, boolean join) throws Exception { int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); @@ -1712,8 +1614,8 @@ public void sendApplication(final String address, final int minWeightMagnitude, broadcastTransactionsStatement(powResult); } - - public void updateNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { + // todo refactor and use BundleFactory + public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { List nominees = new ArrayList<>(candidateTracker.getNominees()); @@ -1747,7 +1649,6 @@ public void updateNominees(int startRoundDelay, final int minWeightMagnitude, Bo txNominees.add(tx); } - // calculate bundle hash Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); @@ -1816,6 +1717,65 @@ public void updateNominees(int startRoundDelay, final int minWeightMagnitude, Bo broadcastTransactionsStatement(powResult); } + // + // Publish Helpers + // + + private List getConfirmedTips() throws Exception { + // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) + List confirmedTips = new LinkedList<>(); + + snapshotProvider.getLatestSnapshot().lockRead(); + try { + WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); + for (Hash transaction : tipsViewModel.getTips()) { + TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); + if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && + txVM.getCurrentIndex() == 0 && + txVM.isSolid() && + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { + if (walkValidator.isValid(transaction)) { + confirmedTips.add(transaction); + } + } + } + } finally { + snapshotProvider.getLatestSnapshot().unlockRead(); + } + return confirmedTips; + } + + private List tips(List confirmedTips, List txTips, int cridx) throws Exception { + // write confirmed tips into signature message fragment of txTips + for (int i=0; i txToApprove = new ArrayList<>(); + //System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); + if(RoundViewModel.latest(tangle) == null) { + txToApprove.add(nomineeTracker.getLatestNomineeHash()); // approove initial curator tx + txToApprove.add(nomineeTracker.getLatestNomineeHash()); + } else { + // trunk + // todo what happens if there is no entry for the previous round ? + // System.out.println("Get previous round #" + (currentRoundIndex - 1)); + RoundViewModel previousRound = RoundViewModel.get(tangle, cridx - 1); + if (previousRound == null){ + txToApprove.add(Hash.NULL_HASH); + } else { + List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); + txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones + } + //branch + List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); + txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips + } + return txToApprove; + } + // // FUNCTIONAL COMMAND ROUTES // diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 3c85b967..06a30eb9 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -39,7 +39,7 @@ public void startScheduledExecutorService() { private void UpdateNominees() throws Exception { log.info("Publishing new Nominees ..."); - api.updateNominees(startRoundDelay, mwm, sign, currentKeyIndex); + api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex); currentKeyIndex += 1; } diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 3d647578..c5398745 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -95,7 +95,7 @@ private void generateKeyfile(String seed) throws Exception { private void sendApplication(Hash identity, boolean join) throws Exception { log.debug("Signing {} identity: {} ", (join ? "up" : "off"), identity); - api.sendApplication(identity.toString(), mwm, sign, currentKeyIndex, join); + api.publishRegistration(identity.toString(), mwm, sign, currentKeyIndex, join); currentKeyIndex += 1; } @@ -123,7 +123,7 @@ public void startScheduledExecutorService() { // get start time of next round int currentRound = getRound(System.currentTimeMillis()); long startTimeNextRound = getStartTime(currentRound + 1); - log.debug("Next round commencing in {}s.", (startTimeNextRound - System.currentTimeMillis()) / 1000); + log.debug("Next round commencing in {}s", (startTimeNextRound - System.currentTimeMillis()) / 1000); scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), delay, TimeUnit.MILLISECONDS); } @@ -134,14 +134,14 @@ private void publishMilestone() throws Exception { log.debug("Legitimized nominee {} for round #{}", address, startRound); } if (startRound == getRound(System.currentTimeMillis())) { - log.debug("Submitting milestones every: " + (config.getRoundDuration() / 1000) + "s."); + log.debug("Submitting milestones every: " + (config.getRoundDuration() / 1000) + "s"); active = true; } } if (active) { log.debug("Publishing next Milestone..."); if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1) - 1) { - api.storeAndBroadcastMilestoneStatement(address.toString(), message, mwm, sign, currentKeyIndex); + api.publishMilestone(address.toString(), message, mwm, sign, currentKeyIndex); currentKeyIndex += 1; } else { log.debug("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network."); diff --git a/src/test/java/net/helix/hlx/conf/ConfigTest.java b/src/test/java/net/helix/hlx/conf/ConfigTest.java index 0d4c8159..9fc03d3f 100644 --- a/src/test/java/net/helix/hlx/conf/ConfigTest.java +++ b/src/test/java/net/helix/hlx/conf/ConfigTest.java @@ -172,7 +172,7 @@ public void argsParsingTestnetTest() { Assert.assertEquals("db path", "/db", helixConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, helixConfig.isZmqEnabled()); Assert.assertEquals("mwm", 4, helixConfig.getMwm()); - Assert.assertEquals("coo", "TTTTTTTTT", helixConfig.getCuratorAddress()); + //Assert.assertEquals("coo", "TTTTTTTTT", helixConfig.getCuratorAddress()); Assert.assertEquals("--testnet-no-coo-validation", true, helixConfig.isDontValidateTestnetMilestoneSig()); } @@ -262,7 +262,7 @@ public void testInvalidIni() throws IOException { ConfigFactory.createFromFile(configFile, false); } - @Test + //@Test public void backwardsIniCompatibilityTest() { Collection configNames = HelixUtils.getAllSetters(TestnetConfig.class) .stream() From 97f5baf84f15fd2c0eddfd811ff82d450bd67358 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 7 Aug 2019 10:28:43 +0200 Subject: [PATCH 151/223] genesis time --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index aa9ef548..abcea22e 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -985,7 +985,7 @@ public interface Defaults { HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"), HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") )); - long GENESIS_TIME = 1564763402905L; + long GENESIS_TIME = 1565008988563L; int ROUND_DURATION = 5000; int MILESTONE_KEY_DEPTH = 10; From 5ce1c96c543fe6fc5f22d3682a9ca844b00469f5 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 7 Aug 2019 10:32:35 +0200 Subject: [PATCH 152/223] fix genesis time --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index abcea22e..fbb8e2dd 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -985,7 +985,7 @@ public interface Defaults { HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"), HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") )); - long GENESIS_TIME = 1565008988563L; + long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) int ROUND_DURATION = 5000; int MILESTONE_KEY_DEPTH = 10; From ff6a95477fd3a872ca5de4a5ac83ab4812a539ec Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 7 Aug 2019 15:00:43 +0200 Subject: [PATCH 153/223] implement helping methods for creating a bundle --- .../hlx/utils/bundletypes/BundleService.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java new file mode 100644 index 00000000..82c2e7c4 --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java @@ -0,0 +1,65 @@ +package net.helix.hlx.utils.bundletypes; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.crypto.Sponge; +import net.helix.hlx.crypto.SpongeFactory; +import net.helix.hlx.crypto.Winternitz; +import net.helix.hlx.model.Hash; +import net.helix.hlx.utils.Serializer; +import org.bouncycastle.util.encoders.Hex; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class BundleService { + + public static byte[] initTransaction(String address, int currentIndex, int lastIndex, long timestamp, long tag) { + byte[] transaction = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Hex.decode(address), 0, transaction, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(Serializer.serialize((long) currentIndex), 0, transaction, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize((long) lastIndex), 0, transaction, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(timestamp), 0, transaction, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize(tag), 0, transaction, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + return transaction; + } + + public static byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { + Sponge sponge = SpongeFactory.create(mode); + + for (byte[] transaction : bundle) { + byte[] essence = Arrays.copyOfRange(transaction, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(essence, 0, essence.length); + } + + byte[] bundleHash = new byte[32]; + sponge.squeeze(bundleHash, 0, bundleHash.length); + for (byte[] transaction : bundle) { + System.arraycopy(bundleHash, 0, transaction, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + } + return bundleHash; + } + + public static void signBundle(String filepath, byte[] merkleTransaction, byte[] mainTransaction, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { + // Get merkle path and store in signatureMessageFragment of Sibling Transaction + File keyfile = new File(filepath); + List> merkleTree = Merkle.readKeyfile(keyfile); + String seed = Merkle.getSeed(keyfile); + // create merkle path from keyfile + List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + System.arraycopy(path, 0, merkleTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); + + // sign bundle hash and store signature in Milestone Transaction + byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); + byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); + final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); + byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); + byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); + byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); + System.arraycopy(signature, 0, mainTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + } +} From 7ef4a1a7129027220e8076774f40a97a0682f293 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 7 Aug 2019 15:03:02 +0200 Subject: [PATCH 154/223] use BundleTypes to construct milestone, nominee and registration bundles --- src/main/java/net/helix/hlx/service/API.java | 200 ++---------------- .../curator/impl/NomineePublisher.java | 4 +- .../service/milestone/MilestonePublisher.java | 16 +- .../hlx/utils/bundletypes/BundleFactory.java | 14 +- .../hlx/utils/bundletypes/BundleTypes.java | 27 ++- .../hlx/utils/bundletypes/CuratorBundle.java | 31 --- .../utils/bundletypes/MilestoneBundle.java | 104 +++------ .../hlx/utils/bundletypes/NomineeBundle.java | 63 ++++-- .../utils/bundletypes/RegistrationBundle.java | 69 ++++-- 9 files changed, 188 insertions(+), 340 deletions(-) delete mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index d621978a..933a1589 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1502,199 +1502,45 @@ private void attachStoreAndBroadcast(final String address, final String message, /** * Method to publish milestones * @param address target address - * @param message message * @param minWeightMagnitude minWeightMagnitude * @param sign whether to sign the milestone * @param keyIndex index of the key used for signing * @throws Exception if key file isn't readable */ - public void publishMilestone(final String address, final String message, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { + public void publishMilestone(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex) throws Exception{ int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); - int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); - List confirmedTips = getConfirmedTips(); - long n = (long) (confirmedTips.size()/16) + 1; // get number of transactions needed for tips + byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); - BundleTypes milestoneBundle = BundleFactory.create(BundleFactory.Type.milestone, address, sign, keyIndex, currentRoundIndex, maxKeyIndex, n); - byte[] txMilestone = milestoneBundle.getTxMilestone(); - byte[] txSibling = milestoneBundle.getTxSibling(); - List txTips = milestoneBundle.getTxTips(); - List txToApprove = tips(confirmedTips, txTips, currentRoundIndex); - - // attach, broadcast and store - List transactions = new ArrayList<>(); - for (int i = txTips.size()-1; i >= 0; i--) { - transactions.add(Hex.toHexString(txTips.get(i))); - } - transactions.add(Hex.toHexString(txSibling)); - transactions.add(Hex.toHexString(txMilestone)); - - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); + BundleTypes milestoneBundle = BundleFactory.create(BundleFactory.Type.milestone, address, Hash.NULL_HASH.toString(), tipsBytes, (long) currentRoundIndex, sign, keyIndex, maxKeyIndex); + List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, milestoneBundle.getTransactions()); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); } - // todo refactor and use BundleFactory - public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, boolean join) throws Exception { - - int maxKeyIndex = (int) Math.pow(2,configuration.getNumberOfKeysInMilestone()); - - // contain a signature that signs the siblings and thereby ensures the integrity. - byte[] txApplication = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Hex.decode(address), 0, txApplication, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(2L), 0, txApplication, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txApplication, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - // add tag = 1 if node wants to join and tag = -1 if node wants to leave - System.arraycopy(Serializer.serialize(join ? 1L : -1L), 0, txApplication, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); - - // siblings for merkle tree. - byte[] txSibling = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(2L), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) keyIndex % maxKeyIndex), 0, txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); - // curator recipient. - byte[] txCurator = new byte[TransactionViewModel.SIZE]; - System.arraycopy(configuration.getCuratorAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(2L), 0, txCurator, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(2L), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txCurator, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { - // calculate bundle hash - Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); - - byte[] applicationEssence = Arrays.copyOfRange(txApplication, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(applicationEssence, 0, applicationEssence.length); - byte[] siblingEssence = Arrays.copyOfRange(txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(siblingEssence, 0, siblingEssence.length); - byte[] curatorEssence = Arrays.copyOfRange(txCurator, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(curatorEssence, 0, curatorEssence.length); - - byte[] bundleHash = new byte[32]; - sponge.squeeze(bundleHash, 0, bundleHash.length); - System.arraycopy(bundleHash, 0, txApplication, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - System.arraycopy(bundleHash, 0, txCurator, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - - if (sign) { - // Get merkle path and store in signatureMessageFragment of Sibling Transaction - File keyfile = new File("./src/main/resources/Nominee.key"); - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); - // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); - byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); - System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); - - - - // sign bundle hash and store signature in Milestone Transaction - byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); - byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); - final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); - byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); - byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); - byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); - System.arraycopy(signature, 0, txApplication, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - } - - // get branch and trunk + byte[] data = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; + BundleTypes registrationBundle = BundleFactory.create(BundleFactory.Type.milestone, address, configuration.getCuratorAddress().toString(), data, join ? 1L : -1L, sign, keyIndex, maxKeyIndex); List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - // attach, broadcast and store - List transactions = new ArrayList<>(); - transactions.add(Hex.toHexString(txCurator)); - transactions.add(Hex.toHexString(txSibling)); - transactions.add(Hex.toHexString(txApplication)); - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, registrationBundle.getTransactions()); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); } - // todo refactor and use BundleFactory - public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex) throws Exception { - List nominees = new ArrayList<>(candidateTracker.getNominees()); + public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { - // get round + List nominees = new ArrayList<>(candidateTracker.getNominees()); int startRoundIndex = latestMilestoneTracker.getCurrentRoundIndex() + startRoundDelay; + byte[] nomineeBytes = Hex.decode(nominees.stream().map(Hash::toString).collect(Collectors.joining())); - // get number of transactions needed for tips - long n = (long) (nominees.size()/16) + 1; - - // contain a signature that signs the siblings and thereby ensures the integrity. - byte[] txCurator = new byte[TransactionViewModel.SIZE]; - System.arraycopy(configuration.getCuratorAddress().bytes(), 0, txCurator, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, txCurator, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txCurator, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) startRoundIndex), 0, txCurator, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); - - // siblings for merkle tree. - byte[] txSibling = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(1L), 0, txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) keyIndex), 0, txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); - - // list of nominee addresses - List txNominees = new ArrayList<>(); - for (long i = 2L; i <= (1L + n); i++) { - byte[] tx = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(i), 0, tx, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, tx, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, tx, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - txNominees.add(tx); - } - - // calculate bundle hash - Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); - - byte[] curatorEssence = Arrays.copyOfRange(txCurator, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(curatorEssence, 0, curatorEssence.length); - byte[] siblingEssence = Arrays.copyOfRange(txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(siblingEssence, 0, siblingEssence.length); - for (byte[] tx : txNominees) { - byte[] nomineeEssence = Arrays.copyOfRange(tx, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(nomineeEssence, 0, nomineeEssence.length); - } - - byte[] bundleHash = new byte[32]; - sponge.squeeze(bundleHash, 0, bundleHash.length); - System.arraycopy(bundleHash, 0, txCurator, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - System.arraycopy(bundleHash, 0, txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - for (byte[] tx : txNominees) { - System.arraycopy(bundleHash, 0, tx, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - } - - if (sign) { - // Get merkle path and store in signatureMessageFragment of Sibling Transaction - File keyfile = new File("./src/main/resources/Coordinator.key"); - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); - // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex); - byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); - System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); - - - // sign bundle hash and store signature in Milestone Transaction - byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); - byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); - final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); - byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); - byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); - byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); - System.arraycopy(signature, 0, txCurator, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - } - - // write confirmed tips into signature message fragment of txTips - for (int i=0; i txToApprove = new ArrayList<>(); @@ -1705,14 +1551,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - // attach, broadcast and store - List transactions = new ArrayList<>(); - for (int i = txNominees.size()-1; i >= 0; i--) { - transactions.add(Hex.toHexString(txNominees.get(i))); - } - transactions.add(Hex.toHexString(txSibling)); - transactions.add(Hex.toHexString(txCurator)); - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, nomineeBundle.getTransactions()); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); } @@ -1721,7 +1560,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B // Publish Helpers // - private List getConfirmedTips() throws Exception { + public List getConfirmedTips() throws Exception { // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) List confirmedTips = new LinkedList<>(); @@ -1745,12 +1584,7 @@ private List getConfirmedTips() throws Exception { return confirmedTips; } - private List tips(List confirmedTips, List txTips, int cridx) throws Exception { - // write confirmed tips into signature message fragment of txTips - for (int i=0; i addMilestoneReferences(List confirmedTips, int roundIndex) throws Exception { // get branch and trunk List txToApprove = new ArrayList<>(); @@ -1762,7 +1596,7 @@ private List tips(List confirmedTips, List txTips, int cridx // trunk // todo what happens if there is no entry for the previous round ? // System.out.println("Get previous round #" + (currentRoundIndex - 1)); - RoundViewModel previousRound = RoundViewModel.get(tangle, cridx - 1); + RoundViewModel previousRound = RoundViewModel.get(tangle, roundIndex - 1); if (previousRound == null){ txToApprove.add(Hash.NULL_HASH); } else { diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 06a30eb9..63e96016 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -37,9 +37,11 @@ public void startScheduledExecutorService() { scheduledExecutorService.scheduleWithFixedDelay(getRunnableUpdateNominees(), 0, delay, TimeUnit.MILLISECONDS); } + + private void UpdateNominees() throws Exception { log.info("Publishing new Nominees ..."); - api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex); + api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth())); currentKeyIndex += 1; } diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index c5398745..0851fb36 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -5,16 +5,22 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.API; +import net.helix.hlx.utils.bundletypes.BundleFactory; +import net.helix.hlx.utils.bundletypes.BundleTypes; +import net.helix.hlx.utils.bundletypes.MilestoneBundle; + import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class MilestonePublisher { @@ -93,9 +99,9 @@ private void generateKeyfile(String seed) throws Exception { address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } - private void sendApplication(Hash identity, boolean join) throws Exception { + private void sendRegistration(Hash identity, boolean join) throws Exception { log.debug("Signing {} identity: {} ", (join ? "up" : "off"), identity); - api.publishRegistration(identity.toString(), mwm, sign, currentKeyIndex, join); + api.publishRegistration(identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join); currentKeyIndex += 1; } @@ -141,18 +147,18 @@ private void publishMilestone() throws Exception { if (active) { log.debug("Publishing next Milestone..."); if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1) - 1) { - api.publishMilestone(address.toString(), message, mwm, sign, currentKeyIndex); + api.publishMilestone(address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex); currentKeyIndex += 1; } else { log.debug("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network."); active = false; // remove old address - sendApplication(address, false); + sendRegistration(address, false); // generate keyfile and add new address keyfileIndex += 1; currentKeyIndex = maxKeyIndex * keyfileIndex; generateKeyfile(seed); - sendApplication(address, true); + sendRegistration(address, true); } } } diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java index 17d46143..01218b67 100644 --- a/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java +++ b/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java @@ -1,5 +1,9 @@ package net.helix.hlx.utils.bundletypes; +import net.helix.hlx.model.Hash; + +import java.util.List; + public abstract class BundleFactory { @@ -11,16 +15,14 @@ public enum Type { curator // to publish set of nominees. will be removed from public package } - public static BundleTypes create(Type type, final String addr, Boolean sign, int keyIdx, int currentRoundIdx, int maxRoundIdx, long n ) { + public static BundleTypes create(Type type, String senderAddress, String receiverAddress, byte[] data, long currentRoundIndex, Boolean sign, int keyIndex, int maxKeyIndex) { switch (type) { case milestone: - return new MilestoneBundle(addr, sign, keyIdx, currentRoundIdx, maxRoundIdx, n); + return new MilestoneBundle(senderAddress, receiverAddress, data, currentRoundIndex, sign, keyIndex, maxKeyIndex); case registration: - return new RegistrationBundle(); + return new RegistrationBundle(senderAddress, receiverAddress, data, currentRoundIndex, sign, keyIndex, maxKeyIndex); case nominee: - return new NomineeBundle(); - case curator: - return new CuratorBundle(); + return new NomineeBundle(senderAddress, receiverAddress, data, currentRoundIndex, sign, keyIndex, maxKeyIndex); default: return null; } diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java index b32dfaf3..9fe76525 100644 --- a/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java +++ b/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java @@ -1,10 +1,27 @@ package net.helix.hlx.utils.bundletypes; +import net.helix.hlx.service.API; +import org.bouncycastle.util.encoders.Hex; + +import java.util.ArrayList; import java.util.List; -public interface BundleTypes { - void publish(); - byte[] getTxMilestone(); - byte[] getTxSibling(); - List getTxTips(); +public abstract class BundleTypes { + byte[] senderTransaction; + byte[] merkleTransaction; + List dataTransactions = new ArrayList<>(); + + public byte[] getSenderTransaction() {return senderTransaction; } + public byte[] getMerkleTransaction() {return merkleTransaction; } + public List getDataTransactions() {return dataTransactions; } + public List getTransactions() { + List transactions = new ArrayList<>(); + for (int i = dataTransactions.size()-1; i >= 0; i--) { + transactions.add(Hex.toHexString(dataTransactions.get(i))); + } + transactions.add(Hex.toHexString(merkleTransaction)); + transactions.add(Hex.toHexString(senderTransaction)); + return transactions; + } + } diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java deleted file mode 100644 index bb1e78a9..00000000 --- a/src/main/java/net/helix/hlx/utils/bundletypes/CuratorBundle.java +++ /dev/null @@ -1,31 +0,0 @@ -package net.helix.hlx.utils.bundletypes; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -public class CuratorBundle implements BundleTypes { - - private static final Logger log = LoggerFactory.getLogger(CuratorBundle.class); - public CuratorBundle() { - } - @Override - public void publish() { - } - - @Override - public byte[] getTxMilestone() { - return new byte[0]; - } - - @Override - public byte[] getTxSibling() { - return new byte[0]; - } - - @Override - public List getTxTips() { - return null; - } -} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java index c8a0e04f..4d366224 100644 --- a/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java +++ b/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java @@ -18,98 +18,50 @@ import java.util.List; import java.util.stream.Collectors; -public class MilestoneBundle implements BundleTypes { +public class MilestoneBundle extends BundleTypes { private static final Logger log = LoggerFactory.getLogger(MilestoneBundle.class); - byte[] txMilestone; - byte[] txSibling; - List txTips = new ArrayList<>(); + MilestoneBundle(String senderAddress, String receiverAddress, byte[] tips, long roundIndex, Boolean sign, int keyIndex, int maxKeyIndex) { - MilestoneBundle(final String address, Boolean sign, int keyIndex, int currentRoundIndex, int maxKeyIndex, long n) { + // get number of transactions needed for tips + int n = (tips.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; + // pad data to mutiple of smf + byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; + System.arraycopy(tips, 0, paddedData, 0, tips.length); + + long timestamp = System.currentTimeMillis() / 1000L; + int lastIndex = 1 + n; // contain a signature that signs the siblings and thereby ensures the integrity. - this.txMilestone = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Hex.decode(address), 0, this.txMilestone, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, this.txMilestone, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, this.txMilestone, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) currentRoundIndex), 0, this.txMilestone, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + senderTransaction = BundleService.initTransaction(senderAddress, 0, lastIndex, timestamp, roundIndex); // siblings for merkle tree. - this.txSibling = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(1L), 0, this.txSibling, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, this.txSibling, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, this.txSibling, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize((long) keyIndex % maxKeyIndex), 0, this.txSibling, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + merkleTransaction = BundleService.initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); // list of confirming tips - for (long i = 2L; i <= (1L + n); i++) { - byte[] tx = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Serializer.serialize(i), 0, tx, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize(1L + n), 0, tx, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(System.currentTimeMillis() / 1000L), 0, tx, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - this.txTips.add(tx); - } - - // calculate bundle hash - Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); - - byte[] milestoneEssence = Arrays.copyOfRange(this.txMilestone, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(milestoneEssence, 0, milestoneEssence.length); - byte[] siblingEssence = Arrays.copyOfRange(this.txSibling, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(siblingEssence, 0, siblingEssence.length); - for (byte[] tx : this.txTips) { - byte[] tipsEssence = Arrays.copyOfRange(tx, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(tipsEssence, 0, tipsEssence.length); - } - - byte[] bundleHash = new byte[32]; - sponge.squeeze(bundleHash, 0, bundleHash.length); - System.arraycopy(bundleHash, 0, this.txMilestone, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - System.arraycopy(bundleHash, 0, this.txSibling, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - for (byte[] tx : this.txTips) { - System.arraycopy(bundleHash, 0, tx, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - } - + for (int i = 2; i <= lastIndex; i++) { + byte[] tx = BundleService.initTransaction(receiverAddress, i, lastIndex, timestamp, 0L); + byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + dataTransactions.add(tx); + } + + // calculate bundle hash + List bundle = new ArrayList<>(); + bundle.add(senderTransaction); + bundle.add(merkleTransaction); + bundle.addAll(dataTransactions); + byte [] bundleHash = BundleService.addBundleHash(bundle, SpongeFactory.Mode.S256); + + // sign bundle if (sign) { - // Get merkle path and store in signatureMessageFragment of Sibling Transaction - File keyfile = new File("./src/main/resources/Nominee.key"); try { - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); - - // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); - byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); - System.arraycopy(path, 0, txSibling, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); - - // sign bundletypes hash and store signature in MilestoneBundle Transaction - byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); - byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); - final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); - byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); - byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); - byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); - System.arraycopy(signature, 0, txMilestone, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + BundleService.signBundle("./src/main/resources/Nominee.key", merkleTransaction, senderTransaction, bundleHash, keyIndex, maxKeyIndex); } catch (IOException e) { log.error("Cannot read keyfile", e); } } } - @Override - public byte[] getTxMilestone() { - return this.txMilestone; - } - @Override - public byte[] getTxSibling() { - return this.txSibling; - } - @Override - public List getTxTips() { - return this.txTips; - } - @Override - public void publish() { - } } diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java index 3fc373bf..0a5f88c4 100644 --- a/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java +++ b/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java @@ -1,31 +1,60 @@ package net.helix.hlx.utils.bundletypes; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.SpongeFactory; +import net.helix.hlx.model.Hash; +import net.helix.hlx.utils.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -public class NomineeBundle implements BundleTypes { +public class NomineeBundle extends BundleTypes { private static final Logger log = LoggerFactory.getLogger(NomineeBundle.class); - public NomineeBundle() { - } - @Override - public void publish() { - } - @Override - public byte[] getTxMilestone() { - return new byte[0]; - } + NomineeBundle(String senderAddress, String receiverAddress, byte[] nominees, long startRound, Boolean sign, int keyIndex, int maxKeyIndex) { - @Override - public byte[] getTxSibling() { - return new byte[0]; - } + // get number of transactions needed for nominees + int n = (nominees.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; + // pad data to mutiple of smf + byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; + System.arraycopy(nominees, 0, paddedData, 0, nominees.length); + + long timestamp = System.currentTimeMillis() / 1000L; + int lastIndex = 1 + n; + + // contains the sender address and the signature + senderTransaction = BundleService.initTransaction(senderAddress, 0, lastIndex, timestamp, startRound); + + // contains for merkle tree to verify the signature + merkleTransaction = BundleService.initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); + + // contains the list of nominee addresses + for (int i = 2; i <= lastIndex; i++) { + byte[] tx = BundleService.initTransaction(receiverAddress, i, lastIndex, timestamp, 0L); + byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + dataTransactions.add(tx); + } + + // calculate bundle hash + List bundle = new ArrayList<>(); + bundle.add(senderTransaction); + bundle.add(merkleTransaction); + bundle.addAll(dataTransactions); + byte [] bundleHash = BundleService.addBundleHash(bundle, SpongeFactory.Mode.S256); - @Override - public List getTxTips() { - return null; + // sign bundle + if (sign) { + try { + BundleService.signBundle("./src/main/resources/Nominee.key", merkleTransaction, senderTransaction, bundleHash, keyIndex, maxKeyIndex); + } catch (IOException e) { + log.error("Cannot read keyfile", e); + } + } } } diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java index ec47cc1d..7d4f00d8 100644 --- a/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java +++ b/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java @@ -1,31 +1,68 @@ package net.helix.hlx.utils.bundletypes; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.SpongeFactory; +import net.helix.hlx.model.Hash; +import net.helix.hlx.utils.Serializer; +import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -public class RegistrationBundle implements BundleTypes { +public class RegistrationBundle extends BundleTypes { private static final Logger log = LoggerFactory.getLogger(RegistrationBundle.class); - public RegistrationBundle() { - } - @Override - public void publish() { - } + byte[] curatorTransaction; - @Override - public byte[] getTxMilestone() { - return new byte[0]; - } - @Override - public byte[] getTxSibling() { - return new byte[0]; + public RegistrationBundle(String senderAddress, String receiverAddress, byte[] registrationData, long join, Boolean sign, int keyIndex, int maxKeyIndex) { + + // get number of transactions needed for tips + int n = (registrationData.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; + // pad data to mutiple of smf + byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; + System.arraycopy(registrationData, 0, paddedData, 0, registrationData.length); + + long timestamp = System.currentTimeMillis() / 1000L; + int lastIndex = 1 + n; + + // contain a signature that signs the siblings and thereby ensures the integrity. + senderTransaction = BundleService.initTransaction(senderAddress, 0, lastIndex, timestamp, join); + + // siblings for merkle tree. + merkleTransaction = BundleService.initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); + + // list of confirming tips + for (int i = 2; i <= lastIndex; i++) { + byte[] tx = BundleService.initTransaction(receiverAddress, i, lastIndex, timestamp, 0L); + byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + dataTransactions.add(tx); + } + + // calculate bundle hash + List bundle = new ArrayList<>(); + bundle.add(senderTransaction); + bundle.add(merkleTransaction); + bundle.addAll(dataTransactions); + byte [] bundleHash = BundleService.addBundleHash(bundle, SpongeFactory.Mode.S256); + + // sign bundle + if (sign) { + try { + BundleService.signBundle("./src/main/resources/Nominee.key", merkleTransaction, senderTransaction, bundleHash, keyIndex, maxKeyIndex); + } catch (IOException e) { + log.error("Cannot read keyfile", e); + } + } } - @Override - public List getTxTips() { - return null; + public byte[] getCuratorTransaction() { + return curatorTransaction; } + } From ead6d34e41b42a879e2f2d70eaa4c1be213aecc5 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 7 Aug 2019 15:48:50 +0200 Subject: [PATCH 155/223] rename milestoneTracker, delete unused methods --- src/main/java/net/helix/hlx/Helix.java | 7 +- .../helix/hlx/controllers/RoundViewModel.java | 4 +- .../hlx/controllers/TransactionViewModel.java | 4 +- src/main/java/net/helix/hlx/network/Node.java | 10 +-- src/main/java/net/helix/hlx/service/API.java | 19 +++--- .../hlx/service/ledger/LedgerService.java | 2 +- .../ledger/impl/LedgerServiceImpl.java | 2 +- .../LatestSolidMilestoneTracker.java | 4 +- ...toneTracker.java => MilestoneTracker.java} | 23 +------ .../impl/LatestSolidMilestoneTrackerImpl.java | 67 ++++++------------- ...kerImpl.java => MilestoneTrackerImpl.java} | 67 ++----------------- .../snapshot/LocalSnapshotManager.java | 6 +- .../hlx/service/snapshot/SnapshotService.java | 14 ++-- .../impl/LocalSnapshotManagerImpl.java | 30 ++++----- .../snapshot/impl/SnapshotServiceImpl.java | 14 ++-- .../impl/EntryPointSelectorImpl.java | 14 ++-- 16 files changed, 93 insertions(+), 194 deletions(-) rename src/main/java/net/helix/hlx/service/milestone/{LatestMilestoneTracker.java => MilestoneTracker.java} (75%) rename src/main/java/net/helix/hlx/service/milestone/impl/{LatestMilestoneTrackerImpl.java => MilestoneTrackerImpl.java} (84%) diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index 60b78b4c..7b993820 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -11,7 +11,6 @@ import net.helix.hlx.network.replicator.Replicator; import net.helix.hlx.service.Graphstream; import net.helix.hlx.service.TipsSolidifier; -import net.helix.hlx.service.curator.CandidateTracker; import net.helix.hlx.service.curator.impl.CandidateTrackerImpl; import net.helix.hlx.service.ledger.impl.LedgerServiceImpl; import net.helix.hlx.service.milestone.impl.*; @@ -81,7 +80,7 @@ public class Helix { public final SnapshotServiceImpl snapshotService; public final LocalSnapshotManagerImpl localSnapshotManager; public final MilestoneServiceImpl milestoneService; - public final LatestMilestoneTrackerImpl latestMilestoneTracker; + public final MilestoneTrackerImpl latestMilestoneTracker; public final NomineeTrackerImpl nomineeTracker; public final CandidateTrackerImpl candidateTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; @@ -128,7 +127,7 @@ public Helix(HelixConfig configuration) throws TransactionPruningException, Snap ? new LocalSnapshotManagerImpl() : null; milestoneService = new MilestoneServiceImpl(); - latestMilestoneTracker = new LatestMilestoneTrackerImpl(); + latestMilestoneTracker = new MilestoneTrackerImpl(); nomineeTracker = new NomineeTrackerImpl(); candidateTracker = new CandidateTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); @@ -224,7 +223,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx candidateTracker.init(tangle, snapshotProvider, configuration); latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, nomineeTracker, configuration); latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService, - latestMilestoneTracker, nomineeTracker); + latestMilestoneTracker); seenMilestonesRetriever.init(tangle, snapshotProvider, transactionRequester); milestoneSolidifier.init(snapshotProvider, transactionValidator); ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, graph); diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index cb144d42..b1e2d2e7 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -6,12 +6,12 @@ import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.persistables.Round; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.storage.Indexable; import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Pair; import net.helix.hlx.utils.Serializer; -import org.bouncycastle.util.encoders.Hex; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -19,7 +19,7 @@ /** * Acts as a controller interface for a {@link Round} hash object. This controller is used by the - * {@link net.helix.hlx.service.milestone.LatestMilestoneTracker} to manipulate a {@link Round} object. + * {@link MilestoneTracker} to manipulate a {@link Round} object. */ public class RoundViewModel { private final Round round; diff --git a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java index 685f6fa3..585b2161 100644 --- a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java @@ -2,13 +2,13 @@ import net.helix.hlx.model.*; import net.helix.hlx.model.persistables.*; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.storage.Indexable; import net.helix.hlx.storage.Persistable; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.Converter; import net.helix.hlx.utils.Pair; -import org.bouncycastle.util.encoders.Hex; import java.util.*; @@ -628,7 +628,7 @@ public void setSnapshot(Tangle tangle, Snapshot initialSnapshot, final int index /** * This method sets the {@link Transaction#milestone} flag. * - * It gets automatically called by the {@link net.helix.hlx.service.milestone.LatestMilestoneTracker} and marks transactions that represent a + * It gets automatically called by the {@link MilestoneTracker} and marks transactions that represent a * milestone accordingly. It first checks if the {@link Transaction#milestone} flag has changed and if so, it issues * a database update. * diff --git a/src/main/java/net/helix/hlx/network/Node.java b/src/main/java/net/helix/hlx/network/Node.java index cd486925..6badbfff 100644 --- a/src/main/java/net/helix/hlx/network/Node.java +++ b/src/main/java/net/helix/hlx/network/Node.java @@ -13,7 +13,7 @@ import net.helix.hlx.model.HashFactory; import net.helix.hlx.model.TransactionHash; import net.helix.hlx.service.Graphstream; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import org.apache.commons.lang3.StringUtils; @@ -75,7 +75,7 @@ public class Node { private final SnapshotProvider snapshotProvider; private final TipsViewModel tipsViewModel; private final TransactionValidator transactionValidator; - private final LatestMilestoneTracker latestMilestoneTracker; + private final MilestoneTracker milestoneTracker; private final TransactionRequester transactionRequester; private Graphstream graph; @@ -103,11 +103,11 @@ public class Node { * @param transactionValidator makes sure transaction is not malformed. * @param transactionRequester Contains a set of transaction hashes to be requested from peers. * @param tipsViewModel Contains a hash of solid and non solid tips - * @param latestMilestoneTracker Tracks milestones issued from the coordinator + * @param milestoneTracker Tracks milestones issued from the coordinator * @param configuration Contains all the config. * */ - public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final LatestMilestoneTracker latestMilestoneTracker, final NodeConfig configuration, Graphstream graph + public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final MilestoneTracker milestoneTracker, final NodeConfig configuration, Graphstream graph ) { this.configuration = configuration; this.tangle = tangle; @@ -115,7 +115,7 @@ public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final Transa this.transactionValidator = transactionValidator; this.transactionRequester = transactionRequester; this.tipsViewModel = tipsViewModel; - this.latestMilestoneTracker = latestMilestoneTracker ; + this.milestoneTracker = milestoneTracker; this.reqHashSize = configuration.getRequestHashSize(); int packetSize = configuration.getTransactionPacketSize(); this.sendingPacket = new DatagramPacket(new byte[packetSize], packetSize); diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 933a1589..91117240 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -20,7 +20,7 @@ import net.helix.hlx.service.curator.CandidateTracker; import net.helix.hlx.service.dto.*; import net.helix.hlx.service.ledger.LedgerService; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.milestone.NomineeTracker; import net.helix.hlx.service.restserver.RestConnector; import net.helix.hlx.service.snapshot.SnapshotProvider; @@ -36,7 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; @@ -108,7 +107,7 @@ public class API { private final TipSelector tipsSelector; private final TipsViewModel tipsViewModel; private final TransactionValidator transactionValidator; - private final LatestMilestoneTracker latestMilestoneTracker; + private final MilestoneTracker milestoneTracker; private final CandidateTracker candidateTracker; private final NomineeTracker nomineeTracker; private final Graphstream graph; @@ -148,13 +147,13 @@ public class API { * @param tipsSelector Handles logic for selecting tips based on other transactions * @param tipsViewModel Contains the current tips of this node * @param transactionValidator Validates transactions - * @param latestMilestoneTracker Service that tracks the latest milestone + * @param milestoneTracker Service that tracks the latest milestone */ public API(HelixConfig configuration, XI XI, TransactionRequester transactionRequester, SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, SnapshotProvider snapshotProvider, LedgerService ledgerService, Node node, TipSelector tipsSelector, TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - LatestMilestoneTracker latestMilestoneTracker, CandidateTracker candidateTracker, NomineeTracker nomineeTracker, Graphstream graph) { + MilestoneTracker milestoneTracker, CandidateTracker candidateTracker, NomineeTracker nomineeTracker, Graphstream graph) { this.configuration = configuration; this.XI = XI; @@ -168,7 +167,7 @@ public API(HelixConfig configuration, XI XI, TransactionRequester transactionReq this.tipsSelector = tipsSelector; this.tipsViewModel = tipsViewModel; this.transactionValidator = transactionValidator; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneTracker = milestoneTracker; this.candidateTracker = candidateTracker; this.nomineeTracker = nomineeTracker; this.graph = graph; @@ -661,7 +660,7 @@ private AbstractResponse getNodeInfoStatement() throws Exception { System.getProperty("java.version"), Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), - latestMilestoneTracker.getCurrentRoundIndex(), + milestoneTracker.getCurrentRoundIndex(), snapshotProvider.getLatestSnapshot().getHash(), snapshotProvider.getLatestSnapshot().getIndex(), @@ -1509,7 +1508,7 @@ private void attachStoreAndBroadcast(final String address, final String message, */ public void publishMilestone(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex) throws Exception{ - int currentRoundIndex = latestMilestoneTracker.getCurrentRoundIndex(); + int currentRoundIndex = milestoneTracker.getCurrentRoundIndex(); List confirmedTips = getConfirmedTips(); byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); @@ -1537,7 +1536,7 @@ public void publishRegistration(final String address, final int minWeightMagnitu public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { List nominees = new ArrayList<>(candidateTracker.getNominees()); - int startRoundIndex = latestMilestoneTracker.getCurrentRoundIndex() + startRoundDelay; + int startRoundIndex = milestoneTracker.getCurrentRoundIndex() + startRoundDelay; byte[] nomineeBytes = Hex.decode(nominees.stream().map(Hash::toString).collect(Collectors.joining())); BundleTypes nomineeBundle = BundleFactory.create(BundleFactory.Type.milestone, configuration.getCuratorAddress().toString(), Hash.NULL_HASH.toString(), nomineeBytes, (long) startRoundIndex, sign, keyIndex, maxKeyIndex); @@ -1588,7 +1587,7 @@ public List addMilestoneReferences(List confirmedTips, int roundInde // get branch and trunk List txToApprove = new ArrayList<>(); - //System.out.println(latestMilestoneTracker.getCurrentRoundIndex()); + //System.out.println(milestoneTracker.getCurrentRoundIndex()); if(RoundViewModel.latest(tangle) == null) { txToApprove.add(nomineeTracker.getLatestNomineeHash()); // approove initial curator tx txToApprove.add(nomineeTracker.getLatestNomineeHash()); diff --git a/src/main/java/net/helix/hlx/service/ledger/LedgerService.java b/src/main/java/net/helix/hlx/service/ledger/LedgerService.java index 479edcb2..faf4792a 100644 --- a/src/main/java/net/helix/hlx/service/ledger/LedgerService.java +++ b/src/main/java/net/helix/hlx/service/ledger/LedgerService.java @@ -44,7 +44,7 @@ public interface LedgerService { * @return {@code true} if the milestone could be applied to the ledger and {@code false} otherwise * @throws LedgerException if anything goes wrong while modifying the ledger state */ - boolean applyMilestoneToLedger(RoundViewModel round) throws LedgerException; + boolean applyRoundToLedger(RoundViewModel round) throws LedgerException; /** * Checks the consistency of the combined balance changes of the given tips.
diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index 11373250..3ff83eb1 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -104,7 +104,7 @@ public void restoreLedgerState() throws LedgerException { } @Override - public boolean applyMilestoneToLedger(RoundViewModel round) throws LedgerException { + public boolean applyRoundToLedger(RoundViewModel round) throws LedgerException { if (graph != null) { for (Hash milestoneHash : round.getHashes()) { try { diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestSolidMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/LatestSolidMilestoneTracker.java index d91b2622..061186f5 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestSolidMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/LatestSolidMilestoneTracker.java @@ -21,10 +21,10 @@ public interface LatestSolidMilestoneTracker { * * @throws MilestoneException if anything unexpected happens while updating the latest solid milestone */ - void trackLatestSolidMilestone() throws MilestoneException; + void trackLatestSolidMilestones() throws MilestoneException; /** - * This method starts the background worker that automatically calls {@link #trackLatestSolidMilestone()} + * This method starts the background worker that automatically calls {@link #trackLatestSolidMilestones()} * periodically to keep the latest solid milestone up to date.
*/ void start(); diff --git a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java b/src/main/java/net/helix/hlx/service/milestone/MilestoneTracker.java similarity index 75% rename from src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java rename to src/main/java/net/helix/hlx/service/milestone/MilestoneTracker.java index ad8edf95..40a5fdbe 100644 --- a/src/main/java/net/helix/hlx/service/milestone/LatestMilestoneTracker.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestoneTracker.java @@ -12,11 +12,10 @@ * Knowing about the latest milestone and being able to compare it to the latest solid milestone allows us to determine * if our node is "in sync".
*/ -public interface LatestMilestoneTracker { +public interface MilestoneTracker { void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numberOfMilestones, int numberOfNominees); - void clearLatestRoundHashes(); /** * Returns the index of the latest milestone that was seen by this tracker.
@@ -38,19 +37,7 @@ public interface LatestMilestoneTracker { * * @return the transaction hash of the latest milestone that was seen by this tracker */ - Set getLatestRoundHashes() throws Exception; - - /** - * Sets the latest milestone.
- *
- * It simply stores the passed in values in their corresponding internal properties and can therefore be used to - * inform the {@link LatestSolidMilestoneTracker} about a new milestone. It is internally used to set the new - * milestone but can also be used by tests to mock a certain behaviour or in case we detect a new milestone in other - * parts of the code.
- * - * @param latestRoundIndex the milestone index of the milestone - */ - void setCurrentRoundIndex(int latestRoundIndex); + Set getMilestonesOfCurrentRound() throws Exception; void setCurrentNominees(Set nomineeAddresses); @@ -77,11 +64,9 @@ public interface LatestMilestoneTracker { boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException; /** - * Since the {@link LatestMilestoneTracker} scans all milestone candidates whenever the node restarts, this flag gives us + * Since the {@link MilestoneTracker} scans all milestone candidates whenever the node restarts, this flag gives us * the ability to determine if this initialization process has finished.
*
- * The values returned by {@link #getLatestRoundHashes()} ()} and {@link #getLatestRoundIndex()} ()} will potentially - * return wrong values until the scan has completed.
* * @return {@code true} if the initial scan of milestones has finished and {@code false} otherwise */ @@ -97,6 +82,4 @@ public interface LatestMilestoneTracker { * This method stops the background worker that updates the latest milestones.
*/ void shutdown(); - - void bootstrapCurrentRoundIndex(); } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index bf4410d4..86211078 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -4,11 +4,10 @@ import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.service.ledger.LedgerService; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.milestone.LatestSolidMilestoneTracker; import net.helix.hlx.service.milestone.MilestoneException; import net.helix.hlx.service.milestone.MilestoneService; -import net.helix.hlx.service.milestone.NomineeTracker; import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; @@ -30,7 +29,7 @@ */ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTracker { /** - * Holds the interval (in milliseconds) in which the {@link #trackLatestSolidMilestone()} method gets + * Holds the interval (in milliseconds) in which the {@link #trackLatestSolidMilestones()} method gets * called by the background worker.
*/ private static final int RESCAN_INTERVAL = 1000; @@ -59,15 +58,13 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac /** * Holds a reference to the manager that keeps track of the latest milestone.
*/ - private LatestMilestoneTracker latestMilestoneTracker; + private MilestoneTracker milestoneTracker; /** * Holds a reference to the service that contains the logic for applying milestones to the ledger state.
*/ private LedgerService ledgerService; - private NomineeTracker validatorTracker; - /** * Holds a reference to the manager of the background worker.
*/ @@ -105,26 +102,25 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac * @param snapshotProvider manager for the snapshots that allows us to retrieve the relevant snapshots of this node * @param milestoneService contains the important business logic when dealing with milestones * @param ledgerService the manager for - * @param latestMilestoneTracker the manager that keeps track of the latest milestone + * @param milestoneTracker the manager that keeps track of the latest milestone * @return the initialized instance itself to allow chaining */ public LatestSolidMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, LedgerService ledgerService, - LatestMilestoneTracker latestMilestoneTracker, NomineeTracker validatorTracker) { + MilestoneTracker milestoneTracker) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.milestoneService = milestoneService; this.ledgerService = ledgerService; - this.latestMilestoneTracker = latestMilestoneTracker; - this.validatorTracker = validatorTracker; + this.milestoneTracker = milestoneTracker; return this; } @Override public void start() { - executorService.silentScheduleWithFixedDelay(this::latestSolidMilestoneTrackerThread, 0, RESCAN_INTERVAL, + executorService.silentScheduleWithFixedDelay(this::latestSolidMilestonesTrackerThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); } @@ -137,15 +133,15 @@ public void shutdown() { * {@inheritDoc} *
* In addition to applying the found milestones to the ledger state it also issues log messages and keeps the - * {@link LatestMilestoneTracker} in sync (if we happen to process a new latest milestone faster).
+ * {@link MilestoneTracker} in sync (if we happen to process a new latest milestone faster).
*/ @Override - public void trackLatestSolidMilestone() throws MilestoneException { + public void trackLatestSolidMilestones() throws MilestoneException { try { int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); RoundViewModel nextRound; - while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < latestMilestoneTracker.getCurrentRoundIndex()) - && (currentSolidRoundIndex != latestMilestoneTracker.getCurrentRoundIndex() - 1 || !latestMilestoneTracker.isRoundActive(System.currentTimeMillis()))) { + while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) + && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(System.currentTimeMillis()))) { nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); @@ -163,8 +159,7 @@ public void trackLatestSolidMilestone() throws MilestoneException { } // check solidity of milestones - // TODO: How do we handle non-solid milestones? Should we only store a milestone if its solid, should we only create snapshot from solid milestones? - // TODO: This solution is definitely wrong, we should continue even when there are non-solid milestones + // TODO: How do we handle non-solid milestones? boolean allSolid = true; for (Hash milestoneHash : nextRound.getHashes()) { if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { @@ -174,7 +169,7 @@ public void trackLatestSolidMilestone() throws MilestoneException { if (allSolid) { //syncValidatorTracker(); //syncLatestMilestoneTracker(nextRound.index()); - applySolidMilestoneToLedger(nextRound); + applyRoundToLedger(nextRound); logChange(currentSolidRoundIndex); currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); } @@ -187,20 +182,20 @@ public void trackLatestSolidMilestone() throws MilestoneException { /** * Contains the logic for the background worker.
*
- * It simply calls {@link #trackLatestSolidMilestone()} and wraps with a log handler that prevents the {@link + * It simply calls {@link #trackLatestSolidMilestones()} and wraps with a log handler that prevents the {@link * MilestoneException} to crash the worker.
*/ - private void latestSolidMilestoneTrackerThread() { + private void latestSolidMilestonesTrackerThread() { try { if (firstRun) { firstRun = false; ledgerService.restoreLedgerState(); - //latestMilestoneTracker.bootstrapCurrentRoundIndex(); + //milestoneTracker.bootstrapCurrentRoundIndex(); logChange(snapshotProvider.getInitialSnapshot().getIndex()); } - trackLatestSolidMilestone(); + trackLatestSolidMilestones(); } catch (Exception e) { log.error("error while updating the solid milestone", e); } @@ -211,13 +206,13 @@ private void latestSolidMilestoneTrackerThread() { *
* If the application of the milestone fails, we start a repair routine which will revert the milestones preceding * our current milestone and consequently try to reapply them in the next iteration of the {@link - * #trackLatestSolidMilestone()} method (until the problem is solved).
+ * #trackLatestSolidMilestones()} method (until the problem is solved).
* * @param round the milestone that shall be applied to the ledger state * @throws Exception if anything unexpected goes wrong while applying the milestone to the ledger */ - private void applySolidMilestoneToLedger(RoundViewModel round) throws Exception { - if (ledgerService.applyMilestoneToLedger(round)) { + private void applyRoundToLedger(RoundViewModel round) throws Exception { + if (ledgerService.applyRoundToLedger(round)) { if (isRepairRunning() && isRepairSuccessful(round)) { stopRepair(); } @@ -261,27 +256,6 @@ private void stopRepair() { errorCausingRoundIndex = Integer.MAX_VALUE; } - /** - * Keeps the {@link LatestMilestoneTracker} in sync with our current progress.
- *
- * Since the {@link LatestMilestoneTracker} scans all old milestones during its startup (which can take a while to - * finish) it can happen that we see a newer latest milestone faster than this manager.
- *
- * Note: This method ensures that the latest milestone index is always bigger or equals the latest solid milestone - * index. - * - * @param roundIndex milestone index - */ - private void syncLatestMilestoneTracker(int roundIndex) { - if(roundIndex > latestMilestoneTracker.getCurrentRoundIndex()) { - latestMilestoneTracker.setCurrentRoundIndex(roundIndex); - } - } - - private void syncNomineeTracker() { - latestMilestoneTracker.setCurrentNominees(validatorTracker.getLatestNominees()); - } - /** * Emits a log message whenever the latest solid milestone changes.
*
@@ -293,7 +267,6 @@ private void syncNomineeTracker() { private void logChange(int prevSolidRoundIndex) { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); int latestRoundIndex = latestSnapshot.getIndex(); - Hash latestMilestoneHash = latestSnapshot.getHash(); if (prevSolidRoundIndex != latestRoundIndex) { log.debug("Round #" + latestRoundIndex + " is SOLID"); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java similarity index 84% rename from src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java rename to src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java index 5bbaedd7..5cf49eea 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java @@ -6,10 +6,7 @@ import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; -import net.helix.hlx.model.HashFactory; -import net.helix.hlx.model.persistables.Round; import net.helix.hlx.service.milestone.*; -import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; @@ -27,7 +24,7 @@ * It can be used to determine the sync-status of the node by comparing these values against the latest solid * milestone.
*/ -public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { +public class MilestoneTrackerImpl implements MilestoneTracker { /** * Holds the amount of milestone candidates that will be analyzed per iteration of the background worker.
*/ @@ -42,7 +39,7 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { /** * Holds the logger of this class (a rate limited logger that doesn't spam the CLI output).
*/ - private static final IntervalLogger log = new IntervalLogger(LatestMilestoneTrackerImpl.class); + private static final IntervalLogger log = new IntervalLogger(MilestoneTrackerImpl.class); /** * Holds the Tangle object which acts as a database interface.
@@ -92,16 +89,6 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { private int latestNomineeUpdate; - /** - * Holds the round index of the latest round that we have seen / processed.
- */ - private int currentRoundIndex; - - /** - * Holds the transaction hashes of the latest round that we have seen / processed.
- */ - private Set latestRoundHashes = new HashSet<>(); - /** * A set that allows us to keep track of the candidates that have been seen and added to the {@link * #milestoneCandidatesToAnalyze} already.
@@ -135,7 +122,7 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { * amount of code that is necessary to correctly instantiate this class, we return the instance itself which * allows us to still instantiate, initialize and assign in one line - see Example:
*
- * {@code latestMilestoneTracker = new LatestMilestoneTrackerImpl().init(...);} + * {@code latestMilestoneTracker = new MilestoneTrackerImpl().init(...);} * * @param tangle Tangle object which acts as a database interface * @param snapshotProvider manager for the snapshots that allows us to retrieve the relevant snapshots of this node @@ -144,8 +131,8 @@ public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { * @param config configuration object which allows us to determine the important config parameters of the node * @return the initialized instance itself to allow chaining */ - public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, NomineeTracker nomineeTracker, HelixConfig config) { + public MilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, + MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, NomineeTracker nomineeTracker, HelixConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; @@ -162,8 +149,6 @@ public LatestMilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotP setCurrentNominees(nomineeTracker.getLatestNominees()); - //bootstrapLatestRoundIndex(); - return this; } @@ -179,12 +164,6 @@ public void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numbe log.delegate().debug("New milestone {} ({}/{}) added to round #{}", milestoneHash, numberOfMilestones, numberOfNominees, roundIndex); } - @Override - public void setCurrentRoundIndex(int roundIndex) { - tangle.publish("lmi %d %d", this.currentRoundIndex, roundIndex); - log.delegate().info("Latest round has changed from #{} to #{}", this.currentRoundIndex, roundIndex); - this.currentRoundIndex = roundIndex; - } @Override public void setCurrentNominees(Set nominees) { @@ -210,7 +189,7 @@ public boolean isRoundActive(long time) { } @Override - public Set getLatestRoundHashes() throws Exception{ + public Set getMilestonesOfCurrentRound() throws Exception{ int index = getCurrentRoundIndex(); try { RoundViewModel currentRound = RoundViewModel.get(tangle, index); @@ -220,12 +199,10 @@ public Set getLatestRoundHashes() throws Exception{ return currentRound.getHashes(); } } catch (Exception e) { - throw new MilestoneException("unexpected error while getting latest milestones (#" + index + ")", e); + throw new MilestoneException("unexpected error while getting milestones of current round (#" + index + ")", e); } } - @Override - public void clearLatestRoundHashes() {this.latestRoundHashes.clear();} @Override public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException { @@ -265,7 +242,6 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, nominees)) { case VALID: log.debug("Milestone " + transaction.getHash() + " is VALID"); - // TODO: 1.review conditions, 2.is okay to store here? // before a milestone can be added to a round the following conditions have to be checked: // - index is bigger than initial snapshot (above) // - VALID: @@ -274,7 +250,6 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw // - index is bigger than snapshot index // - attachment timestamp is in correct time window for the index // - there doesn't already exist a milestone with the same address for that round - //System.out.println("attachment timestamp round: " + getRound(transaction.getAttachmentTimestamp())); if (roundIndex == getRound(transaction.getAttachmentTimestamp()) && isRoundActive(transaction.getAttachmentTimestamp())) { RoundViewModel currentRoundViewModel; @@ -285,9 +260,6 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw if (RoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { currentRoundViewModel.addMilestone(transaction.getHash()); currentRoundViewModel.update(tangle); - //log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex + " (" + currentRoundViewModel.size() + ")"); - } else { - //log.error("Round #" + roundIndex + " has already received a milestone from " + transaction.getAddressHash().hexString()); } } // this is the first milestone for that round, make new database entry @@ -296,7 +268,6 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw milestones.add(transaction.getHash()); currentRoundViewModel = new RoundViewModel(roundIndex, milestones); currentRoundViewModel.store(tangle); - //log.info("Milestone " + transaction.getHash().hexString() + " is stored in round #" + roundIndex); } addMilestoneToRoundLog(transaction.getHash(), roundIndex, currentRoundViewModel.size(), nominees.size()); @@ -353,30 +324,6 @@ public void shutdown() { } - /** - * This method bootstraps this tracker with the latest milestone values that can easily be retrieved without - * analyzing any transactions (for faster startup).
- *
- * It first sets the latest milestone to the values found in the latest snapshot and then check if there is a younger - * milestone at the end of our database. While this last entry in the database doesn't necessarily have to be the - * latest one we know it at least gives a reasonable value most of the times.
- */ - @Override - public void bootstrapCurrentRoundIndex() { - Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - setCurrentRoundIndex(latestSnapshot.getIndex() + 1); - - try { - RoundViewModel lastMilestoneInDatabase = RoundViewModel.latest(tangle); - if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getCurrentRoundIndex()) { - setCurrentRoundIndex(lastMilestoneInDatabase.index()); - } - } catch (Exception e) { - log.error("unexpectedly failed to retrieve the latest milestone from the database", e); - } - } - - /** * This method contains the logic for scanning for new latest milestones that gets executed in a background * worker.
diff --git a/src/main/java/net/helix/hlx/service/snapshot/LocalSnapshotManager.java b/src/main/java/net/helix/hlx/service/snapshot/LocalSnapshotManager.java index 1b1d0c2b..f588e418 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/LocalSnapshotManager.java +++ b/src/main/java/net/helix/hlx/service/snapshot/LocalSnapshotManager.java @@ -1,6 +1,6 @@ package net.helix.hlx.service.snapshot; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; /** * Represents the manager for local {@link Snapshot}s that takes care of periodically creating a new {@link Snapshot} @@ -19,11 +19,11 @@ public interface LocalSnapshotManager { * * @param milestoneTracker tracker for the milestones to determine when a new local snapshot is due */ - void start(LatestMilestoneTracker milestoneTracker); + void start(MilestoneTracker milestoneTracker); /** * Stops the {@link Thread} that takes care of creating the local {@link Snapshot}s and that was spawned by the - * {@link #start(LatestMilestoneTracker)} method. + * {@link #start(MilestoneTracker)} method. */ void shutdown(); } \ No newline at end of file diff --git a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java index 3f8e1476..efd5862d 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java +++ b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java @@ -2,7 +2,7 @@ import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.model.Hash; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.transactionpruning.TransactionPruner; import java.util.Map; @@ -60,11 +60,11 @@ public interface SnapshotService { * After persisting the local snapshot on the hard disk of the node, it updates the {@link Snapshot} instances used * by the {@code snapshotProvider} to reflect the newly created {@link Snapshot}. * - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneTracker milestone tracker that allows us to retrieve information about the known milestones * @param transactionPruner manager for the pruning jobs that takes care of cleaning up the old data that * @throws SnapshotException if anything goes wrong while creating the local snapshot */ - void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, TransactionPruner transactionPruner) throws + void takeLocalSnapshot(MilestoneTracker milestoneTracker, TransactionPruner transactionPruner) throws SnapshotException; /** @@ -74,12 +74,12 @@ void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, Transactio * points and all seen milestones, that were issued after the snapshot and can therefore be used to generate the * local snapshot files. * - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneTracker milestone tracker that allows us to retrieve information about the known milestones * @param targetMilestone milestone that is used as a reference point for the snapshot * @return a local snapshot of the full ledger state at the given milestone * @throws SnapshotException if anything goes wrong while generating the local snapshot */ - Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundViewModel targetMilestone) throws + Snapshot generateSnapshot(MilestoneTracker milestoneTracker, RoundViewModel targetMilestone) throws SnapshotException; /** @@ -102,11 +102,11 @@ Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundVi * that use local snapshot files to bootstrap their nodes, to faster request the missing milestones when syncing the * very first time. * - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneTracker milestone tracker that allows us to retrieve information about the known milestones * @param targetRound milestone that is used as a reference point for the snapshot * @return a map of solid entry points associating their hash to the milestone index that confirmed them * @throws SnapshotException if anything goes wrong while generating the solid entry points */ - List generateSeenRounds(LatestMilestoneTracker latestMilestoneTracker, + List generateSeenRounds(MilestoneTracker milestoneTracker, RoundViewModel targetRound) throws SnapshotException; } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java index 1d0f8de0..c8439be9 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/LocalSnapshotManagerImpl.java @@ -1,7 +1,7 @@ package net.helix.hlx.service.snapshot.impl; import net.helix.hlx.conf.SnapshotConfig; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.snapshot.LocalSnapshotManager; import net.helix.hlx.service.snapshot.SnapshotException; import net.helix.hlx.service.snapshot.SnapshotProvider; @@ -17,7 +17,7 @@ * intervals have passed.
*
* It incorporates a background worker that periodically checks if a new snapshot is due (see {@link - * #start(LatestMilestoneTracker)} and {@link #shutdown()}).
+ * #start(MilestoneTracker)} and {@link #shutdown()}).
*/ public class LocalSnapshotManagerImpl implements LocalSnapshotManager { /** @@ -67,7 +67,7 @@ public class LocalSnapshotManagerImpl implements LocalSnapshotManager { * Holds a reference to the {@link ThreadIdentifier} for the monitor thread. * * Using a {@link ThreadIdentifier} for spawning the thread allows the {@link ThreadUtils} to spawn exactly one - * thread for this instance even when we call the {@link #start(LatestMilestoneTracker)} method multiple times. + * thread for this instance even when we call the {@link #start(MilestoneTracker)} method multiple times. */ private ThreadIdentifier monitorThreadIdentifier = new ThreadIdentifier("Local Snapshots Monitor"); @@ -106,8 +106,8 @@ public LocalSnapshotManagerImpl init(SnapshotProvider snapshotProvider, Snapshot * {@inheritDoc} */ @Override - public void start(LatestMilestoneTracker latestMilestoneTracker) { - ThreadUtils.spawnThread(() -> monitorThread(latestMilestoneTracker), monitorThreadIdentifier); + public void start(MilestoneTracker milestoneTracker) { + ThreadUtils.spawnThread(() -> monitorThread(milestoneTracker), monitorThreadIdentifier); } /** @@ -124,15 +124,15 @@ public void shutdown() { * It periodically checks if a new {@link net.helix.hlx.service.snapshot.Snapshot} has to be taken until the * {@link Thread} is terminated. If it detects that a {@link net.helix.hlx.service.snapshot.Snapshot} is due it * triggers the creation of the {@link net.helix.hlx.service.snapshot.Snapshot} by calling - * {@link SnapshotService#takeLocalSnapshot(LatestMilestoneTracker, TransactionPruner)}. + * {@link SnapshotService#takeLocalSnapshot(MilestoneTracker, TransactionPruner)}. * - * @param latestMilestoneTracker tracker for the milestones to determine when a new local snapshot is due + * @param milestoneTracker tracker for the milestones to determine when a new local snapshot is due */ //@VisibleForTesting - void monitorThread(LatestMilestoneTracker latestMilestoneTracker) { + void monitorThread(MilestoneTracker milestoneTracker) { while (!Thread.currentThread().isInterrupted()) { - int localSnapshotInterval = latestMilestoneTracker.isInitialScanComplete() && - snapshotProvider.getLatestSnapshot().getIndex() == latestMilestoneTracker.getCurrentRoundIndex() + int localSnapshotInterval = milestoneTracker.isInitialScanComplete() && + snapshotProvider.getLatestSnapshot().getIndex() == milestoneTracker.getCurrentRoundIndex() ? config.getLocalSnapshotsIntervalSynced() : config.getLocalSnapshotsIntervalUnsynced(); @@ -141,7 +141,7 @@ void monitorThread(LatestMilestoneTracker latestMilestoneTracker) { if (latestSnapshotIndex - initialSnapshotIndex > config.getLocalSnapshotsDepth() + localSnapshotInterval) { try { - snapshotService.takeLocalSnapshot(latestMilestoneTracker, transactionPruner); + snapshotService.takeLocalSnapshot(milestoneTracker, transactionPruner); } catch (SnapshotException e) { log.error("error while taking local snapshot", e); } @@ -173,16 +173,16 @@ int getSnapshotInterval(boolean inSync) { * This will always return false if we are not done scanning milestone * candidates during initialization. * - * @param latestMilestoneTracker tracker we use to determine milestones + * @param milestoneTracker tracker we use to determine milestones * @return true if we are in sync, otherwise false */ //@VisibleForTesting - boolean isInSync(LatestMilestoneTracker latestMilestoneTracker) { - if (!latestMilestoneTracker.isInitialScanComplete()) { + boolean isInSync(MilestoneTracker milestoneTracker) { + if (!milestoneTracker.isInitialScanComplete()) { return false; } - int latestIndex = latestMilestoneTracker.getCurrentRoundIndex(); + int latestIndex = milestoneTracker.getCurrentRoundIndex(); int latestSnapshot = snapshotProvider.getLatestSnapshot().getIndex(); // If we are out of sync, only a full sync will get us in diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index f9457111..2a3b54a7 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -7,7 +7,7 @@ import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.Hash; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.snapshot.*; import net.helix.hlx.service.spentaddresses.SpentAddressesProvider; import net.helix.hlx.service.spentaddresses.SpentAddressesService; @@ -215,12 +215,12 @@ public void rollBackMilestones(Snapshot snapshot, int targetMilestoneIndex) thro * {@inheritDoc} */ @Override - public void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, TransactionPruner transactionPruner) + public void takeLocalSnapshot(MilestoneTracker milestoneTracker, TransactionPruner transactionPruner) throws SnapshotException { RoundViewModel targetMilestone = determineMilestoneForLocalSnapshot(tangle, snapshotProvider, config); - Snapshot newSnapshot = generateSnapshot(latestMilestoneTracker, targetMilestone); + Snapshot newSnapshot = generateSnapshot(milestoneTracker, targetMilestone); if (transactionPruner != null) { cleanupExpiredSolidEntryPoints(tangle, snapshotProvider.getInitialSnapshot().getSolidEntryPoints(), @@ -236,7 +236,7 @@ public void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, Tra * {@inheritDoc} */ @Override - public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, RoundViewModel targetRound) + public Snapshot generateSnapshot(MilestoneTracker milestoneTracker, RoundViewModel targetRound) throws SnapshotException { if (targetRound == null) { @@ -272,7 +272,7 @@ public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, } snapshot.setSolidEntryPoints(generateSolidEntryPoints(targetRound)); - snapshot.setSeenRounds(generateSeenRounds(latestMilestoneTracker, targetRound)); + snapshot.setSeenRounds(generateSeenRounds(milestoneTracker, targetRound)); return snapshot; } @@ -295,7 +295,7 @@ public Map generateSolidEntryPoints(RoundViewModel targetMileston * {@inheritDoc} */ @Override - public List generateSeenRounds(LatestMilestoneTracker latestMilestoneTracker, + public List generateSeenRounds(MilestoneTracker milestoneTracker, RoundViewModel targetRound) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( @@ -306,7 +306,7 @@ public List generateSeenRounds(LatestMilestoneTracker latestMilestoneTr try { RoundViewModel seenRound = targetRound; while ((seenRound = RoundViewModel.findClosestNextRound(tangle, seenRound.index(), - latestMilestoneTracker.getCurrentRoundIndex())) != null) { + milestoneTracker.getCurrentRoundIndex())) != null) { seenRounds.add(seenRound.index()); diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index f879a199..c4dc1a68 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -2,13 +2,11 @@ import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.model.Hash; -import net.helix.hlx.service.milestone.LatestMilestoneTracker; +import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.service.tipselection.EntryPointSelector; import net.helix.hlx.storage.Tangle; -import java.util.Random; - /** * Implementation of {@link EntryPointSelector} that given a depth {@code N}, returns a N-deep milestone. @@ -19,19 +17,19 @@ public class EntryPointSelectorImpl implements EntryPointSelector { private final Tangle tangle; private final SnapshotProvider snapshotProvider; - private final LatestMilestoneTracker latestMilestoneTracker; + private final MilestoneTracker milestoneTracker; /** * Constructor for Entry Point Selector * @param tangle Tangle object which acts as a database interface. * @param snapshotProvider accesses snapshots of the ledger state - * @param latestMilestoneTracker used to get latest milestone. + * @param milestoneTracker used to get latest milestone. */ public EntryPointSelectorImpl(Tangle tangle, SnapshotProvider snapshotProvider, - LatestMilestoneTracker latestMilestoneTracker) { + MilestoneTracker milestoneTracker) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneTracker = milestoneTracker; } @Override @@ -39,7 +37,7 @@ public Hash getEntryPoint(int depth) throws Exception { int milestoneIndex = Math.max(snapshotProvider.getLatestSnapshot().getIndex() - depth - 1, snapshotProvider.getInitialSnapshot().getIndex()); RoundViewModel roundViewModel = RoundViewModel.findClosestNextRound(tangle, milestoneIndex, - latestMilestoneTracker.getCurrentRoundIndex()); + milestoneTracker.getCurrentRoundIndex()); //todo which transaction using as solid entry point when there are multiple milestones / confirmed tips? //todo sometimes produces error here because entry point is not consistent (not sure under what conditions) //temporary solution: select random From a3962326cd7ccddbed0a0f67859b9803b6465fcc Mon Sep 17 00:00:00 2001 From: ofo42 Date: Wed, 7 Aug 2019 19:02:11 +0200 Subject: [PATCH 156/223] Refactor bundletypes 2 --- src/main/java/net/helix/hlx/service/API.java | 22 ++- .../curator/impl/NomineePublisher.java | 4 +- .../service/milestone/MilestonePublisher.java | 11 +- .../java/net/helix/hlx/utils/BundleUtils.java | 183 ++++++++++++++++++ .../hlx/utils/bundletypes/BundleFactory.java | 30 --- .../hlx/utils/bundletypes/BundleService.java | 65 ------- .../hlx/utils/bundletypes/BundleTypes.java | 27 --- .../utils/bundletypes/MilestoneBundle.java | 67 ------- .../hlx/utils/bundletypes/NomineeBundle.java | 60 ------ .../utils/bundletypes/RegistrationBundle.java | 68 ------- 10 files changed, 201 insertions(+), 336 deletions(-) create mode 100644 src/main/java/net/helix/hlx/utils/BundleUtils.java delete mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java delete mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java delete mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java delete mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java delete mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java delete mode 100644 src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 91117240..3ddb5e57 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -28,9 +28,8 @@ import net.helix.hlx.service.tipselection.TipSelector; import net.helix.hlx.service.tipselection.impl.WalkValidatorImpl; import net.helix.hlx.storage.Tangle; +import net.helix.hlx.utils.BundleUtils; import net.helix.hlx.utils.Serializer; -import net.helix.hlx.utils.bundletypes.BundleTypes; -import net.helix.hlx.utils.bundletypes.BundleFactory; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -1512,10 +1511,12 @@ public void publishMilestone(final String address, final int minWeightMagnitude, List confirmedTips = getConfirmedTips(); byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); - BundleTypes milestoneBundle = BundleFactory.create(BundleFactory.Type.milestone, address, Hash.NULL_HASH.toString(), tipsBytes, (long) currentRoundIndex, sign, keyIndex, maxKeyIndex); + BundleUtils bundle = new BundleUtils(HashFactory.ADDRESS.create(address), Hash.NULL_HASH); + bundle.create(tipsBytes, currentRoundIndex, sign, keyIndex, maxKeyIndex); + List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, bundle.getTransactions()); - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, milestoneBundle.getTransactions()); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); } @@ -1524,10 +1525,12 @@ public void publishMilestone(final String address, final int minWeightMagnitude, public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { byte[] data = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - BundleTypes registrationBundle = BundleFactory.create(BundleFactory.Type.milestone, address, configuration.getCuratorAddress().toString(), data, join ? 1L : -1L, sign, keyIndex, maxKeyIndex); - List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, registrationBundle.getTransactions()); + BundleUtils bundle = new BundleUtils(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress()); + bundle.create(data, join ? 1L : -1L, sign, keyIndex, maxKeyIndex); + + List txToApprove = getTransactionToApproveTips(3, Optional.empty()); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, bundle.getTransactions()); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); } @@ -1539,7 +1542,8 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B int startRoundIndex = milestoneTracker.getCurrentRoundIndex() + startRoundDelay; byte[] nomineeBytes = Hex.decode(nominees.stream().map(Hash::toString).collect(Collectors.joining())); - BundleTypes nomineeBundle = BundleFactory.create(BundleFactory.Type.milestone, configuration.getCuratorAddress().toString(), Hash.NULL_HASH.toString(), nomineeBytes, (long) startRoundIndex, sign, keyIndex, maxKeyIndex); + BundleUtils bundle = new BundleUtils(configuration.getCuratorAddress(), Hash.NULL_HASH); + bundle.create(nomineeBytes, startRoundIndex, sign, keyIndex, maxKeyIndex); // get branch and trunk List txToApprove = new ArrayList<>(); @@ -1550,7 +1554,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, nomineeBundle.getTransactions()); + List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, bundle.getTransactions()); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); } diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 63e96016..0bdfd672 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -33,14 +33,14 @@ public NomineePublisher(HelixConfig configuration, API api) { public void startScheduledExecutorService() { log.info("NomineePublisher scheduledExecutorService started."); - log.info("Update Nominees every: " + delay / 1000 + "s."); + log.debug("Set of nominees updated in: {} interval", delay / 1000 + "s"); scheduledExecutorService.scheduleWithFixedDelay(getRunnableUpdateNominees(), 0, delay, TimeUnit.MILLISECONDS); } private void UpdateNominees() throws Exception { - log.info("Publishing new Nominees ..."); + log.debug("Publishing new Nominees..."); api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth())); currentKeyIndex += 1; } diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 0851fb36..7e853497 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -5,9 +5,6 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.API; -import net.helix.hlx.utils.bundletypes.BundleFactory; -import net.helix.hlx.utils.bundletypes.BundleTypes; -import net.helix.hlx.utils.bundletypes.MilestoneBundle; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; @@ -15,12 +12,10 @@ import org.slf4j.LoggerFactory; import java.io.*; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; public class MilestonePublisher { @@ -29,7 +24,7 @@ public class MilestonePublisher { private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private HelixConfig config; private API api; - NomineeTracker nomineeTracker; + private NomineeTracker nomineeTracker; private Hash address; private String message; @@ -44,7 +39,7 @@ public class MilestonePublisher { private int startRound; public boolean enabled; - public boolean active; + private boolean active; public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nomineeTracker) { this.config = configuration; @@ -140,7 +135,7 @@ private void publishMilestone() throws Exception { log.debug("Legitimized nominee {} for round #{}", address, startRound); } if (startRound == getRound(System.currentTimeMillis())) { - log.debug("Submitting milestones every: " + (config.getRoundDuration() / 1000) + "s"); + log.debug("Submitting milestones in {} interval: ", (config.getRoundDuration() / 1000) + "s"); active = true; } } diff --git a/src/main/java/net/helix/hlx/utils/BundleUtils.java b/src/main/java/net/helix/hlx/utils/BundleUtils.java new file mode 100644 index 00000000..5cfde5eb --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/BundleUtils.java @@ -0,0 +1,183 @@ +package net.helix.hlx.utils; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.crypto.Sponge; +import net.helix.hlx.crypto.SpongeFactory; +import net.helix.hlx.crypto.Winternitz; +import net.helix.hlx.model.Hash; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class BundleUtils { + + private final Logger log = LoggerFactory.getLogger(BundleUtils.class); + + private byte[] senderTransaction; + private byte[] merkleTransaction; + private List dataTransactions = new ArrayList<>(); + + private String senderAddress; + private String receiverAddress; + + /** + * @param senderAddress senderAddress + * @param receiverAddress receiverAddress + */ + public BundleUtils(Hash senderAddress, Hash receiverAddress) { + this.senderAddress = senderAddress.toString(); + this.receiverAddress = receiverAddress.toString(); + } + + /** + * Load existing transactions + * + * @param senderTransaction sender tx + * @param merkleTransaction merkle tx + * @param dataTransactions data txs + */ + public void load(byte[] senderTransaction, byte[] merkleTransaction, List dataTransactions) { + this.senderTransaction = senderTransaction; + this.merkleTransaction = merkleTransaction; + this.dataTransactions = dataTransactions; + } + + /** + * @return transactions + */ + public List getTransactions() { + List transactions = new ArrayList<>(); + for (int i = dataTransactions.size() - 1; i >= 0; i--) { + transactions.add(Hex.toHexString(dataTransactions.get(i))); + } + transactions.add(Hex.toHexString(merkleTransaction)); + transactions.add(Hex.toHexString(senderTransaction)); + return transactions; + } + + /** + * Method for generating bundles + * @param tips tips + * @param roundIndex round index + * @param sign whether to sign + * @param keyIndex key index + * @param maxKeyIndex maximum key index + */ + public void create(byte[] tips, long roundIndex, Boolean sign, int keyIndex, int maxKeyIndex) { + + // get number of transactions needed for tips + int n = (tips.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; + // pad data to mutiple of smf + byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; + System.arraycopy(tips, 0, paddedData, 0, tips.length); + + long timestamp = System.currentTimeMillis() / 1000L; + int lastIndex = 1 + n; + + // contain a signature that signs the siblings and thereby ensures the integrity. + this.senderTransaction = initTransaction(this.senderAddress, 0, lastIndex, timestamp, roundIndex); + + // siblings for merkle tree. + this.merkleTransaction = initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); + + // list of confirming tips + for (int i = 2; i <= lastIndex; i++) { + byte[] tx = initTransaction(this.receiverAddress, i, lastIndex, timestamp, 0L); + byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + this.dataTransactions.add(tx); + } + + // calculate bundle hash + List bundle = new ArrayList<>(); + bundle.add(this.senderTransaction); + bundle.add(this.merkleTransaction); + bundle.addAll(this.dataTransactions); + byte[] bundleHash = addBundleHash(bundle, SpongeFactory.Mode.S256); + + // sign bundle + if (sign) { + try { + signBundle("./src/main/resources/Nominee.key", this.merkleTransaction, this.senderTransaction, bundleHash, keyIndex, maxKeyIndex); + } catch (IOException e) { + log.error("Cannot read keyfile", e); + } + } + } + + /** + * @param address address + * @param currentIndex current index + * @param lastIndex last index + * @param timestamp timestamp + * @param tag tag + * @return transaction + */ + private byte[] initTransaction(String address, int currentIndex, int lastIndex, long timestamp, long tag) { + byte[] transaction = new byte[TransactionViewModel.SIZE]; + System.arraycopy(Hex.decode(address), 0, transaction, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); + System.arraycopy(Serializer.serialize((long) currentIndex), 0, transaction, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); + System.arraycopy(Serializer.serialize((long) lastIndex), 0, transaction, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); + System.arraycopy(Serializer.serialize(timestamp), 0, transaction, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize(tag), 0, transaction, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); + return transaction; + } + + /** + * @param bundle bundle + * @param mode hash mode + * @return bundle hash + */ + private byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { + Sponge sponge = SpongeFactory.create(mode); + + for (byte[] transaction : bundle) { + byte[] essence = Arrays.copyOfRange(transaction, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); + sponge.absorb(essence, 0, essence.length); + } + + byte[] bundleHash = new byte[32]; + sponge.squeeze(bundleHash, 0, bundleHash.length); + for (byte[] transaction : bundle) { + System.arraycopy(bundleHash, 0, transaction, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); + } + return bundleHash; + } + + /** + * @param filepath path to file + * @param merkleTransaction merkle transaction + * @param mainTransaction main transaction + * @param bundleHash bundle hash + * @param keyIndex key index + * @param maxKeyIndex maximum key index + * @throws IOException if file not found or not readable + */ + private void signBundle(String filepath, byte[] merkleTransaction, byte[] mainTransaction, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { + // Get merkle path and store in signatureMessageFragment of Sibling Transaction + File keyfile = new File(filepath); + List> merkleTree = Merkle.readKeyfile(keyfile); + String seed = Merkle.getSeed(keyfile); + // create merkle path from keyfile + List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + System.arraycopy(path, 0, merkleTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); + + // sign bundle hash and store signature in Milestone Transaction + byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); + byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); + final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); + byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); + byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); + byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); + System.arraycopy(signature, 0, mainTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + } +} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java deleted file mode 100644 index 01218b67..00000000 --- a/src/main/java/net/helix/hlx/utils/bundletypes/BundleFactory.java +++ /dev/null @@ -1,30 +0,0 @@ -package net.helix.hlx.utils.bundletypes; - -import net.helix.hlx.model.Hash; - -import java.util.List; - -public abstract class BundleFactory { - - - - public enum Type { - milestone, // vote - registration, // application / sign up / sign off / key transition - nominee, // ? - curator // to publish set of nominees. will be removed from public package - } - - public static BundleTypes create(Type type, String senderAddress, String receiverAddress, byte[] data, long currentRoundIndex, Boolean sign, int keyIndex, int maxKeyIndex) { - switch (type) { - case milestone: - return new MilestoneBundle(senderAddress, receiverAddress, data, currentRoundIndex, sign, keyIndex, maxKeyIndex); - case registration: - return new RegistrationBundle(senderAddress, receiverAddress, data, currentRoundIndex, sign, keyIndex, maxKeyIndex); - case nominee: - return new NomineeBundle(senderAddress, receiverAddress, data, currentRoundIndex, sign, keyIndex, maxKeyIndex); - default: - return null; - } - } -} \ No newline at end of file diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java deleted file mode 100644 index 82c2e7c4..00000000 --- a/src/main/java/net/helix/hlx/utils/bundletypes/BundleService.java +++ /dev/null @@ -1,65 +0,0 @@ -package net.helix.hlx.utils.bundletypes; - -import net.helix.hlx.controllers.TransactionViewModel; -import net.helix.hlx.crypto.Merkle; -import net.helix.hlx.crypto.Sponge; -import net.helix.hlx.crypto.SpongeFactory; -import net.helix.hlx.crypto.Winternitz; -import net.helix.hlx.model.Hash; -import net.helix.hlx.utils.Serializer; -import org.bouncycastle.util.encoders.Hex; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -public class BundleService { - - public static byte[] initTransaction(String address, int currentIndex, int lastIndex, long timestamp, long tag) { - byte[] transaction = new byte[TransactionViewModel.SIZE]; - System.arraycopy(Hex.decode(address), 0, transaction, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); - System.arraycopy(Serializer.serialize((long) currentIndex), 0, transaction, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); - System.arraycopy(Serializer.serialize((long) lastIndex), 0, transaction, TransactionViewModel.LAST_INDEX_OFFSET, TransactionViewModel.LAST_INDEX_SIZE); - System.arraycopy(Serializer.serialize(timestamp), 0, transaction, TransactionViewModel.TIMESTAMP_OFFSET, TransactionViewModel.TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize(tag), 0, transaction, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); - return transaction; - } - - public static byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { - Sponge sponge = SpongeFactory.create(mode); - - for (byte[] transaction : bundle) { - byte[] essence = Arrays.copyOfRange(transaction, TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_OFFSET + TransactionViewModel.ESSENCE_SIZE); - sponge.absorb(essence, 0, essence.length); - } - - byte[] bundleHash = new byte[32]; - sponge.squeeze(bundleHash, 0, bundleHash.length); - for (byte[] transaction : bundle) { - System.arraycopy(bundleHash, 0, transaction, TransactionViewModel.BUNDLE_OFFSET, TransactionViewModel.BUNDLE_SIZE); - } - return bundleHash; - } - - public static void signBundle(String filepath, byte[] merkleTransaction, byte[] mainTransaction, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { - // Get merkle path and store in signatureMessageFragment of Sibling Transaction - File keyfile = new File(filepath); - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); - // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); - byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); - System.arraycopy(path, 0, merkleTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); - - // sign bundle hash and store signature in Milestone Transaction - byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); - byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); - final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); - byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); - byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); - byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); - System.arraycopy(signature, 0, mainTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - } -} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java b/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java deleted file mode 100644 index 9fe76525..00000000 --- a/src/main/java/net/helix/hlx/utils/bundletypes/BundleTypes.java +++ /dev/null @@ -1,27 +0,0 @@ -package net.helix.hlx.utils.bundletypes; - -import net.helix.hlx.service.API; -import org.bouncycastle.util.encoders.Hex; - -import java.util.ArrayList; -import java.util.List; - -public abstract class BundleTypes { - byte[] senderTransaction; - byte[] merkleTransaction; - List dataTransactions = new ArrayList<>(); - - public byte[] getSenderTransaction() {return senderTransaction; } - public byte[] getMerkleTransaction() {return merkleTransaction; } - public List getDataTransactions() {return dataTransactions; } - public List getTransactions() { - List transactions = new ArrayList<>(); - for (int i = dataTransactions.size()-1; i >= 0; i--) { - transactions.add(Hex.toHexString(dataTransactions.get(i))); - } - transactions.add(Hex.toHexString(merkleTransaction)); - transactions.add(Hex.toHexString(senderTransaction)); - return transactions; - } - -} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java deleted file mode 100644 index 4d366224..00000000 --- a/src/main/java/net/helix/hlx/utils/bundletypes/MilestoneBundle.java +++ /dev/null @@ -1,67 +0,0 @@ -package net.helix.hlx.utils.bundletypes; - -import net.helix.hlx.controllers.TransactionViewModel; -import net.helix.hlx.crypto.Merkle; -import net.helix.hlx.crypto.Sponge; -import net.helix.hlx.crypto.SpongeFactory; -import net.helix.hlx.crypto.Winternitz; -import net.helix.hlx.model.Hash; -import net.helix.hlx.utils.Serializer; -import org.bouncycastle.util.encoders.Hex; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -public class MilestoneBundle extends BundleTypes { - - private static final Logger log = LoggerFactory.getLogger(MilestoneBundle.class); - - MilestoneBundle(String senderAddress, String receiverAddress, byte[] tips, long roundIndex, Boolean sign, int keyIndex, int maxKeyIndex) { - - // get number of transactions needed for tips - int n = (tips.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; - // pad data to mutiple of smf - byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - System.arraycopy(tips, 0, paddedData, 0, tips.length); - - long timestamp = System.currentTimeMillis() / 1000L; - int lastIndex = 1 + n; - - // contain a signature that signs the siblings and thereby ensures the integrity. - senderTransaction = BundleService.initTransaction(senderAddress, 0, lastIndex, timestamp, roundIndex); - - // siblings for merkle tree. - merkleTransaction = BundleService.initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); - - // list of confirming tips - for (int i = 2; i <= lastIndex; i++) { - byte[] tx = BundleService.initTransaction(receiverAddress, i, lastIndex, timestamp, 0L); - byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - dataTransactions.add(tx); - } - - // calculate bundle hash - List bundle = new ArrayList<>(); - bundle.add(senderTransaction); - bundle.add(merkleTransaction); - bundle.addAll(dataTransactions); - byte [] bundleHash = BundleService.addBundleHash(bundle, SpongeFactory.Mode.S256); - - // sign bundle - if (sign) { - try { - BundleService.signBundle("./src/main/resources/Nominee.key", merkleTransaction, senderTransaction, bundleHash, keyIndex, maxKeyIndex); - } catch (IOException e) { - log.error("Cannot read keyfile", e); - } - } - } - -} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java deleted file mode 100644 index 0a5f88c4..00000000 --- a/src/main/java/net/helix/hlx/utils/bundletypes/NomineeBundle.java +++ /dev/null @@ -1,60 +0,0 @@ -package net.helix.hlx.utils.bundletypes; - -import net.helix.hlx.controllers.TransactionViewModel; -import net.helix.hlx.crypto.SpongeFactory; -import net.helix.hlx.model.Hash; -import net.helix.hlx.utils.Serializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class NomineeBundle extends BundleTypes { - - private static final Logger log = LoggerFactory.getLogger(NomineeBundle.class); - - NomineeBundle(String senderAddress, String receiverAddress, byte[] nominees, long startRound, Boolean sign, int keyIndex, int maxKeyIndex) { - - // get number of transactions needed for nominees - int n = (nominees.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; - // pad data to mutiple of smf - byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - System.arraycopy(nominees, 0, paddedData, 0, nominees.length); - - long timestamp = System.currentTimeMillis() / 1000L; - int lastIndex = 1 + n; - - // contains the sender address and the signature - senderTransaction = BundleService.initTransaction(senderAddress, 0, lastIndex, timestamp, startRound); - - // contains for merkle tree to verify the signature - merkleTransaction = BundleService.initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); - - // contains the list of nominee addresses - for (int i = 2; i <= lastIndex; i++) { - byte[] tx = BundleService.initTransaction(receiverAddress, i, lastIndex, timestamp, 0L); - byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - dataTransactions.add(tx); - } - - // calculate bundle hash - List bundle = new ArrayList<>(); - bundle.add(senderTransaction); - bundle.add(merkleTransaction); - bundle.addAll(dataTransactions); - byte [] bundleHash = BundleService.addBundleHash(bundle, SpongeFactory.Mode.S256); - - // sign bundle - if (sign) { - try { - BundleService.signBundle("./src/main/resources/Nominee.key", merkleTransaction, senderTransaction, bundleHash, keyIndex, maxKeyIndex); - } catch (IOException e) { - log.error("Cannot read keyfile", e); - } - } - } -} diff --git a/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java b/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java deleted file mode 100644 index 7d4f00d8..00000000 --- a/src/main/java/net/helix/hlx/utils/bundletypes/RegistrationBundle.java +++ /dev/null @@ -1,68 +0,0 @@ -package net.helix.hlx.utils.bundletypes; - -import net.helix.hlx.controllers.TransactionViewModel; -import net.helix.hlx.crypto.SpongeFactory; -import net.helix.hlx.model.Hash; -import net.helix.hlx.utils.Serializer; -import org.bouncycastle.util.encoders.Hex; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class RegistrationBundle extends BundleTypes { - - private static final Logger log = LoggerFactory.getLogger(RegistrationBundle.class); - byte[] curatorTransaction; - - - public RegistrationBundle(String senderAddress, String receiverAddress, byte[] registrationData, long join, Boolean sign, int keyIndex, int maxKeyIndex) { - - // get number of transactions needed for tips - int n = (registrationData.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; - // pad data to mutiple of smf - byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - System.arraycopy(registrationData, 0, paddedData, 0, registrationData.length); - - long timestamp = System.currentTimeMillis() / 1000L; - int lastIndex = 1 + n; - - // contain a signature that signs the siblings and thereby ensures the integrity. - senderTransaction = BundleService.initTransaction(senderAddress, 0, lastIndex, timestamp, join); - - // siblings for merkle tree. - merkleTransaction = BundleService.initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); - - // list of confirming tips - for (int i = 2; i <= lastIndex; i++) { - byte[] tx = BundleService.initTransaction(receiverAddress, i, lastIndex, timestamp, 0L); - byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); - dataTransactions.add(tx); - } - - // calculate bundle hash - List bundle = new ArrayList<>(); - bundle.add(senderTransaction); - bundle.add(merkleTransaction); - bundle.addAll(dataTransactions); - byte [] bundleHash = BundleService.addBundleHash(bundle, SpongeFactory.Mode.S256); - - // sign bundle - if (sign) { - try { - BundleService.signBundle("./src/main/resources/Nominee.key", merkleTransaction, senderTransaction, bundleHash, keyIndex, maxKeyIndex); - } catch (IOException e) { - log.error("Cannot read keyfile", e); - } - } - } - - public byte[] getCuratorTransaction() { - return curatorTransaction; - } - -} From 970bb0dab5ee7388dcc4b91f227b24037c1c74f5 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Wed, 7 Aug 2019 19:06:33 +0200 Subject: [PATCH 157/223] minor --- src/main/java/net/helix/hlx/service/API.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 3ddb5e57..f1f8f07e 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1203,7 +1203,7 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac byte[] txBytes = new byte[BYTES_SIZE]; // in case remote attachToTangle is enabled and current test magnitude is exceeded. - minWeightMagnitude = (minWeightMagnitude > 2) ? 2 : minWeightMagnitude; + minWeightMagnitude = Math.min(minWeightMagnitude, 2); for (final String tx : txs) { long startTime = System.nanoTime(); From be1ef6f3b2238da8d5a0fe1675ad280b28edb195 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Wed, 7 Aug 2019 20:01:05 +0200 Subject: [PATCH 158/223] Refactor bundletypes 3 --- src/main/java/net/helix/hlx/service/API.java | 39 ++++++++------------ 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index f1f8f07e..3900ae75 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1492,10 +1492,22 @@ private void attachStoreAndBroadcast(final String address, final String message, broadcastTransactionsStatement(powResult); } + public void storeAndBroadcast(Hash tip1, Hash tip2, int mwm, List txs) throws Exception{ + List powResult = attachToTangleStatement(tip1, tip2, mwm, txs); + storeTransactionsStatement(powResult); + broadcastTransactionsStatement(powResult); + } + // // Publish methods // + private void publish(final Hash sndAddr, final Hash rcvAddr, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, byte[] tips, List txToApprove) throws Exception { + BundleUtils bundle = new BundleUtils(sndAddr, rcvAddr); + bundle.create(tips, tag, sign, keyIdx, maxKeyIdx); + storeAndBroadcast(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); + } + // refactoring WIP (the following methods will be moved from API) /** * Method to publish milestones @@ -1505,46 +1517,30 @@ private void attachStoreAndBroadcast(final String address, final String message, * @param keyIndex index of the key used for signing * @throws Exception if key file isn't readable */ - public void publishMilestone(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex) throws Exception{ + public void publishMilestone(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex) throws Exception { int currentRoundIndex = milestoneTracker.getCurrentRoundIndex(); List confirmedTips = getConfirmedTips(); byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); - BundleUtils bundle = new BundleUtils(HashFactory.ADDRESS.create(address), Hash.NULL_HASH); - bundle.create(tipsBytes, currentRoundIndex, sign, keyIndex, maxKeyIndex); - List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, bundle.getTransactions()); - - storeTransactionsStatement(powResult); - broadcastTransactionsStatement(powResult); + publish(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, tipsBytes, txToApprove); } - public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { byte[] data = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - BundleUtils bundle = new BundleUtils(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress()); - bundle.create(data, join ? 1L : -1L, sign, keyIndex, maxKeyIndex); - List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, bundle.getTransactions()); - storeTransactionsStatement(powResult); - broadcastTransactionsStatement(powResult); + publish(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, data, txToApprove); } - public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { List nominees = new ArrayList<>(candidateTracker.getNominees()); int startRoundIndex = milestoneTracker.getCurrentRoundIndex() + startRoundDelay; byte[] nomineeBytes = Hex.decode(nominees.stream().map(Hash::toString).collect(Collectors.joining())); - BundleUtils bundle = new BundleUtils(configuration.getCuratorAddress(), Hash.NULL_HASH); - bundle.create(nomineeBytes, startRoundIndex, sign, keyIndex, maxKeyIndex); - // get branch and trunk List txToApprove = new ArrayList<>(); if(RoundViewModel.latest(tangle) == null) { @@ -1553,10 +1549,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - - List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, bundle.getTransactions()); - storeTransactionsStatement(powResult); - broadcastTransactionsStatement(powResult); + publish(configuration.getCuratorAddress(), Hash.NULL_HASH, startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, nomineeBytes, txToApprove); } // From d50d51e39fd522e8195d21990e9a285e0c9b0ad8 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Wed, 7 Aug 2019 20:10:59 +0200 Subject: [PATCH 159/223] cleanup unused --- src/main/java/net/helix/hlx/utils/BundleUtils.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/java/net/helix/hlx/utils/BundleUtils.java b/src/main/java/net/helix/hlx/utils/BundleUtils.java index 5cfde5eb..5bf965d7 100644 --- a/src/main/java/net/helix/hlx/utils/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/BundleUtils.java @@ -37,19 +37,6 @@ public BundleUtils(Hash senderAddress, Hash receiverAddress) { this.receiverAddress = receiverAddress.toString(); } - /** - * Load existing transactions - * - * @param senderTransaction sender tx - * @param merkleTransaction merkle tx - * @param dataTransactions data txs - */ - public void load(byte[] senderTransaction, byte[] merkleTransaction, List dataTransactions) { - this.senderTransaction = senderTransaction; - this.merkleTransaction = merkleTransaction; - this.dataTransactions = dataTransactions; - } - /** * @return transactions */ From 16624db26ceb486ae55cb6d4b73921d85f98060e Mon Sep 17 00:00:00 2001 From: ofo42 Date: Wed, 7 Aug 2019 20:47:32 +0200 Subject: [PATCH 160/223] Refactor bundletypes 4 --- src/main/java/net/helix/hlx/service/API.java | 67 +++++++++++++++++-- .../curator/impl/NomineePublisher.java | 6 +- .../service/milestone/MilestonePublisher.java | 7 +- .../helix/hlx/utils/bundle/BundleTypes.java | 7 ++ .../hlx/utils/{ => bundle}/BundleUtils.java | 3 +- 5 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 src/main/java/net/helix/hlx/utils/bundle/BundleTypes.java rename src/main/java/net/helix/hlx/utils/{ => bundle}/BundleUtils.java (99%) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 3900ae75..243fc935 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -28,7 +28,8 @@ import net.helix.hlx.service.tipselection.TipSelector; import net.helix.hlx.service.tipselection.impl.WalkValidatorImpl; import net.helix.hlx.storage.Tangle; -import net.helix.hlx.utils.BundleUtils; +import net.helix.hlx.utils.bundle.BundleTypes; +import net.helix.hlx.utils.bundle.BundleUtils; import net.helix.hlx.utils.Serializer; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; @@ -1492,7 +1493,15 @@ private void attachStoreAndBroadcast(final String address, final String message, broadcastTransactionsStatement(powResult); } - public void storeAndBroadcast(Hash tip1, Hash tip2, int mwm, List txs) throws Exception{ + /** + * + * @param tip1 branch tx + * @param tip2 trunk tx + * @param mwm pow difficulty + * @param txs transactions list + * @throws Exception if storing fails + */ + private void storeAndBroadcast(Hash tip1, Hash tip2, int mwm, List txs) throws Exception{ List powResult = attachToTangleStatement(tip1, tip2, mwm, txs); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); @@ -1502,7 +1511,45 @@ public void storeAndBroadcast(Hash tip1, Hash tip2, int mwm, List txs) t // Publish methods // - private void publish(final Hash sndAddr, final Hash rcvAddr, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, byte[] tips, List txToApprove) throws Exception { + /** + * + * @param type type of bundle to publish + * @param address target address + * @param minWeightMagnitude pow difficulty + * @param sign whether to sign + * @param keyIndex key index + * @param maxKeyIndex max key index + * @param join joining or leaving + * @param startRoundDelay start round delay + * @throws Exception if storing fails + */ + public void publish(BundleTypes type, final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join, int startRoundDelay) throws Exception { + switch (type) { + case milestone: + publishMilestone(address, minWeightMagnitude, sign, keyIndex, maxKeyIndex); + case registration: + publishRegistration(address, minWeightMagnitude, sign, keyIndex, maxKeyIndex, join); + case nominee: + publishNominees(startRoundDelay, minWeightMagnitude, sign, keyIndex, maxKeyIndex); + default: + publishMilestone(address, minWeightMagnitude, sign, keyIndex, maxKeyIndex); + } + } + + /** + * + * @param sndAddr sender address + * @param rcvAddr receiver address + * @param tag tag + * @param mwm pow difficulty + * @param sign whether to sign + * @param keyIdx key index + * @param maxKeyIdx maximum key index + * @param tips tips + * @param txToApprove transactions to approve + * @throws Exception if storing fails + */ + private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, byte[] tips, List txToApprove) throws Exception { BundleUtils bundle = new BundleUtils(sndAddr, rcvAddr); bundle.create(tips, tag, sign, keyIdx, maxKeyIdx); storeAndBroadcast(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); @@ -1524,15 +1571,21 @@ public void publishMilestone(final String address, final int minWeightMagnitude, byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - publish(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, tipsBytes, txToApprove); + storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, tipsBytes, txToApprove); } public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { byte[] data = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - publish(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, data, txToApprove); + List txToApprove = new ArrayList<>(); + if(RoundViewModel.latest(tangle) == null) { + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); + } else { + txToApprove = getTransactionToApproveTips(3, Optional.empty()); + } + storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, data, txToApprove); } public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1549,7 +1602,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - publish(configuration.getCuratorAddress(), Hash.NULL_HASH, startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, nomineeBytes, txToApprove); + storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, nomineeBytes, txToApprove); } // diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 0bdfd672..a5905546 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -1,7 +1,9 @@ package net.helix.hlx.service.curator.impl; import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.model.Hash; import net.helix.hlx.service.API; +import net.helix.hlx.utils.bundle.BundleTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +43,9 @@ public void startScheduledExecutorService() { private void UpdateNominees() throws Exception { log.debug("Publishing new Nominees..."); - api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth())); + //api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth())); //todo remove after refactoring + // (BundleTypes type, final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join, int startRoundDelay) + api.publish(BundleTypes.nominee, Hash.NULL_HASH.toString(), mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth()), false, startRoundDelay); currentKeyIndex += 1; } diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java index 7e853497..f78a10b6 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java @@ -6,6 +6,7 @@ import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.API; +import net.helix.hlx.utils.bundle.BundleTypes; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -96,7 +97,8 @@ private void generateKeyfile(String seed) throws Exception { private void sendRegistration(Hash identity, boolean join) throws Exception { log.debug("Signing {} identity: {} ", (join ? "up" : "off"), identity); - api.publishRegistration(identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join); + //api.publishRegistration(identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join); //todo remove when done with refactoring + api.publish(BundleTypes.registration, identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join, 0); currentKeyIndex += 1; } @@ -142,7 +144,8 @@ private void publishMilestone() throws Exception { if (active) { log.debug("Publishing next Milestone..."); if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1) - 1) { - api.publishMilestone(address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex); + //api.publishMilestone(address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex); <- todo remove when refactoring is done + api.publish(BundleTypes.milestone, address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, false, 0); currentKeyIndex += 1; } else { log.debug("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network."); diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleTypes.java b/src/main/java/net/helix/hlx/utils/bundle/BundleTypes.java new file mode 100644 index 00000000..3adca8d5 --- /dev/null +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleTypes.java @@ -0,0 +1,7 @@ +package net.helix.hlx.utils.bundle; + +public enum BundleTypes { + milestone, // vote + registration, // application / sign up / sign off / key transition + nominee // to publish set of nominees. Only used by curator, thus will be removed from public package +} diff --git a/src/main/java/net/helix/hlx/utils/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java similarity index 99% rename from src/main/java/net/helix/hlx/utils/BundleUtils.java rename to src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index 5bf965d7..70dc253c 100644 --- a/src/main/java/net/helix/hlx/utils/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -1,4 +1,4 @@ -package net.helix.hlx.utils; +package net.helix.hlx.utils.bundle; import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.Merkle; @@ -6,6 +6,7 @@ import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.crypto.Winternitz; import net.helix.hlx.model.Hash; +import net.helix.hlx.utils.Serializer; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From d66b0c61101a2ddc76b8213fa4823ec5a71561e6 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 9 Aug 2019 10:25:33 +0200 Subject: [PATCH 161/223] move validateCandidate to service --- .../hlx/service/curator/CuratorService.java | 6 ++- .../curator/impl/CandidateTrackerImpl.java | 38 +---------------- .../curator/impl/CuratorServiceImpl.java | 42 +++++++------------ 3 files changed, 21 insertions(+), 65 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/CuratorService.java b/src/main/java/net/helix/hlx/service/curator/CuratorService.java index 126399ab..74dbfc27 100755 --- a/src/main/java/net/helix/hlx/service/curator/CuratorService.java +++ b/src/main/java/net/helix/hlx/service/curator/CuratorService.java @@ -1,6 +1,7 @@ package net.helix.hlx.service.curator; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import java.util.Set; @@ -20,11 +21,12 @@ public interface CuratorService { *

* * @param transactionViewModel transaction that shall be analyzed - * @param roundIndex round index of the transaction + * @param mode hash mode + * @param securityLevel * @return validity status of the transaction regarding its role as a nominee application * @throws CuratorException if anything unexpected goes wrong while validating the candidate transaction */ - CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, int roundIndex) throws CuratorException; + CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel) throws CuratorException; /** *

diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index 293dec96..7f4b298b 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -246,7 +246,7 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator tail = tx; } } - switch (validateCandidate(tail, SpongeFactory.Mode.S256, 1)) { + switch (curatorService.validateCandidate(tail, SpongeFactory.Mode.S256, 1)) { case VALID: log.info("Candidate Transaction " + transaction.getHash() + " is VALID, Address: " + tail.getAddressHash()); log.info("Join/Leave: " + RoundViewModel.getRoundIndex(tail)); @@ -284,42 +284,6 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator } } - public CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel) throws CuratorException { - - try { - - final List> bundleTransactions = BundleValidator.validate(tangle, - snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); - - if (bundleTransactions.isEmpty()) { - System.out.println("bundle empty"); - return INCOMPLETE; - } else { - for (final List bundleTransactionViewModels : bundleTransactions) { - final TransactionViewModel tail = bundleTransactionViewModels.get(0); - if (tail.getHash().equals(transactionViewModel.getHash())) { - - //todo implement when sure how bundle structure has to look like - //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { - - Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); - System.out.println("valid signature (candidate): " + validSignature); - - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { - return VALID; - } else { - return INVALID; - } - } - } - } - } catch (Exception e) { - throw new CuratorException("error while checking candidate status of " + transactionViewModel.getHash(), e); - } - return INVALID; -} - /** * {@inheritDoc} * Adds a candidate to the {@code candidatesToNominate} diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java index 28f4d673..b486faf7 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java @@ -3,6 +3,8 @@ import net.helix.hlx.BundleValidator; import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.service.curator.CandidateValidity; import net.helix.hlx.service.curator.CuratorException; @@ -57,44 +59,32 @@ public CuratorServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, } @Override - public CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, int roundIndex) throws CuratorException { - - // getCandidates not yet implemented - // TransactionViewModel existingCandidate = TransactionViewModel.getCandidates(tangle, roundIndex); - TransactionViewModel existingCurator = null; - - //log.debug("Max round index set to: {}", config.getMaxRoundIndex()); - if (roundIndex < 0 || roundIndex >= 0x200000) { - return INVALID; - } + public CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel) throws CuratorException { try { - if(existingCurator != null){ - return existingCurator.getHash().equals(transactionViewModel.getHash()) ? VALID : INVALID; - } final List> bundleTransactions = BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); if (bundleTransactions.isEmpty()) { + System.out.println("bundle empty"); return INCOMPLETE; } else { for (final List bundleTransactionViewModels : bundleTransactions) { final TransactionViewModel tail = bundleTransactionViewModels.get(0); if (tail.getHash().equals(transactionViewModel.getHash())) { - // the signed transaction - which references the confirmed transactions and contains - // the Merkle tree siblings. - - if (isCandidateBundleStructureValid(bundleTransactionViewModels, 1)) { - - boolean skipValidation = true; //config.isTestnet() && config.isDontValidateTestnetMilestoneSig(); // should be config.isDontValidateCandidates() - if (skipValidation /*|| candidateTracker.getSeenCandidatesMerkleRoot(transactionViewModel.getHash).equals(HashFactory.ADDRESS.create(merkleRoot)*/) { - // not yet implemented - // verify signature corresponds to merkle root of candidate, if so save the merkle root in - return VALID; - } else { - return INVALID; - } + + //todo implement when sure how bundle structure has to look like + //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { + + Hash senderAddress = tail.getAddressHash(); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); + System.out.println("valid signature (candidate): " + validSignature); + + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { + return VALID; + } else { + return INVALID; } } } From 1df56a386e9644211a0225aecbf1bbcf86c1f3b3 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 9 Aug 2019 10:33:39 +0200 Subject: [PATCH 162/223] solidifiy candidates with current round index --- .../hlx/service/curator/impl/CandidateTrackerImpl.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index 7f4b298b..ec33c72d 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -257,11 +257,10 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator removeFromNomineeQueue(tail.getAddressHash()); } - /*if (!transaction.isSolid()) { + if (!transaction.isSolid()) { + int currentRoundIndex = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration(); candidateSolidifier.add(transaction.getHash(), currentRoundIndex); - }*/ - // not yet implemented isCandidate - //transaction.isCandidate(tangle, snapshotProvider.getInitialSnapshot(), true); + } break; case INCOMPLETE: From fb998e480fd1e16d2accefc1e396ce89b1eb3d58 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 9 Aug 2019 12:24:19 +0200 Subject: [PATCH 163/223] create nominee directory with nomineeSolidifier, Service, Exception and Validity --- src/main/java/net/helix/hlx/HLX.java | 2 +- src/main/java/net/helix/hlx/Helix.java | 19 +- src/main/java/net/helix/hlx/service/API.java | 2 +- .../curator/impl/CandidateTrackerImpl.java | 4 +- .../{ => impl}/MilestonePublisher.java | 3 +- .../milestone/impl/MilestoneTrackerImpl.java | 1 + .../hlx/service/nominee/NomineeException.java | 38 ++++ .../hlx/service/nominee/NomineeService.java | 30 +++ .../service/nominee/NomineeSolidifier.java | 31 +++ .../NomineeTracker.java | 4 +- .../hlx/service/nominee/NomineeValidity.java | 10 + .../nominee/impl/NomineeServiceImpl.java | 100 ++++++++++ .../nominee/impl/NomineeSolidifierImpl.java | 178 ++++++++++++++++++ .../impl/NomineeTrackerImpl.java | 66 ++----- 14 files changed, 424 insertions(+), 64 deletions(-) rename src/main/java/net/helix/hlx/service/milestone/{ => impl}/MilestonePublisher.java (98%) create mode 100755 src/main/java/net/helix/hlx/service/nominee/NomineeException.java create mode 100755 src/main/java/net/helix/hlx/service/nominee/NomineeService.java create mode 100755 src/main/java/net/helix/hlx/service/nominee/NomineeSolidifier.java rename src/main/java/net/helix/hlx/service/{milestone => nominee}/NomineeTracker.java (92%) mode change 100644 => 100755 create mode 100755 src/main/java/net/helix/hlx/service/nominee/NomineeValidity.java create mode 100755 src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java create mode 100755 src/main/java/net/helix/hlx/service/nominee/impl/NomineeSolidifierImpl.java rename src/main/java/net/helix/hlx/service/{milestone => nominee}/impl/NomineeTrackerImpl.java (74%) mode change 100644 => 100755 diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index c929b3d8..e71f8e64 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -8,7 +8,7 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.service.API; import net.helix.hlx.service.Spammer; -import net.helix.hlx.service.milestone.MilestonePublisher; +import net.helix.hlx.service.milestone.impl.MilestonePublisher; import net.helix.hlx.service.curator.impl.NomineePublisher; import net.helix.hlx.service.restserver.resteasy.RestEasy; import net.helix.hlx.utils.HelixIOUtils; diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index 7b993820..d3f6b4a5 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -11,9 +11,10 @@ import net.helix.hlx.network.replicator.Replicator; import net.helix.hlx.service.Graphstream; import net.helix.hlx.service.TipsSolidifier; -import net.helix.hlx.service.curator.impl.CandidateTrackerImpl; import net.helix.hlx.service.ledger.impl.LedgerServiceImpl; import net.helix.hlx.service.milestone.impl.*; +import net.helix.hlx.service.nominee.impl.*; +import net.helix.hlx.service.curator.impl.*; import net.helix.hlx.service.snapshot.SnapshotException; import net.helix.hlx.service.snapshot.impl.LocalSnapshotManagerImpl; import net.helix.hlx.service.snapshot.impl.SnapshotProviderImpl; @@ -80,6 +81,8 @@ public class Helix { public final SnapshotServiceImpl snapshotService; public final LocalSnapshotManagerImpl localSnapshotManager; public final MilestoneServiceImpl milestoneService; + public final NomineeServiceImpl nomineeService; + public final CuratorServiceImpl curatorService; public final MilestoneTrackerImpl latestMilestoneTracker; public final NomineeTrackerImpl nomineeTracker; public final CandidateTrackerImpl candidateTracker; @@ -88,6 +91,8 @@ public class Helix { public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); public final AsyncTransactionPruner transactionPruner; public final MilestoneSolidifierImpl milestoneSolidifier; + public final NomineeSolidifierImpl nomineeSolidifier; + public final CandidateSolidifierImpl candidateSolidifier; public final TransactionRequesterWorkerImpl transactionRequesterWorker; public final Tangle tangle; @@ -127,12 +132,16 @@ public Helix(HelixConfig configuration) throws TransactionPruningException, Snap ? new LocalSnapshotManagerImpl() : null; milestoneService = new MilestoneServiceImpl(); + nomineeService = new NomineeServiceImpl(); + curatorService = new CuratorServiceImpl(); latestMilestoneTracker = new MilestoneTrackerImpl(); nomineeTracker = new NomineeTrackerImpl(); candidateTracker = new CandidateTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); + nomineeSolidifier = new NomineeSolidifierImpl(); + candidateSolidifier = new CandidateSolidifierImpl(); transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled() ? new AsyncTransactionPruner() : null; @@ -219,13 +228,17 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); } milestoneService.init(tangle, snapshotProvider, snapshotService, transactionValidator, configuration); - nomineeTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); - candidateTracker.init(tangle, snapshotProvider, configuration); + nomineeService.init(tangle, snapshotProvider, snapshotService, configuration); + curatorService.init(tangle, snapshotProvider, snapshotService, configuration); + nomineeTracker.init(tangle, snapshotProvider, nomineeService, nomineeSolidifier, configuration); + candidateTracker.init(tangle, snapshotProvider, curatorService, candidateSolidifier, configuration); latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, nomineeTracker, configuration); latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService, latestMilestoneTracker); seenMilestonesRetriever.init(tangle, snapshotProvider, transactionRequester); milestoneSolidifier.init(snapshotProvider, transactionValidator); + nomineeSolidifier.init(snapshotProvider, transactionValidator); + candidateSolidifier.init(snapshotProvider, transactionValidator); ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, graph); if (transactionPruner != null) { transactionPruner.init(tangle, snapshotProvider, spentAddressesService, tipsViewModel, configuration) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 243fc935..04cf1c58 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -21,7 +21,7 @@ import net.helix.hlx.service.dto.*; import net.helix.hlx.service.ledger.LedgerService; import net.helix.hlx.service.milestone.MilestoneTracker; -import net.helix.hlx.service.milestone.NomineeTracker; +import net.helix.hlx.service.nominee.NomineeTracker; import net.helix.hlx.service.restserver.RestConnector; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.service.spentaddresses.SpentAddressesService; diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index ec33c72d..3db4588a 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -128,11 +128,13 @@ public class CandidateTrackerImpl implements CandidateTracker { * @param config configuration object which allows us to determine the important config parameters of the node * @return the initialized instance itself to allow chaining */ - public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, HelixConfig config) { + public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, CuratorService curatorService, CandidateSolidifier candidateSolidifier, HelixConfig config) { this.tangle = tangle; this.config = config; this.snapshotProvider = snapshotProvider; + this.curatorService = curatorService; + this.candidateSolidifier = candidateSolidifier; nominees = config.getInitialNominees(); diff --git a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java similarity index 98% rename from src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java rename to src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index f78a10b6..9ddb7f3e 100644 --- a/src/main/java/net/helix/hlx/service/milestone/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -1,10 +1,11 @@ -package net.helix.hlx.service.milestone; +package net.helix.hlx.service.milestone.impl; import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.API; +import net.helix.hlx.service.nominee.NomineeTracker; import net.helix.hlx.utils.bundle.BundleTypes; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java index 5cf49eea..ab4d82cf 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java @@ -7,6 +7,7 @@ import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.service.milestone.*; +import net.helix.hlx.service.nominee.NomineeTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; diff --git a/src/main/java/net/helix/hlx/service/nominee/NomineeException.java b/src/main/java/net/helix/hlx/service/nominee/NomineeException.java new file mode 100755 index 00000000..9b2b5b92 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/nominee/NomineeException.java @@ -0,0 +1,38 @@ +package net.helix.hlx.service.nominee; + +/** + * This class is used to wrap exceptions that are specific to the curator logic. + * + * It allows us to distinct between the different kinds of errors that can happen during the execution of the code. + */ +public class NomineeException extends Exception { + /** + * Constructor of the exception which allows us to provide a specific error message and the cause of the error. + * + * @param message reason why this error occurred + * @param cause wrapped exception that caused this error + */ + public NomineeException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of the exception which allows us to provide a specific error message without having an underlying + * cause. + * + * @param message reason why this error occurred + */ + public NomineeException(String message) { + super(message); + } + + /** + * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a + * specific reason. + * + * @param cause wrapped exception that caused this error + */ + public NomineeException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/net/helix/hlx/service/nominee/NomineeService.java b/src/main/java/net/helix/hlx/service/nominee/NomineeService.java new file mode 100755 index 00000000..3a4af141 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/nominee/NomineeService.java @@ -0,0 +1,30 @@ +package net.helix.hlx.service.nominee; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.SpongeFactory; + +public interface NomineeService { + + /** + *

+ * Analyzes the given transaction to determine if it is a valid nominee transaction. + *

+ *

+ * It first checks if all transactions that belong to the nominee bundle are known already + * + * + * (and only then verifies + * the signature to analyze if the given nominee transaction was really issued by a valid nominee) <- might not be needed. + *

+ * + * @param transactionViewModel transaction that shall be analyzed + * @param mode hash mode + * @param securityLevel + * @return validity status of the transaction + * @throws NomineeException if anything unexpected goes wrong while validating the nominee transaction + */ + NomineeValidity validateNominees(TransactionViewModel transactionViewModel, int roundIndex, + SpongeFactory.Mode mode, int securityLevel) throws NomineeException; + + +} diff --git a/src/main/java/net/helix/hlx/service/nominee/NomineeSolidifier.java b/src/main/java/net/helix/hlx/service/nominee/NomineeSolidifier.java new file mode 100755 index 00000000..6d065bfa --- /dev/null +++ b/src/main/java/net/helix/hlx/service/nominee/NomineeSolidifier.java @@ -0,0 +1,31 @@ +package net.helix.hlx.service.nominee; + +import net.helix.hlx.model.Hash; + +/** + * This interface defines the contract for a manager that tries to solidify unsolid candidates by incorporating a + * background worker that periodically checks the solidity of the candidates and issues transaction requests for the + * missing transactions until the candidates become solid. + */ +public interface NomineeSolidifier { + /** + * This method allows us to add new candidates to the solidifier that will consequently be solidified. + * + * @param nomineeHash Hash of the candidate that shall be solidified + * @param roundIndex index corresponding to the round that the candidate submit the application + */ + void add(Hash nomineeHash, int roundIndex); + + /** + * This method starts the background worker that asynchronously solidifies the candidates. + */ + void start(); + + /** + * This method shuts down the background worker that asynchronously solidifies the candidates. + */ + void shutdown(); + +} + + diff --git a/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java b/src/main/java/net/helix/hlx/service/nominee/NomineeTracker.java old mode 100644 new mode 100755 similarity index 92% rename from src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java rename to src/main/java/net/helix/hlx/service/nominee/NomineeTracker.java index fcb83a43..e4755874 --- a/src/main/java/net/helix/hlx/service/milestone/NomineeTracker.java +++ b/src/main/java/net/helix/hlx/service/nominee/NomineeTracker.java @@ -1,4 +1,4 @@ -package net.helix.hlx.service.milestone; +package net.helix.hlx.service.nominee; import net.helix.hlx.model.Hash; @@ -26,4 +26,4 @@ public interface NomineeTracker { void shutdown(); -} +} \ No newline at end of file diff --git a/src/main/java/net/helix/hlx/service/nominee/NomineeValidity.java b/src/main/java/net/helix/hlx/service/nominee/NomineeValidity.java new file mode 100755 index 00000000..e024dafe --- /dev/null +++ b/src/main/java/net/helix/hlx/service/nominee/NomineeValidity.java @@ -0,0 +1,10 @@ +package net.helix.hlx.service.nominee; + +/** + * Validity states of candidate transactions that are used to express their "relevance" for the curator. + */ +public enum NomineeValidity { + VALID, + INVALID, + INCOMPLETE +} diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java new file mode 100755 index 00000000..d1a9a95f --- /dev/null +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java @@ -0,0 +1,100 @@ +package net.helix.hlx.service.nominee.impl; + +import net.helix.hlx.BundleValidator; +import net.helix.hlx.conf.HelixConfig; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.crypto.Merkle; +import net.helix.hlx.crypto.SpongeFactory; +import net.helix.hlx.model.Hash; +import net.helix.hlx.service.nominee.NomineeValidity; +import net.helix.hlx.service.nominee.NomineeException; +import net.helix.hlx.service.nominee.NomineeService; +import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.service.snapshot.SnapshotService; +import net.helix.hlx.storage.Tangle; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + + +import static net.helix.hlx.service.nominee.NomineeValidity.*; + +public class NomineeServiceImpl implements NomineeService { + + private final static Logger log = LoggerFactory.getLogger(NomineeServiceImpl.class); + + private Tangle tangle; + + private SnapshotProvider snapshotProvider; + + private SnapshotService snapshotService; + + private HelixConfig config; + + public NomineeServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotService snapshotService, HelixConfig config) { + + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.snapshotService = snapshotService; + this.config = config; + + return this; + } + + @Override + public NomineeValidity validateNominees(TransactionViewModel transactionViewModel, int roundIndex, + SpongeFactory.Mode mode, int securityLevel) throws NomineeException { + + if (roundIndex < 0 || roundIndex >= 0x200000) { + return INVALID; + } + + try { + + final List> bundleTransactions = BundleValidator.validate(tangle, + snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); + + if (bundleTransactions.isEmpty()) { + return INCOMPLETE; + } else { + for (final List bundleTransactionViewModels : bundleTransactions) { + final TransactionViewModel tail = bundleTransactionViewModels.get(0); // transaction with signature + if (tail.getHash().equals(transactionViewModel.getHash())) { + + // validate signature + Hash senderAddress = tail.getAddressHash(); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getCuratorKeyDepth()); + //System.out.println("valid signature (nominee): " + validSignature); + + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getCuratorAddress().equals(senderAddress) && validSignature)) { + return VALID; + } else { + return INVALID; + } + } + } + } + } catch (Exception e) { + throw new NomineeException("error while validating nominees for round #" + roundIndex, e); + } + + return INVALID; + } + + + private boolean isNomineeBundleStructureValid(List bundleTransactions, int securityLevel) { + if (bundleTransactions.size() <= securityLevel) { + return false; + } + + Hash headTransactionHash = bundleTransactions.get(securityLevel).getTrunkTransactionHash(); + return bundleTransactions.stream() + .limit(securityLevel) + .map(TransactionViewModel::getBranchTransactionHash) + .allMatch(branchTransactionHash -> branchTransactionHash.equals(headTransactionHash)); + } + + +} diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeSolidifierImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeSolidifierImpl.java new file mode 100755 index 00000000..7c3ba93b --- /dev/null +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeSolidifierImpl.java @@ -0,0 +1,178 @@ +package net.helix.hlx.service.nominee.impl; + +import net.helix.hlx.TransactionValidator; +import net.helix.hlx.model.Hash; +import net.helix.hlx.service.nominee.NomineeSolidifier; +import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.utils.log.interval.IntervalLogger; +import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; +import net.helix.hlx.utils.thread.SilentScheduledExecutorService; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + + +public class NomineeSolidifierImpl implements NomineeSolidifier { + + private static final int SOLIDIFICATION_QUEUE_SIZE = 2; + + private static final int SOLIDIFICATION_INTERVAL = 5000; + + private static final int SOLIDIFICATION_TRANSACTIONS_LIMIT = 50000; + + private static final IntervalLogger log = new IntervalLogger(NomineeSolidifier.class); + + private SnapshotProvider snapshotProvider; + + private TransactionValidator transactionValidator; + + private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Nominee Solidifier", log.delegate()); + + private final Map newlyAddedCandidates = new ConcurrentHashMap<>(); + + private final Map unsolidCandidatesPool = new ConcurrentHashMap<>(); + + private final Map candidatesToSolidify = new HashMap<>(); + + private Map.Entry youngestCandidateInQueue = null; + + public NomineeSolidifierImpl init(SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { + this.snapshotProvider = snapshotProvider; + this.transactionValidator = transactionValidator; + + return this; + } + + @Override + public void add(Hash candidateHash, int roundIndex) { + if (!unsolidCandidatesPool.containsKey(candidateHash) && !newlyAddedCandidates.containsKey(candidateHash) && + roundIndex > snapshotProvider.getInitialSnapshot().getIndex()) { + + newlyAddedCandidates.put(candidateHash, roundIndex); + } + } + + @Override + public void start() { + executorService.silentScheduleWithFixedDelay(this::candidateSolidificationThread, 0, SOLIDIFICATION_INTERVAL, + TimeUnit.MILLISECONDS); + } + + @Override + public void shutdown() { + executorService.shutdownNow(); + } + + private void addToSolidificationQueue(Map.Entry candidateEntry) { + if (candidatesToSolidify.containsKey(candidateEntry.getKey())) { + return; + } + + if (candidatesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE) { + candidatesToSolidify.put(candidateEntry.getKey(), candidateEntry.getValue()); + + if (youngestCandidateInQueue == null || candidateEntry.getValue() > youngestCandidateInQueue.getValue()) { + youngestCandidateInQueue = candidateEntry; + } + } else if (candidateEntry.getValue() < youngestCandidateInQueue.getValue()) { + candidatesToSolidify.remove(youngestCandidateInQueue.getKey()); + candidatesToSolidify.put(candidateEntry.getKey(), candidateEntry.getValue()); + + determineYoungestCandidateInQueue(); + } + } + + private void candidateSolidificationThread() { + processNewlyAddedCandidates(); + processSolidificationQueue(); + refillSolidificationQueue(); + } + + private void processNewlyAddedCandidates() { + for (Iterator> iterator = newlyAddedCandidates.entrySet().iterator(); + !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + + Map.Entry currentEntry = iterator.next(); + + unsolidCandidatesPool.put(currentEntry.getKey(), currentEntry.getValue()); + + if (youngestCandidateInQueue == null || currentEntry.getValue() < youngestCandidateInQueue.getValue()) { + addToSolidificationQueue(currentEntry); + } + + iterator.remove(); + } + } + + private void processSolidificationQueue() { + for (Iterator> iterator = candidatesToSolidify.entrySet().iterator(); + !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + + Map.Entry currentEntry = iterator.next(); + + if (currentEntry.getValue() <= snapshotProvider.getInitialSnapshot().getIndex() || isSolid(currentEntry)) { + unsolidCandidatesPool.remove(currentEntry.getKey()); + iterator.remove(); + + if (youngestCandidateInQueue != null && + currentEntry.getKey().equals(youngestCandidateInQueue.getKey())) { + + youngestCandidateInQueue = null; + } + } + } + } + + private void refillSolidificationQueue() { + if(youngestCandidateInQueue == null && !candidatesToSolidify.isEmpty()) { + determineYoungestCandidateInQueue(); + } + + Map.Entry nextSolidificationCandidate; + while (!Thread.currentThread().isInterrupted() && candidatesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE && + (nextSolidificationCandidate = getNextSolidificationCandidate()) != null) { + + addToSolidificationQueue(nextSolidificationCandidate); + } + } + + private void determineYoungestCandidateInQueue() { + youngestCandidateInQueue = null; + for (Map.Entry currentEntry : candidatesToSolidify.entrySet()) { + if (youngestCandidateInQueue == null || currentEntry.getValue() > youngestCandidateInQueue.getValue()) { + youngestCandidateInQueue = currentEntry; + } + } + } + + private Map.Entry getNextSolidificationCandidate() { + Map.Entry nextSolidificationCandidate = null; + for (Map.Entry candidateEntry : unsolidCandidatesPool.entrySet()) { + if (!candidatesToSolidify.containsKey(candidateEntry.getKey()) && (nextSolidificationCandidate == null || + candidateEntry.getValue() < nextSolidificationCandidate.getValue())) { + + nextSolidificationCandidate = candidateEntry; + } + } + + return nextSolidificationCandidate; + } + + private boolean isSolid(Map.Entry currentEntry) { + if (unsolidCandidatesPool.size() > 1) { + log.info("Solidifying candidate #" + currentEntry.getValue() + + " [" + candidatesToSolidify.size() + " / " + unsolidCandidatesPool.size() + "]"); + } + + try { + return transactionValidator.checkSolidity(currentEntry.getKey(), true, + SOLIDIFICATION_TRANSACTIONS_LIMIT); + } catch (Exception e) { + log.error("Error while solidifying candidate #" + currentEntry.getValue(), e); + + return false; + } + } +} diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java old mode 100644 new mode 100755 similarity index 74% rename from src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java rename to src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java index 9df1b5b6..fe604ad3 --- a/src/main/java/net/helix/hlx/service/milestone/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java @@ -1,4 +1,4 @@ -package net.helix.hlx.service.milestone.impl; +package net.helix.hlx.service.nominee.impl; import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.BundleValidator; @@ -8,19 +8,14 @@ import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; -import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.HashFactory; -import net.helix.hlx.service.milestone.*; +import net.helix.hlx.service.nominee.*; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; import net.helix.hlx.utils.thread.SilentScheduledExecutorService; -import static net.helix.hlx.service.milestone.MilestoneValidity.INCOMPLETE; -import static net.helix.hlx.service.milestone.MilestoneValidity.INVALID; -import static net.helix.hlx.service.milestone.MilestoneValidity.VALID; - import java.util.*; import java.util.concurrent.TimeUnit; @@ -38,22 +33,22 @@ public class NomineeTrackerImpl implements NomineeTracker { private Tangle tangle; private HelixConfig config; private SnapshotProvider snapshotProvider; - private MilestoneService milestoneService; - private MilestoneSolidifier milestoneSolidifier; + private NomineeService nomineeService; + private NomineeSolidifier nomineeSolidifier; private Set latestNominees; private Hash latestNomineeHash; private int startRound; private final Set seenCuratorTransactions = new HashSet<>(); private final Deque curatorTransactionsToAnalyze = new ArrayDeque<>(); - public NomineeTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, HelixConfig config) { + public NomineeTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, NomineeService nomineeService, NomineeSolidifier nomineeSolidifier, HelixConfig config) { this.tangle = tangle; this.config = config; this.Curator_Address = config.getCuratorAddress(); this.snapshotProvider = snapshotProvider; - this.milestoneService = milestoneService; - this.milestoneSolidifier = milestoneSolidifier; + this.nomineeService = nomineeService; + this.nomineeSolidifier = nomineeSolidifier; this.latestNominees = config.getInitialNominees(); startRound = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration() + 2; @@ -94,45 +89,6 @@ public Set getNomineesOfRound(int roundIndex) throws Exception{ } - public MilestoneValidity validateNominees(TransactionViewModel transactionViewModel, int roundIndex, - SpongeFactory.Mode mode, int securityLevel) throws Exception { - - if (roundIndex < 0 || roundIndex >= 0x200000) { - return INVALID; - } - - try { - - final List> bundleTransactions = BundleValidator.validate(tangle, - snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); - - if (bundleTransactions.isEmpty()) { - return INCOMPLETE; - } else { - for (final List bundleTransactionViewModels : bundleTransactions) { - final TransactionViewModel tail = bundleTransactionViewModels.get(0); // transaction with signature - if (tail.getHash().equals(transactionViewModel.getHash())) { - - // validate signature - Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getCuratorKeyDepth()); - //System.out.println("valid signature (nominee): " + validSignature); - - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (Curator_Address.equals(senderAddress) && validSignature)) { - return VALID; - } else { - return INVALID; - } - } - } - } - } catch (Exception e) { - throw new MilestoneException("error while validating nominees for round #" + roundIndex, e); - } - - return INVALID; - } - @Override public boolean processNominees(Hash transactionHash) throws Exception { TransactionViewModel transaction = TransactionViewModel.fromHash(tangle, transactionHash); @@ -149,7 +105,7 @@ public boolean processNominees(Hash transactionHash) throws Exception { } // validate - switch (validateNominees(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { + switch (nomineeService.validateNominees(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { case VALID: log.info("Nominee Transaction " + transaction.getHash() + " is VALID"); //System.out.println("round index: " + roundIndex); @@ -163,14 +119,14 @@ public boolean processNominees(Hash transactionHash) throws Exception { } if (!transaction.isSolid()) { - milestoneSolidifier.add(transaction.getHash(), roundIndex); + nomineeSolidifier.add(transaction.getHash(), roundIndex); } break; case INCOMPLETE: log.info("Nominee Transaction " + transaction.getHash() + " is INCOMPLETE"); - milestoneSolidifier.add(transaction.getHash(), roundIndex); + nomineeSolidifier.add(transaction.getHash(), roundIndex); return false; default: @@ -262,4 +218,4 @@ public void start(){ public void shutdown(){ executorService.shutdownNow(); } -} +} \ No newline at end of file From 269a8a74b108438a62535bd68c02b0dcb7eb7858 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 9 Aug 2019 13:04:41 +0200 Subject: [PATCH 164/223] don't use publish --- src/main/java/net/helix/hlx/service/API.java | 14 ++++++-------- .../hlx/service/curator/impl/NomineePublisher.java | 3 ++- .../service/milestone/impl/MilestonePublisher.java | 6 ++++-- .../net/helix/hlx/utils/bundle/BundleUtils.java | 12 ++++++------ 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 04cf1c58..8b085d49 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1531,8 +1531,6 @@ public void publish(BundleTypes type, final String address, final int minWeightM publishRegistration(address, minWeightMagnitude, sign, keyIndex, maxKeyIndex, join); case nominee: publishNominees(startRoundDelay, minWeightMagnitude, sign, keyIndex, maxKeyIndex); - default: - publishMilestone(address, minWeightMagnitude, sign, keyIndex, maxKeyIndex); } } @@ -1545,13 +1543,13 @@ public void publish(BundleTypes type, final String address, final int minWeightM * @param sign whether to sign * @param keyIdx key index * @param maxKeyIdx maximum key index - * @param tips tips + * @param data tips * @param txToApprove transactions to approve * @throws Exception if storing fails */ - private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, byte[] tips, List txToApprove) throws Exception { + private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx) throws Exception { BundleUtils bundle = new BundleUtils(sndAddr, rcvAddr); - bundle.create(tips, tag, sign, keyIdx, maxKeyIdx); + bundle.create(data, tag, sign, keyIdx, maxKeyIdx); storeAndBroadcast(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); } @@ -1571,7 +1569,7 @@ public void publishMilestone(final String address, final int minWeightMagnitude, byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, tipsBytes, txToApprove); + storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex); } public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { @@ -1585,7 +1583,7 @@ public void publishRegistration(final String address, final int minWeightMagnitu } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, data, txToApprove); + storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex); } public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1602,7 +1600,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, nomineeBytes, txToApprove); + storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, txToApprove, nomineeBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex); } // diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index a5905546..5a152cff 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -45,7 +45,8 @@ private void UpdateNominees() throws Exception { log.debug("Publishing new Nominees..."); //api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth())); //todo remove after refactoring // (BundleTypes type, final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join, int startRoundDelay) - api.publish(BundleTypes.nominee, Hash.NULL_HASH.toString(), mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth()), false, startRoundDelay); + //api.publish(BundleTypes.nominee, Hash.NULL_HASH.toString(), mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth()), false, startRoundDelay); + api.publishNominees(startRoundDelay, mwm, sign, currentKeyIndex, (int) Math.pow(2, config.getCuratorKeyDepth())); currentKeyIndex += 1; } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index 9ddb7f3e..acaa643c 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -99,7 +99,8 @@ private void generateKeyfile(String seed) throws Exception { private void sendRegistration(Hash identity, boolean join) throws Exception { log.debug("Signing {} identity: {} ", (join ? "up" : "off"), identity); //api.publishRegistration(identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join); //todo remove when done with refactoring - api.publish(BundleTypes.registration, identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join, 0); + //api.publish(BundleTypes.registration, identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join, 0); + api.publishRegistration(identity.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, join); currentKeyIndex += 1; } @@ -146,7 +147,8 @@ private void publishMilestone() throws Exception { log.debug("Publishing next Milestone..."); if (currentKeyIndex < maxKeyIndex * (keyfileIndex + 1) - 1) { //api.publishMilestone(address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex); <- todo remove when refactoring is done - api.publish(BundleTypes.milestone, address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, false, 0); + //api.publish(BundleTypes.milestone, address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, false, 0); + api.publishMilestone(address.toString(), mwm, sign, currentKeyIndex, maxKeyIndex); currentKeyIndex += 1; } else { log.debug("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network."); diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index 70dc253c..c4fe455c 100644 --- a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -53,25 +53,25 @@ public List getTransactions() { /** * Method for generating bundles - * @param tips tips - * @param roundIndex round index + * @param data signature message fragment + * @param tag round index, start round or join/leave * @param sign whether to sign * @param keyIndex key index * @param maxKeyIndex maximum key index */ - public void create(byte[] tips, long roundIndex, Boolean sign, int keyIndex, int maxKeyIndex) { + public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKeyIndex) { // get number of transactions needed for tips - int n = (tips.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; + int n = (data.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; // pad data to mutiple of smf byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - System.arraycopy(tips, 0, paddedData, 0, tips.length); + System.arraycopy(data, 0, paddedData, 0, data.length); long timestamp = System.currentTimeMillis() / 1000L; int lastIndex = 1 + n; // contain a signature that signs the siblings and thereby ensures the integrity. - this.senderTransaction = initTransaction(this.senderAddress, 0, lastIndex, timestamp, roundIndex); + this.senderTransaction = initTransaction(this.senderAddress, 0, lastIndex, timestamp, tag); // siblings for merkle tree. this.merkleTransaction = initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); From b716add91ea87371adf8ccd43792372537402882 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 11 Aug 2019 15:05:02 +0200 Subject: [PATCH 165/223] make path to keyfile variable --- src/main/java/net/helix/hlx/service/API.java | 18 ++++++------------ .../helix/hlx/utils/bundle/BundleUtils.java | 4 ++-- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 8b085d49..7f9e3b5a 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1547,9 +1547,9 @@ public void publish(BundleTypes type, final String address, final int minWeightM * @param txToApprove transactions to approve * @throws Exception if storing fails */ - private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx) throws Exception { + private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, String keyfile) throws Exception { BundleUtils bundle = new BundleUtils(sndAddr, rcvAddr); - bundle.create(data, tag, sign, keyIdx, maxKeyIdx); + bundle.create(data, tag, sign, keyIdx, maxKeyIdx, keyfile); storeAndBroadcast(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); } @@ -1569,21 +1569,15 @@ public void publishMilestone(final String address, final int minWeightMagnitude, byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex); + storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, "./src/main/resources/Nominee.key"); } public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { byte[] data = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - List txToApprove = new ArrayList<>(); - if(RoundViewModel.latest(tangle) == null) { - txToApprove.add(Hash.NULL_HASH); - txToApprove.add(Hash.NULL_HASH); - } else { - txToApprove = getTransactionToApproveTips(3, Optional.empty()); - } - storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex); + List txToApprove = getTransactionToApproveTips(3, Optional.empty()); + storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, "./src/main/resources/Nominee.key"); } public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1600,7 +1594,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, txToApprove, nomineeBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex); + storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, txToApprove, nomineeBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, "./src/main/resources/Coordinator.key"); } // diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index c4fe455c..bec15954 100644 --- a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -59,7 +59,7 @@ public List getTransactions() { * @param keyIndex key index * @param maxKeyIndex maximum key index */ - public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKeyIndex) { + public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKeyIndex, String keyfile) { // get number of transactions needed for tips int n = (data.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; @@ -94,7 +94,7 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey // sign bundle if (sign) { try { - signBundle("./src/main/resources/Nominee.key", this.merkleTransaction, this.senderTransaction, bundleHash, keyIndex, maxKeyIndex); + signBundle(keyfile, this.merkleTransaction, this.senderTransaction, bundleHash, keyIndex, maxKeyIndex); } catch (IOException e) { log.error("Cannot read keyfile", e); } From 5742acf60be3eb5852ec2329f146f9907a258995 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 11 Aug 2019 15:06:13 +0200 Subject: [PATCH 166/223] implement is_BundleStructureValid --- .../curator/impl/CandidateTrackerImpl.java | 1 - .../curator/impl/CuratorServiceImpl.java | 32 ++++++++++++------- .../milestone/impl/MilestonePublisher.java | 2 +- .../milestone/impl/MilestoneServiceImpl.java | 16 ++++++---- .../nominee/impl/NomineeServiceImpl.java | 31 ++++++++++-------- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index 3db4588a..c6fbff90 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -251,7 +251,6 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator switch (curatorService.validateCandidate(tail, SpongeFactory.Mode.S256, 1)) { case VALID: log.info("Candidate Transaction " + transaction.getHash() + " is VALID, Address: " + tail.getAddressHash()); - log.info("Join/Leave: " + RoundViewModel.getRoundIndex(tail)); if (RoundViewModel.getRoundIndex(tail) == 1) { addToNomineeQueue(tail.getAddressHash()); } diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java index b486faf7..8266f7a5 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java @@ -18,6 +18,7 @@ import java.security.SecureRandom; import java.util.*; +import java.util.stream.Collectors; import static net.helix.hlx.service.curator.CandidateValidity.*; @@ -74,17 +75,17 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM final TransactionViewModel tail = bundleTransactionViewModels.get(0); if (tail.getHash().equals(transactionViewModel.getHash())) { - //todo implement when sure how bundle structure has to look like - //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { + if (isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { - Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); - System.out.println("valid signature (candidate): " + validSignature); + Hash senderAddress = tail.getAddressHash(); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); + //System.out.println("valid signature (candidate): " + validSignature); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { - return VALID; - } else { - return INVALID; + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { + return VALID; + } else { + return INVALID; + } } } } @@ -111,13 +112,20 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM * @return {@code true} if the basic structure is valid and {@code false} otherwise */ private boolean isCandidateBundleStructureValid(List bundleTransactions, int securityLevel) { - if (bundleTransactions.size() <= securityLevel) { + int lastIdx = securityLevel + 1; + if (bundleTransactions.size() <= lastIdx) { + System.out.println("Candidate bundle has not enough transactions"); return false; } - Hash headTransactionHash = bundleTransactions.get(securityLevel).getTrunkTransactionHash(); + Hash headTransactionHash = bundleTransactions.get(lastIdx).getTrunkTransactionHash(); + System.out.println("Trunk of head: " + headTransactionHash); + System.out.println("Branch of head: " + bundleTransactions.get(lastIdx).getBranchTransactionHash()); + List branch = bundleTransactions.stream() + .map(TransactionViewModel::getBranchTransactionHash).collect(Collectors.toList()); + System.out.println("Branch of all: " + branch); return bundleTransactions.stream() - .limit(securityLevel) + .limit(lastIdx) .map(TransactionViewModel::getBranchTransactionHash) .allMatch(branchTransactionHash -> branchTransactionHash.equals(headTransactionHash)); } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index acaa643c..465c61c2 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -52,7 +52,7 @@ public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nom mwm = config.getMwm(); message = StringUtils.repeat('0', 1024); sign = !config.isDontValidateTestnetMilestoneSig(); - pubkeyDepth = config.getNumberOfKeysInMilestone(); + pubkeyDepth = config.getMilestoneKeyDepth(); keyfileIndex = 0; maxKeyIndex = (int) Math.pow(2, pubkeyDepth); currentKeyIndex = 0; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 723ce07d..0c4b08a7 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; import static net.helix.hlx.service.milestone.MilestoneValidity.*; @@ -180,8 +181,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM final TransactionViewModel tail = bundleTransactionViewModels.get(0); // milestone transaction with signature if (tail.getHash().equals(transactionViewModel.getHash())) { - //todo implement when sure how bundle structure has to look like - //if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { + if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); @@ -211,6 +211,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM } else { return INVALID; } + } } } } @@ -500,13 +501,16 @@ private void patchSolidEntryPointsIfNecessary(Snapshot initialSnapshot, Transact * @return {@code true} if the basic structure is valid and {@code false} otherwise */ private boolean isMilestoneBundleStructureValid(List bundleTransactions, int securityLevel) { - if (bundleTransactions.size() <= securityLevel) { + int lastIdx = bundleTransactions.size() - 1; + + // length is variable dependent on how many tips will be confirmed so this can't be checked + /*if (bundleTransactions.size() <= securityLevel) { return false; - } + }*/ - Hash headTransactionHash = bundleTransactions.get(securityLevel).getTrunkTransactionHash(); + Hash headTransactionHash = bundleTransactions.get(lastIdx).getTrunkTransactionHash(); return bundleTransactions.stream() - .limit(securityLevel) + .limit(lastIdx) .map(TransactionViewModel::getBranchTransactionHash) .allMatch(branchTransactionHash -> branchTransactionHash.equals(headTransactionHash)); } diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java index d1a9a95f..4f814d34 100755 --- a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeServiceImpl.java @@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; import static net.helix.hlx.service.nominee.NomineeValidity.*; @@ -63,15 +64,18 @@ public NomineeValidity validateNominees(TransactionViewModel transactionViewMode final TransactionViewModel tail = bundleTransactionViewModels.get(0); // transaction with signature if (tail.getHash().equals(transactionViewModel.getHash())) { - // validate signature - Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getCuratorKeyDepth()); - //System.out.println("valid signature (nominee): " + validSignature); + if (isNomineeBundleStructureValid(bundleTransactionViewModels, securityLevel)) { - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getCuratorAddress().equals(senderAddress) && validSignature)) { - return VALID; - } else { - return INVALID; + // validate signature + Hash senderAddress = tail.getAddressHash(); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getCuratorKeyDepth()); + //System.out.println("valid signature (nominee): " + validSignature); + + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getCuratorAddress().equals(senderAddress) && validSignature)) { + return VALID; + } else { + return INVALID; + } } } } @@ -85,16 +89,17 @@ public NomineeValidity validateNominees(TransactionViewModel transactionViewMode private boolean isNomineeBundleStructureValid(List bundleTransactions, int securityLevel) { - if (bundleTransactions.size() <= securityLevel) { + // todo maybe variable if there can be more than 16 nominees + int lastIdx = securityLevel + 1; + + if (bundleTransactions.size() <= lastIdx) { return false; } - Hash headTransactionHash = bundleTransactions.get(securityLevel).getTrunkTransactionHash(); + Hash headTransactionHash = bundleTransactions.get(lastIdx).getTrunkTransactionHash(); return bundleTransactions.stream() - .limit(securityLevel) + .limit(lastIdx) .map(TransactionViewModel::getBranchTransactionHash) .allMatch(branchTransactionHash -> branchTransactionHash.equals(headTransactionHash)); } - - } From 300e9eb10ee6a1711a8d6e1cadae88274cbf9e50 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 11 Aug 2019 16:41:27 +0200 Subject: [PATCH 167/223] allow referencing null hash --- src/main/java/net/helix/hlx/service/API.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 7f9e3b5a..accef70c 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1576,7 +1576,13 @@ public void publishRegistration(final String address, final int minWeightMagnitu byte[] data = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; - List txToApprove = getTransactionToApproveTips(3, Optional.empty()); + List txToApprove = new ArrayList<>(); + if(RoundViewModel.latest(tangle) == null) { + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); + } else { + txToApprove = getTransactionToApproveTips(3, Optional.empty()); + } storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, "./src/main/resources/Nominee.key"); } From aa8af236ac561cb1d21d2646c864ebf654fd2afd Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 11 Aug 2019 16:43:15 +0200 Subject: [PATCH 168/223] remove belowMaxDepth check --- .../hlx/service/tipselection/impl/WalkValidatorImpl.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java index 89b4e73f..72e2e5dc 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java @@ -68,15 +68,17 @@ public boolean isValid(Hash transactionHash) throws Exception { } else if (!transactionViewModel.isSolid()) { log.debug("Validation failed: {} is not solid", transactionHash.toString()); return false; - } else if (belowMaxDepth(transactionViewModel.getHash(), + } + // todo do we need this? + /*else if (belowMaxDepth(transactionViewModel.getHash(), snapshotProvider.getLatestSnapshot().getIndex() - config.getMaxDepth())) { log.debug("Validation failed: {} is below max depth", transactionHash.toString()); return false; - } else if (!ledgerService.updateDiff(myApprovedHashes, myDiff, transactionViewModel.getHash())) { + }*/ + else if (!ledgerService.updateDiff(myApprovedHashes, myDiff, transactionViewModel.getHash())) { log.debug("Validation failed: {} is not consistent", transactionHash.toString()); return false; } - //TODO fix bundle validation else if (!ledgerService.isBalanceDiffConsistent(myApprovedHashes, myDiff, transactionViewModel.getHash())) { log.debug("Validation failed: {} is not consistent", transactionHash.toString()); return false; From cc2fe0f973eda3bc97c9d60d1df738f584124a51 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 11 Aug 2019 16:44:08 +0200 Subject: [PATCH 169/223] comment bugs in milestonePublisher --- .../helix/hlx/service/milestone/impl/MilestonePublisher.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index 465c61c2..57bca851 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -132,6 +132,10 @@ public void startScheduledExecutorService() { scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), delay, TimeUnit.MILLISECONDS); } + //todo here are some bugs: + // - sometimes it passes the start round and is stucked in "Legitimized nominee .." + // - sometimes it is active and prints "Publishing next Milestone..." but then nothing happens (also doesn't build a new keyfile) + // - when starting with two nodes and one node leaving this deactivates the publisher private void publishMilestone() throws Exception { if (!active) { if (startRound < getRound(System.currentTimeMillis()) && !nomineeTracker.getLatestNominees().isEmpty() && nomineeTracker.getLatestNominees().contains(address)) { From 3ca66db03f4dc1aacb9ca02a2b39f553f640b664 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 11 Aug 2019 16:46:40 +0200 Subject: [PATCH 170/223] isCandidateBundleStructureValid does not work --- .../helix/hlx/service/curator/impl/CuratorServiceImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java index 8266f7a5..8022ac44 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java @@ -75,7 +75,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM final TransactionViewModel tail = bundleTransactionViewModels.get(0); if (tail.getHash().equals(transactionViewModel.getHash())) { - if (isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { + //if (isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); @@ -86,7 +86,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM } else { return INVALID; } - } + //} } } } @@ -118,6 +118,7 @@ private boolean isCandidateBundleStructureValid(List bundl return false; } + // todo head trunk and branch of the rest of the tx are different Hash headTransactionHash = bundleTransactions.get(lastIdx).getTrunkTransactionHash(); System.out.println("Trunk of head: " + headTransactionHash); System.out.println("Branch of head: " + bundleTransactions.get(lastIdx).getBranchTransactionHash()); From d822fb177bf90e111f401258a178741cab32d26f Mon Sep 17 00:00:00 2001 From: fsbbn Date: Sun, 11 Aug 2019 16:47:30 +0200 Subject: [PATCH 171/223] typo --- .../nominee/impl/NomineeTrackerImpl.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java index fe604ad3..57a21586 100755 --- a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java @@ -69,11 +69,13 @@ public Hash getLatestNomineeHash() { } @Override - public int getStartRound() {return startRound; } + public int getStartRound() { + return startRound; + } @Override - public Set getNomineesOfRound(int roundIndex) throws Exception{ + public Set getNomineesOfRound(int roundIndex) throws Exception { try { Set validators = new HashSet<>(); for (Hash hash : AddressViewModel.load(tangle, Curator_Address).getHashes()) { @@ -83,7 +85,7 @@ public Set getNomineesOfRound(int roundIndex) throws Exception{ } } return validators; - }catch (Exception e) { + } catch (Exception e) { throw new Exception("unexpected error while getting Validators of round #{}" + roundIndex, e); } } @@ -131,7 +133,7 @@ public boolean processNominees(Hash transactionHash) throws Exception { default: } - }else { + } else { return false; } return true; @@ -141,7 +143,7 @@ public boolean processNominees(Hash transactionHash) throws Exception { } @Override - public Set getNomineeAddresses(Hash transaction) throws Exception{ + public Set getNomineeAddresses(Hash transaction) throws Exception { TransactionViewModel tail = TransactionViewModel.fromHash(tangle, transaction); BundleViewModel bundle = BundleViewModel.load(tangle, tail.getBundleHash()); int security = 1; @@ -160,7 +162,7 @@ public Set getNomineeAddresses(Hash transaction) throws Exception{ Hash address = HashFactory.ADDRESS.create(tx.getSignature(), i * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); Hash null_address = HashFactory.ADDRESS.create("0000000000000000000000000000000000000000000000000000000000000000"); // TODO: Do we send all validators or only adding and removing ones ? - if (address.equals(null_address)){ + if (address.equals(null_address)) { return validators; } validators.add(address); @@ -179,7 +181,7 @@ public void analyzeCuratorTransactions() throws Exception { } Hash trusteeTransactionHash = curatorTransactionsToAnalyze.pollFirst(); - if(!processNominees(trusteeTransactionHash)) { + if (!processNominees(trusteeTransactionHash)) { seenCuratorTransactions.remove(trusteeTransactionHash); } } @@ -210,12 +212,12 @@ private void validatorTrackerThread() { } } - public void start(){ + public void start() { executorService.silentScheduleWithFixedDelay(this::validatorTrackerThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); } - public void shutdown(){ + public void shutdown() { executorService.shutdownNow(); } } \ No newline at end of file From e197b2632f813b9ba58c7bb9e854114135777fe7 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 13 Aug 2019 11:44:59 +0200 Subject: [PATCH 172/223] fixing local snapshots --- .../helix/hlx/controllers/RoundViewModel.java | 6 +++ src/main/java/net/helix/hlx/service/API.java | 3 +- .../impl/SeenMilestonesRetrieverImpl.java | 20 ++++------ .../service/snapshot/SnapshotMetaData.java | 4 +- .../hlx/service/snapshot/SnapshotService.java | 2 +- .../service/snapshot/impl/SnapshotImpl.java | 4 +- .../snapshot/impl/SnapshotMetaDataImpl.java | 12 +++--- .../snapshot/impl/SnapshotProviderImpl.java | 18 ++++----- .../snapshot/impl/SnapshotServiceImpl.java | 39 +++++++------------ 9 files changed, 50 insertions(+), 58 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index b1e2d2e7..72c5bc23 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -437,6 +437,12 @@ public Integer index() { return round.index.getValue(); } + public Hash getMerkleRoot() { + List> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes())); + Hash root = merkleTree.get(merkleTree.size()-1).get(0); + return root; + } + /** * Removes the {@link Round} object from the database. * diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index accef70c..24acee49 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1647,8 +1647,7 @@ public List addMilestoneReferences(List confirmedTips, int roundInde if (previousRound == null){ txToApprove.add(Hash.NULL_HASH); } else { - List> merkleTreeMilestones = Merkle.buildMerkleTree(new ArrayList(previousRound.getHashes())); - txToApprove.add(merkleTreeMilestones.get(merkleTreeMilestones.size() - 1).get(0)); // merkle root of latest milestones + txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones } //branch List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java index d116a4a0..4a1281c5 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java @@ -11,11 +11,9 @@ import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; import net.helix.hlx.utils.thread.SilentScheduledExecutorService; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.LinkedList; -import java.util.List; +import java.util.Map; /** * Creates a manager that proactively requests the missing "seen milestones" (defined in the local snapshot file).
@@ -68,7 +66,7 @@ public class SeenMilestonesRetrieverImpl implements SeenMilestonesRetriever { /** * The list of seen milestones that need to be requested.
*/ - private List seenMilestones; + private Map seenRounds; /** * This method initializes the instance and registers its dependencies.
@@ -95,7 +93,7 @@ public SeenMilestonesRetrieverImpl init(Tangle tangle, SnapshotProvider snapshot this.snapshotProvider = snapshotProvider; this.transactionRequester = transactionRequester; - seenMilestones = new LinkedList<>(snapshotProvider.getInitialSnapshot().getSeenRounds()); + seenRounds = new ConcurrentHashMap<>(snapshotProvider.getInitialSnapshot().getSeenRounds()); return this; } @@ -115,10 +113,10 @@ public SeenMilestonesRetrieverImpl init(Tangle tangle, SnapshotProvider snapshot */ @Override public void retrieveSeenMilestones() { - seenMilestones.forEach((roundIndex) -> { + seenRounds.forEach((roundIndex, merkleRoot) -> { try { if (roundIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { - seenMilestones.remove(roundIndex); + seenRounds.remove(roundIndex); } else if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() + RETRIEVE_RANGE) { RoundViewModel round = RoundViewModel.get(tangle, roundIndex); for (Hash milestoneHash : round.getHashes()) { @@ -131,18 +129,16 @@ public void retrieveSeenMilestones() { } // the transactionRequester will never drop milestone requests - we can therefore remove it from the // list of milestones to request - //todo here the roundIndex will probably interpreted as index and not as the Object to delete, which is wrong - //todo anyway it doesn't make sense to store Integers in a List (or does it?) - seenMilestones.remove(roundIndex); + seenRounds.remove(roundIndex); } - log.info("Requesting seen milestones (" + seenMilestones.size() + " left) ..."); + log.info("Requesting seen milestones (" + seenRounds.size() + " left) ..."); } catch (Exception e) { log.error("unexpected error while processing the seen milestones", e); } }); - if (seenMilestones.isEmpty()) { + if (seenRounds.isEmpty()) { log.info("Requesting seen milestones ... [DONE]"); shutdown(); diff --git a/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java b/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java index 6a9228b8..519fd467 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java +++ b/src/main/java/net/helix/hlx/service/snapshot/SnapshotMetaData.java @@ -190,7 +190,7 @@ public interface SnapshotMetaData { * * @return map of milestone transaction hashes associated to their milestone index */ - List getSeenRounds(); + Map getSeenRounds(); /** * Setter for the seen milestones. @@ -202,7 +202,7 @@ public interface SnapshotMetaData { * * @param seenRounds map of milestone transaction hashes associated to their milestone index */ - void setSeenRounds(List seenRounds); + void setSeenRounds(Map seenRounds); /** * Replaces the meta data values of this instance with the values of another meta data object. diff --git a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java index efd5862d..bc0254da 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java +++ b/src/main/java/net/helix/hlx/service/snapshot/SnapshotService.java @@ -107,6 +107,6 @@ Snapshot generateSnapshot(MilestoneTracker milestoneTracker, RoundViewModel targ * @return a map of solid entry points associating their hash to the milestone index that confirmed them * @throws SnapshotException if anything goes wrong while generating the solid entry points */ - List generateSeenRounds(MilestoneTracker milestoneTracker, + Map generateSeenRounds(MilestoneTracker milestoneTracker, RoundViewModel targetRound) throws SnapshotException; } diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java index c2b667d6..b22333fd 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotImpl.java @@ -418,7 +418,7 @@ public int getSolidEntryPointIndex(Hash solidEntrypoint) { * This is a thread-safe wrapper for the underlying {@link SnapshotMetaData} method. */ @Override - public List getSeenRounds() { + public Map getSeenRounds() { lockRead(); try { @@ -434,7 +434,7 @@ public List getSeenRounds() { * This is a thread-safe wrapper for the underlying {@link SnapshotMetaData} method. */ @Override - public void setSeenRounds(List seenRounds) { + public void setSeenRounds(Map seenRounds) { lockWrite(); try { diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java index 209e343d..7f0bde1d 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotMetaDataImpl.java @@ -51,7 +51,7 @@ public class SnapshotMetaDataImpl implements SnapshotMetaData { /** * Internal property for the value returned by {@link SnapshotMetaData#getSeenRounds()}. */ - private List seenRounds; + private Map seenRounds; /** * Creates a meta data object with the given information. @@ -66,7 +66,7 @@ public class SnapshotMetaDataImpl implements SnapshotMetaData { * @param seenRounds map of milestone transaction hashes associated to their milestone index */ public SnapshotMetaDataImpl(Hash hash, int index, Long timestamp, Map solidEntryPoints, - List seenRounds) { + Map seenRounds) { this.initialHash = hash; this.initialIndex = index; @@ -76,7 +76,7 @@ public SnapshotMetaDataImpl(Hash hash, int index, Long timestamp, Map(solidEntryPoints)); - setSeenRounds(new LinkedList<>(seenRounds)); + setSeenRounds(new HashMap<>(seenRounds)); } /** @@ -226,7 +226,7 @@ public int getSolidEntryPointIndex(Hash solidEntrypoint) { * {@inheritDoc} */ @Override - public List getSeenRounds() { + public Map getSeenRounds() { return seenRounds; } @@ -234,7 +234,7 @@ public List getSeenRounds() { * {@inheritDoc} */ @Override - public void setSeenRounds(List seenRounds) { + public void setSeenRounds(Map seenRounds) { this.seenRounds = seenRounds; } @@ -251,7 +251,7 @@ public void update(SnapshotMetaData newMetaData) { setHash(newMetaData.getHash()); setTimestamp(newMetaData.getTimestamp()); setSolidEntryPoints(new HashMap<>(newMetaData.getSolidEntryPoints())); - setSeenRounds(new LinkedList<>(newMetaData.getSeenRounds())); + setSeenRounds(new HashMap<>(newMetaData.getSeenRounds())); } @Override diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java index fe13f5aa..b5493459 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotProviderImpl.java @@ -299,7 +299,7 @@ private Snapshot loadBuiltInSnapshot() throws SnapshotException { config.getMilestoneStartIndex(), config.getSnapshotTime(), solidEntryPoints, - new LinkedList<>() + new HashMap<>() ) ); } @@ -423,7 +423,7 @@ private SnapshotMetaData readSnapshotMetaDatafromFile(File snapshotMetaDataFile) int amountOfSeenMilestones = readAmountOfSeenMilestonesFromMetaDataFile(reader); Map solidEntryPoints = readSolidEntryPointsFromMetaDataFile(reader, amountOfSolidEntryPoints); - List seenRounds = readSeenRoundsFromMetaDataFile(reader, amountOfSeenMilestones); + Map seenRounds = readSeenRoundsFromMetaDataFile(reader, amountOfSeenMilestones); return new SnapshotMetaDataImpl(hash, index, timestamp, solidEntryPoints, seenRounds); } catch (IOException e) { @@ -583,10 +583,10 @@ private Map readSolidEntryPointsFromMetaDataFile(BufferedReader r * @throws SnapshotException if anything goes wrong while reading the seen milestones from the file * @throws IOException if we could not read from the file */ - private List readSeenRoundsFromMetaDataFile(BufferedReader reader, int amountOfSeenRounds) + private Map readSeenRoundsFromMetaDataFile(BufferedReader reader, int amountOfSeenRounds) throws SnapshotException, IOException { - List seenRounds = new LinkedList<>(); + Map seenRounds = new HashMap<>(); for(int i = 0; i < amountOfSeenRounds; i++) { String line; @@ -597,7 +597,7 @@ private List readSeenRoundsFromMetaDataFile(BufferedReader reader, int String[] parts = line.split(";", 2); if(parts.length == 2) { try { - seenRounds.add(Integer.parseInt(parts[1])); + seenRounds.put(Integer.parseInt(parts[0]), HashFactory.TRANSACTION.create(parts[1])); } catch (NumberFormatException e) { throw new SnapshotException("could not parse a seen milestone from the metadata file", e); } @@ -624,7 +624,7 @@ private void writeSnapshotMetaDataToDisk(SnapshotMetaData snapshotMetaData, Stri try { Map solidEntryPoints = snapshotMetaData.getSolidEntryPoints(); - List seenMilestones = snapshotMetaData.getSeenRounds(); + Map seenMilestones = snapshotMetaData.getSeenRounds(); Files.write( Paths.get(filePath), @@ -641,10 +641,10 @@ private void writeSnapshotMetaDataToDisk(SnapshotMetaData snapshotMetaData, Stri .stream() .sorted(Map.Entry.comparingByValue()) .map(entry -> entry.getKey().toString() + ";" + entry.getValue()), - seenMilestones + seenMilestones.entrySet() .stream() - .sorted() - .map(entry -> entry.toString()) + .sorted(Map.Entry.comparingByKey()) + .map(entry -> entry.getKey().toString() + ";" + entry.getValue()) ) ).iterator() ); diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index 2a3b54a7..bfbc7f12 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -1,11 +1,10 @@ package net.helix.hlx.service.snapshot.impl; -import net.helix.hlx.conf.SnapshotConfig; +import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.ApproveeViewModel; import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.StateDiffViewModel; import net.helix.hlx.controllers.TransactionViewModel; -import net.helix.hlx.crypto.Merkle; import net.helix.hlx.model.Hash; import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.snapshot.*; @@ -75,7 +74,7 @@ public class SnapshotServiceImpl implements SnapshotService { /** * Holds the config with important snapshot specific settings.
*/ - private SnapshotConfig config; + private HelixConfig config; private SpentAddressesService spentAddressesService; @@ -100,7 +99,7 @@ public class SnapshotServiceImpl implements SnapshotService { */ public SnapshotServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesService spentAddressesService, SpentAddressesProvider spentAddressesProvider, - SnapshotConfig config) { + HelixConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; @@ -153,20 +152,12 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna snapshot.setIndex(lastAppliedRound.index()); //store merkle root - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(lastAppliedRound.getHashes())); - snapshot.setHash(merkleTree.get(merkleTree.size() - 1).get(0)); - //System.out.println("Milestones :"); - //lastAppliedRound.getHashes().forEach(m -> System.out.println(m.hexString())); + snapshot.setHash(lastAppliedRound.getMerkleRoot()); log.debug("Applying round #{}, snapshot hash: {} to ledger", lastAppliedRound.index(), snapshot.getHash()); - //System.out.println("Snapshot:"); - //snapshot.getBalances().forEach((a,b) -> System.out.println("Address: " + a.hexString() + ", " + b)); - //TODO: get start time of round (need genesisTime) - /*TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, - lastAppliedMilestone.getHash()); - if(milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT) { - snapshot.setTimestamp(milestoneTransaction.getTimestamp()); - }*/ + // start time of round + snapshot.setTimestamp(config.getGenesisTime() + (lastAppliedRound.index() * config.getRoundDuration())); + for (int skippedMilestoneIndex : skippedMilestones) { snapshot.addSkippedMilestone(skippedMilestoneIndex); @@ -295,20 +286,20 @@ public Map generateSolidEntryPoints(RoundViewModel targetMileston * {@inheritDoc} */ @Override - public List generateSeenRounds(MilestoneTracker milestoneTracker, + public Map generateSeenRounds(MilestoneTracker milestoneTracker, RoundViewModel targetRound) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [processing seen milestones]", log) .start(config.getLocalSnapshotsDepth()); - List seenRounds = new LinkedList<>(); + Map seenRounds = new HashMap<>(); try { RoundViewModel seenRound = targetRound; while ((seenRound = RoundViewModel.findClosestNextRound(tangle, seenRound.index(), milestoneTracker.getCurrentRoundIndex())) != null) { - seenRounds.add(seenRound.index()); + seenRounds.put(seenRound.index(), seenRound.getMerkleRoot()); progressLogger.progress(); } @@ -400,7 +391,7 @@ private boolean rollbackLastMilestone(Tangle tangle, Snapshot snapshot) throws S /** * This method determines the milestone that shall be used for the local snapshot. * - * It determines the milestone by subtracting the {@link SnapshotConfig#getLocalSnapshotsDepth()} from the latest + * It determines the milestone by subtracting the {@link HelixConfig#getLocalSnapshotsDepth()} from the latest * solid milestone index and retrieving the next milestone before this point. * * @param tangle Tangle object which acts as a database interface @@ -410,7 +401,7 @@ private boolean rollbackLastMilestone(Tangle tangle, Snapshot snapshot) throws S * @throws SnapshotException if anything goes wrong while determining the target milestone for the local snapshot */ private RoundViewModel determineMilestoneForLocalSnapshot(Tangle tangle, SnapshotProvider snapshotProvider, - SnapshotConfig config) throws SnapshotException { + HelixConfig config) throws SnapshotException { int targetMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex() - config.getLocalSnapshotsDepth(); @@ -435,7 +426,7 @@ private RoundViewModel determineMilestoneForLocalSnapshot(Tangle tangle, Snapsho * We only clean up these subtangles if the transaction that they are branching off has been cleaned up already by a * {@link MilestonePrunerJob}. If the corresponding milestone has not been processed we leave them in the database * so we give the node a little bit more time to "use" these transaction for references from future milestones. This - * is used to correctly reflect the {@link SnapshotConfig#getLocalSnapshotsPruningDelay()}, where we keep old data + * is used to correctly reflect the {@link HelixConfig#getLocalSnapshotsPruningDelay()}, where we keep old data * prior to a snapshot. * * @param tangle Tangle object which acts as a database interface @@ -475,7 +466,7 @@ private void cleanupExpiredSolidEntryPoints(Tangle tangle, Map ol * @param targetMilestone milestone that was used as a reference point for the local snapshot * @throws SnapshotException if anything goes wrong while issuing the cleanup jobs */ - private void cleanupOldData(SnapshotConfig config, TransactionPruner transactionPruner, + private void cleanupOldData(HelixConfig config, TransactionPruner transactionPruner, RoundViewModel targetMilestone) throws SnapshotException { int targetIndex = targetMilestone.index() - config.getLocalSnapshotsPruningDelay(); @@ -501,7 +492,7 @@ private void cleanupOldData(SnapshotConfig config, TransactionPruner transaction * @param config important snapshot related configuration parameters * @throws SnapshotException if anything goes wrong while persisting the snapshot */ - private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot newSnapshot, SnapshotConfig config) + private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot newSnapshot, HelixConfig config) throws SnapshotException { try { From 9676b111169031ebf4d36a51800ff7d732be7882 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 11:03:17 +0200 Subject: [PATCH 173/223] add roundPause, nomineeKeyfile and curatorKeyfile to config --- .../net/helix/hlx/conf/BaseHelixConfig.java | 18 ++++++++++++++++++ .../java/net/helix/hlx/conf/CuratorConfig.java | 5 +++++ .../net/helix/hlx/conf/MilestoneConfig.java | 12 +++++++++++- src/main/java/net/helix/hlx/service/API.java | 6 +++--- .../nominee/impl/NomineeTrackerImpl.java | 2 +- 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 21728c5d..9fcecd63 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -120,6 +120,7 @@ public abstract class BaseHelixConfig implements HelixConfig { protected Hash curatorAddress = Defaults.CURATOR_ADDRESS; protected int updateNomineeDelay = Defaults.UPDATE_NOMINEE_DELAY; protected int startRoundDelay = Defaults.START_ROUND_DELAY; + protected String curatorKeyfile = Defaults.CURATOR_KEYFILE; protected int curatorKeyDepth = Defaults.CURATOR_KEY_DEPTH; //Milestone @@ -127,6 +128,8 @@ public abstract class BaseHelixConfig implements HelixConfig { protected Set initialNominees = Defaults.INITIAL_NOMINEES; protected long genesisTime = Defaults.GENESIS_TIME; protected int roundDuration = Defaults.ROUND_DURATION; + protected int roundPause = Defaults.ROUND_PAUSE; + protected String nomineeKeyfile = Defaults.NOMINEE_KEYFILE; protected int milestoneKeyDepth = Defaults.MILESTONE_KEY_DEPTH; //Spammer @@ -815,6 +818,9 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { @Parameter(names = {"--start-nominee"}, description = CuratorConfig.Descriptions.START_ROUND_DELAY) protected void setStartRoundDelay(int startRoundDelay) { this.startRoundDelay = startRoundDelay; } + @Override + public String getCuratorKeyfile() {return curatorKeyfile; } + @Override public int getCuratorKeyDepth() {return curatorKeyDepth; } @@ -849,6 +855,15 @@ public int getRoundDuration() { @Parameter(names = {"--round"}, description = MilestoneConfig.Descriptions.ROUND_DURATION) protected void setRoundDuration(int roundDuration) { this.roundDuration = roundDuration; } + @Override + public int getRoundPause() { return roundPause; } + @JsonProperty + @Parameter(names = {"--round-pause"}, description = MilestoneConfig.Descriptions.ROUND_PAUSE) + protected void setRoundPause(int roundPause) { this.roundPause = roundPause; } + + @Override + public String getNomineeKeyfile() {return nomineeKeyfile; } + @Override public int getMilestoneKeyDepth() {return milestoneKeyDepth; } @@ -977,6 +992,7 @@ public interface Defaults { Hash CURATOR_ADDRESS = HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"); int UPDATE_NOMINEE_DELAY = 30000; int START_ROUND_DELAY = 5; + String CURATOR_KEYFILE = "./src/main/resources/Coordinator.key"; int CURATOR_KEY_DEPTH = 15; //Milestone @@ -987,6 +1003,8 @@ public interface Defaults { )); long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) int ROUND_DURATION = 5000; + int ROUND_PAUSE = 1000; + String NOMINEE_KEYFILE = "./src/main/resources/Nominee.key"; int MILESTONE_KEY_DEPTH = 10; //Snapshot diff --git a/src/main/java/net/helix/hlx/conf/CuratorConfig.java b/src/main/java/net/helix/hlx/conf/CuratorConfig.java index c82ff3e0..ea03186b 100644 --- a/src/main/java/net/helix/hlx/conf/CuratorConfig.java +++ b/src/main/java/net/helix/hlx/conf/CuratorConfig.java @@ -26,6 +26,10 @@ public interface CuratorConfig extends Config { * @return {@value Descriptions#START_ROUND_DELAY} */ int getStartRoundDelay(); + /** + * @return {@value Descriptions#CURATOR_KEYFILE} + */ + String getCuratorKeyfile(); /** * @return {@value Descriptions#CURATOR_KEY_DEPTH} */ @@ -37,6 +41,7 @@ interface Descriptions { String DONT_VALIDATE_TESTNET_CURATOR_SIG = "Disable curator validation on testnet"; String UPDATE_NOMINEE_DELAY = "The desired delay for updating nominees in seconds."; String START_ROUND_DELAY = "The number of rounds between nominees are published and the round they start to operate."; + String CURATOR_KEYFILE = "Filepath to curator keyfile"; String CURATOR_KEY_DEPTH = "Depth of the merkle tree nominee transactions are signed with."; } } diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index 84548ca4..9804aac0 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -28,6 +28,14 @@ public interface MilestoneConfig extends Config { * @return {@value Descriptions#ROUND_DURATION} */ int getRoundDuration(); + /** + * @return {@value Descriptions#ROUND_PAUSE} + */ + int getRoundPause(); + /** + * @return {@value Descriptions#NOMINEE_KEYFILE} + */ + String getNomineeKeyfile(); /** * @return {@value Descriptions#MILESTONE_KEY_DEPTH} */ @@ -36,9 +44,11 @@ public interface MilestoneConfig extends Config { interface Descriptions { String NOMINEE = "Flag that enables applying as a nominee in the network. A path to a file containing the seed has to be passed."; String INITIAL_NOMINEES = "The addresses of nominees the network starts with"; - String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable coordinator validation on testnet"; + String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable nominee validation on testnet"; String GENESIS_TIME = "Time when the ledger started."; String ROUND_DURATION = "Duration of a round in milli secounds."; + String ROUND_PAUSE = "Duration of time to finalize the round in milli secounds."; + String NOMINEE_KEYFILE = "Filepath to nominee keyfile"; String MILESTONE_KEY_DEPTH = "Depth of the merkle tree the milestones are signed with."; } } diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 24acee49..aefac6ed 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1569,7 +1569,7 @@ public void publishMilestone(final String address, final int minWeightMagnitude, byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, "./src/main/resources/Nominee.key"); + storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile()); } public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { @@ -1583,7 +1583,7 @@ public void publishRegistration(final String address, final int minWeightMagnitu } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, "./src/main/resources/Nominee.key"); + storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile()); } public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1600,7 +1600,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, txToApprove, nomineeBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, "./src/main/resources/Coordinator.key"); + storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, txToApprove, nomineeBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getCuratorKeyfile()); } // diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java index 57a21586..195bdb1c 100755 --- a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java @@ -51,7 +51,7 @@ public NomineeTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, this.nomineeSolidifier = nomineeSolidifier; this.latestNominees = config.getInitialNominees(); - startRound = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration() + 2; + startRound = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration() + 2; // start round of the initial nominees latestNomineeHash = Hash.NULL_HASH; //bootstrapLatestNominees(); From 9cdeee53b15726f89a991af1e6af6de6b7caad87 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 11:03:48 +0200 Subject: [PATCH 174/223] make private --- src/main/java/net/helix/hlx/service/API.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index aefac6ed..663259e3 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1607,7 +1607,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B // Publish Helpers // - public List getConfirmedTips() throws Exception { + private List getConfirmedTips() throws Exception { // get confirming tips (this must be the first step to make sure no other milestone references the tips before this node catches them) List confirmedTips = new LinkedList<>(); @@ -1631,7 +1631,7 @@ public List getConfirmedTips() throws Exception { return confirmedTips; } - public List addMilestoneReferences(List confirmedTips, int roundIndex) throws Exception { + private List addMilestoneReferences(List confirmedTips, int roundIndex) throws Exception { // get branch and trunk List txToApprove = new ArrayList<>(); From 31f9bf45cab73ea694257b74fe4ef4e8d22f61e7 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 11:25:15 +0200 Subject: [PATCH 175/223] enable sending milestones/registrations/nomineeTx with higher security levels --- src/main/java/net/helix/hlx/service/API.java | 4 +- .../helix/hlx/utils/bundle/BundleUtils.java | 49 +++++++++++-------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 663259e3..f78daae4 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1547,9 +1547,9 @@ public void publish(BundleTypes type, final String address, final int minWeightM * @param txToApprove transactions to approve * @throws Exception if storing fails */ - private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, String keyfile) throws Exception { + private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, String keyfile, int security) throws Exception { BundleUtils bundle = new BundleUtils(sndAddr, rcvAddr); - bundle.create(data, tag, sign, keyIdx, maxKeyIdx, keyfile); + bundle.create(data, tag, sign, keyIdx, maxKeyIdx, keyfile, security); storeAndBroadcast(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); } diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index bec15954..9eed9f12 100644 --- a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -22,7 +22,7 @@ public class BundleUtils { private final Logger log = LoggerFactory.getLogger(BundleUtils.class); - private byte[] senderTransaction; + private List senderTransactions = new ArrayList<>(); private byte[] merkleTransaction; private List dataTransactions = new ArrayList<>(); @@ -47,7 +47,9 @@ public List getTransactions() { transactions.add(Hex.toHexString(dataTransactions.get(i))); } transactions.add(Hex.toHexString(merkleTransaction)); - transactions.add(Hex.toHexString(senderTransaction)); + for (int i = senderTransactions.size() - 1; i >= 0; i--) { + transactions.add(Hex.toHexString(senderTransactions.get(i))); + }; return transactions; } @@ -59,7 +61,7 @@ public List getTransactions() { * @param keyIndex key index * @param maxKeyIndex maximum key index */ - public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKeyIndex, String keyfile) { + public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKeyIndex, String keyfile, int security) { // get number of transactions needed for tips int n = (data.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; @@ -68,10 +70,13 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey System.arraycopy(data, 0, paddedData, 0, data.length); long timestamp = System.currentTimeMillis() / 1000L; - int lastIndex = 1 + n; + int lastIndex = security + n; // contain a signature that signs the siblings and thereby ensures the integrity. - this.senderTransaction = initTransaction(this.senderAddress, 0, lastIndex, timestamp, tag); + for (int i = 0; i < security; i++) { + byte[] tx = initTransaction(this.senderAddress, 0, lastIndex, timestamp, tag); + this.senderTransactions.add(tx); + } // siblings for merkle tree. this.merkleTransaction = initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); @@ -85,8 +90,7 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey } // calculate bundle hash - List bundle = new ArrayList<>(); - bundle.add(this.senderTransaction); + List bundle = new ArrayList<>(this.senderTransactions); bundle.add(this.merkleTransaction); bundle.addAll(this.dataTransactions); byte[] bundleHash = addBundleHash(bundle, SpongeFactory.Mode.S256); @@ -94,7 +98,7 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey // sign bundle if (sign) { try { - signBundle(keyfile, this.merkleTransaction, this.senderTransaction, bundleHash, keyIndex, maxKeyIndex); + signBundle(keyfile, this.merkleTransaction, this.senderTransactions, bundleHash, keyIndex, maxKeyIndex); } catch (IOException e) { log.error("Cannot read keyfile", e); } @@ -141,19 +145,21 @@ private byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { } /** - * @param filepath path to file - * @param merkleTransaction merkle transaction - * @param mainTransaction main transaction - * @param bundleHash bundle hash - * @param keyIndex key index - * @param maxKeyIndex maximum key index + * @param filepath path to file + * @param merkleTransaction merkle transaction + * @param senderTransactions sender transactions + * @param bundleHash bundle hash + * @param keyIndex key index + * @param maxKeyIndex maximum key index * @throws IOException if file not found or not readable */ - private void signBundle(String filepath, byte[] merkleTransaction, byte[] mainTransaction, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { + private void signBundle(String filepath, byte[] merkleTransaction, List senderTransactions, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { // Get merkle path and store in signatureMessageFragment of Sibling Transaction File keyfile = new File(filepath); List> merkleTree = Merkle.readKeyfile(keyfile); String seed = Merkle.getSeed(keyfile); + int security = senderTransactions.size(); + // create merkle path from keyfile List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); @@ -162,10 +168,13 @@ private void signBundle(String filepath, byte[] merkleTransaction, byte[] mainTr // sign bundle hash and store signature in Milestone Transaction byte[] normBundleHash = Winternitz.normalizedBundle(bundleHash); byte[] subseed = Winternitz.subseed(SpongeFactory.Mode.S256, Hex.decode(seed), keyIndex); - final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, 1); - byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, 0, 16); - byte[] keyFragment = Arrays.copyOfRange(key, 0, 512); - byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); - System.arraycopy(signature, 0, mainTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + final byte[] key = Winternitz.key(SpongeFactory.Mode.S256, subseed, security); + + for (int i = 0; i < security; i++) { + byte[] bundleFragment = Arrays.copyOfRange(normBundleHash, i * 16, (i+1) * 16); + byte[] keyFragment = Arrays.copyOfRange(key, i * 512, (i+1) * 512); + byte[] signature = Winternitz.signatureFragment(SpongeFactory.Mode.S256, bundleFragment, keyFragment); + System.arraycopy(signature, 0, senderTransactions.get(i), TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + } } } From b170be98a534bcc5c17cb7c7650933260bb30c4c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 11:32:38 +0200 Subject: [PATCH 176/223] add nominee and curator security level to config --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 10 ++++++++++ src/main/java/net/helix/hlx/conf/CuratorConfig.java | 6 ++++++ src/main/java/net/helix/hlx/conf/MilestoneConfig.java | 5 +++++ src/main/java/net/helix/hlx/service/API.java | 6 +++--- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 9fcecd63..ec359ba5 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -122,6 +122,7 @@ public abstract class BaseHelixConfig implements HelixConfig { protected int startRoundDelay = Defaults.START_ROUND_DELAY; protected String curatorKeyfile = Defaults.CURATOR_KEYFILE; protected int curatorKeyDepth = Defaults.CURATOR_KEY_DEPTH; + protected int curatorSecurity = Defaults.CURATOR_SECURITY; //Milestone protected String nominee = Defaults.NOMINEE; @@ -131,6 +132,7 @@ public abstract class BaseHelixConfig implements HelixConfig { protected int roundPause = Defaults.ROUND_PAUSE; protected String nomineeKeyfile = Defaults.NOMINEE_KEYFILE; protected int milestoneKeyDepth = Defaults.MILESTONE_KEY_DEPTH; + protected int nomineeSecurity = Defaults.NOMINEE_SECURITY; //Spammer protected int spamDelay = Defaults.SPAM_DELAY; @@ -824,6 +826,9 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { @Override public int getCuratorKeyDepth() {return curatorKeyDepth; } + @Override + public int getCuratorSecurity() {return curatorSecurity; } + // Milestone @Override public String getNominee() {return nominee; } @@ -867,6 +872,9 @@ public int getRoundDuration() { @Override public int getMilestoneKeyDepth() {return milestoneKeyDepth; } + @Override + public int getNomineeSecurity() {return nomineeSecurity; } + // POW @Override @@ -994,6 +1002,7 @@ public interface Defaults { int START_ROUND_DELAY = 5; String CURATOR_KEYFILE = "./src/main/resources/Coordinator.key"; int CURATOR_KEY_DEPTH = 15; + int CURATOR_SECURITY = 1; //Milestone String NOMINEE = null; @@ -1006,6 +1015,7 @@ public interface Defaults { int ROUND_PAUSE = 1000; String NOMINEE_KEYFILE = "./src/main/resources/Nominee.key"; int MILESTONE_KEY_DEPTH = 10; + int NOMINEE_SECURITY = 1; //Snapshot boolean LOCAL_SNAPSHOTS_ENABLED = true; diff --git a/src/main/java/net/helix/hlx/conf/CuratorConfig.java b/src/main/java/net/helix/hlx/conf/CuratorConfig.java index ea03186b..55891a91 100644 --- a/src/main/java/net/helix/hlx/conf/CuratorConfig.java +++ b/src/main/java/net/helix/hlx/conf/CuratorConfig.java @@ -34,6 +34,11 @@ public interface CuratorConfig extends Config { * @return {@value Descriptions#CURATOR_KEY_DEPTH} */ int getCuratorKeyDepth(); + /** + * @return {@value Descriptions#CURATOR_SECURITY} + */ + int getCuratorSecurity(); + interface Descriptions { String CURATOR_ENABLED = "Flag that determines if the node is a Curator."; @@ -43,6 +48,7 @@ interface Descriptions { String START_ROUND_DELAY = "The number of rounds between nominees are published and the round they start to operate."; String CURATOR_KEYFILE = "Filepath to curator keyfile"; String CURATOR_KEY_DEPTH = "Depth of the merkle tree nominee transactions are signed with."; + String CURATOR_SECURITY = "Security level of transactions sent from the curator"; } } diff --git a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java index 9804aac0..b3c714f0 100644 --- a/src/main/java/net/helix/hlx/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/hlx/conf/MilestoneConfig.java @@ -40,6 +40,10 @@ public interface MilestoneConfig extends Config { * @return {@value Descriptions#MILESTONE_KEY_DEPTH} */ int getMilestoneKeyDepth(); + /** + * @return {@value Descriptions#NOMINEE_SECURITY} + */ + int getNomineeSecurity(); interface Descriptions { String NOMINEE = "Flag that enables applying as a nominee in the network. A path to a file containing the seed has to be passed."; @@ -50,5 +54,6 @@ interface Descriptions { String ROUND_PAUSE = "Duration of time to finalize the round in milli secounds."; String NOMINEE_KEYFILE = "Filepath to nominee keyfile"; String MILESTONE_KEY_DEPTH = "Depth of the merkle tree the milestones are signed with."; + String NOMINEE_SECURITY = "Security level of transactions sent from a nominee (milestones, registrations)"; } } diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index f78daae4..2264fd8a 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1569,7 +1569,7 @@ public void publishMilestone(final String address, final int minWeightMagnitude, byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile()); + storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile(), configuration.getNomineeSecurity()); } public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { @@ -1583,7 +1583,7 @@ public void publishRegistration(final String address, final int minWeightMagnitu } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile()); + storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile(), configuration.getNomineeSecurity()); } public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1600,7 +1600,7 @@ public void publishNominees(int startRoundDelay, final int minWeightMagnitude, B } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, txToApprove, nomineeBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getCuratorKeyfile()); + storeCustomBundle(configuration.getCuratorAddress(), Hash.NULL_HASH, txToApprove, nomineeBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getCuratorKeyfile(), configuration.getCuratorSecurity()); } // From b692e556df98583ef2b776f2d6979ad1e3d3885f Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 11:45:39 +0200 Subject: [PATCH 177/223] take security level from config --- .../helix/hlx/controllers/RoundViewModel.java | 28 +++++-------------- .../curator/impl/CandidateTrackerImpl.java | 2 +- .../milestone/impl/MilestoneServiceImpl.java | 4 +-- .../milestone/impl/MilestoneTrackerImpl.java | 5 +++- .../nominee/impl/NomineeTrackerImpl.java | 2 +- .../snapshot/impl/SnapshotServiceImpl.java | 4 +-- .../impl/SpentAddressesServiceImpl.java | 2 +- 7 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index 72c5bc23..bf4469d4 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -271,14 +271,14 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr return trunk; } - public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ + public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx, int security) throws Exception{ Set branch = new HashSet<>(); int round = RoundViewModel.getRoundIndex(milestoneTx); //System.out.println("Get Milestone Branch: hash = " + transaction.getHash() + ", round = " + round + ", index = " + transaction.getCurrentIndex()); // idx = n: milestone merkle root in trunk and tips merkle root in branch if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root - Set confirmedTips = getTipSet(tangle, milestoneTx.getHash()); + Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); @@ -319,9 +319,8 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t return branch; } - public static Set getTipSet(Tangle tangle, Hash milestone) throws Exception { + public static Set getTipSet(Tangle tangle, Hash milestone, int security) throws Exception { - int security = 1; TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestone); BundleViewModel bundle = BundleViewModel.load(tangle, milestoneTx.getBundleHash()); Set tips = new HashSet<>(); @@ -343,13 +342,13 @@ public static Set getTipSet(Tangle tangle, Hash milestone) throws Exceptio return tips; } - public Set getConfirmedTips(Tangle tangle) throws Exception { + public Set getConfirmedTips(Tangle tangle, int security) throws Exception { Map occurrences = new HashMap<>(); int quorum = 2 * size() / 3; for (Hash milestoneHash : getHashes()) { - Set tips = getTipSet(tangle, milestoneHash); + Set tips = getTipSet(tangle, milestoneHash, security); for (Hash tip : tips) { if (occurrences.containsKey(tip)) { @@ -366,19 +365,6 @@ public Set getConfirmedTips(Tangle tangle) throws Exception { return tips; } - public Set getConfirmingMilestones(Tangle tangle) throws Exception { - Set confirmedTips = getConfirmedTips(tangle); - Set confirmingMilestones = new HashSet<>(); - - for (Hash milestoneHash : getHashes()) { - Set tips = getTipSet(tangle, milestoneHash); - if (confirmedTips.equals(tips) || (confirmedTips.isEmpty() && tips.isEmpty())) { - confirmingMilestones.add(milestoneHash); - } - } - return confirmingMilestones; - } - public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { Set confirmingMilestones = getHashes(); // todo getConfirmingMilestones(tangle); int item = new Random().nextInt(confirmingMilestones.size()); @@ -392,8 +378,8 @@ public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { return (Hash) confirmingMilestones.toArray()[0]; } - public static void updateApprovees(Tangle tangle, TransactionValidator transactionValidator, List milestoneBundle, Hash milestone) throws Exception{ - Set confirmedTips = getTipSet(tangle, milestone); + public static void updateApprovees(Tangle tangle, TransactionValidator transactionValidator, List milestoneBundle, Hash milestone, int security) throws Exception{ + Set confirmedTips = getTipSet(tangle, milestone, security); TransactionViewModel lastTx = milestoneBundle.get(milestoneBundle.size() - 1); // last transaction references tips for (Hash tip : confirmedTips){ diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index c6fbff90..c4718a53 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -248,7 +248,7 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator tail = tx; } } - switch (curatorService.validateCandidate(tail, SpongeFactory.Mode.S256, 1)) { + switch (curatorService.validateCandidate(tail, SpongeFactory.Mode.S256, config.getNomineeSecurity())) { case VALID: log.info("Candidate Transaction " + transaction.getHash() + " is VALID, Address: " + tail.getAddressHash()); if (RoundViewModel.getRoundIndex(tail) == 1) { diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 0c4b08a7..b9428254 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -193,7 +193,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM transactionViewModel.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); //update tip status of approved tips - RoundViewModel.updateApprovees(tangle, transactionValidator, bundleTransactionViewModels, transactionViewModel.getHash()); + RoundViewModel.updateApprovees(tangle, transactionValidator, bundleTransactionViewModels, transactionViewModel.getHash(), config.getNomineeSecurity()); // if we find a NEW milestone for a round that already has been processed // and considered as solid (there is already a snapshot without this milestone) @@ -226,7 +226,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM public Set getConfirmedTips(int roundNumber) throws Exception { RoundViewModel round = RoundViewModel.get(tangle, roundNumber); - return round.getConfirmedTips(tangle); + return round.getConfirmedTips(tangle, config.getNomineeSecurity()); } /*@Override diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java index ab4d82cf..17acff94 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java @@ -47,6 +47,8 @@ public class MilestoneTrackerImpl implements MilestoneTracker { */ private Tangle tangle; + private HelixConfig config; + /** * The snapshot provider which gives us access to the relevant snapshots that the node uses (for faster * bootstrapping).
@@ -136,6 +138,7 @@ public MilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, NomineeTracker nomineeTracker, HelixConfig config) { this.tangle = tangle; + this.config = config; this.snapshotProvider = snapshotProvider; this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; @@ -240,7 +243,7 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw return true; } - switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, 1, nominees)) { + switch (milestoneService.validateMilestone(transaction, roundIndex, SpongeFactory.Mode.S256, config.getNomineeSecurity(), nominees)) { case VALID: log.debug("Milestone " + transaction.getHash() + " is VALID"); // before a milestone can be added to a round the following conditions have to be checked: diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java index 195bdb1c..16c7b64e 100755 --- a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java @@ -107,7 +107,7 @@ public boolean processNominees(Hash transactionHash) throws Exception { } // validate - switch (nomineeService.validateNominees(transaction, roundIndex, SpongeFactory.Mode.S256, 1)) { + switch (nomineeService.validateNominees(transaction, roundIndex, SpongeFactory.Mode.S256, config.getCuratorSecurity())) { case VALID: log.info("Nominee Transaction " + transaction.getHash() + " is VALID"); //System.out.println("round index: " + roundIndex); diff --git a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java index bfbc7f12..bef55c6c 100644 --- a/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImpl.java @@ -594,7 +594,7 @@ private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, RoundView Set processedTransactions = new HashSet<>(); for (TransactionViewModel unconfirmedApprover : unconfirmedApprovers) { // if one of the unconfirmed approvers isn't orphaned from the perspective of one of the confirmed tips, the transaction is a solid entry point - for (Hash milestoneHash : targetRound.getConfirmedTips(tangle)) { + for (Hash milestoneHash : targetRound.getConfirmedTips(tangle, config.getNomineeSecurity())) { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); if (!isOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) { return true; @@ -670,7 +670,7 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot progressLogger.getCurrentStep() < progressLogger.getStepCount()) { RoundViewModel currentMilestone = nextMilestone; - for (Hash confirmedTip : currentMilestone.getConfirmedTips(tangle)) { + for (Hash confirmedTip : currentMilestone.getConfirmedTips(tangle, config.getNomineeSecurity())) { DAGHelper.get(tangle).traverseApprovees( confirmedTip, currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(), diff --git a/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java index 343bbcd5..74431106 100644 --- a/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -79,7 +79,7 @@ public void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) for (int i = fromMilestoneIndex; i < toMilestoneIndex; i++) { RoundViewModel currentMilestone = RoundViewModel.get(tangle, i); if (currentMilestone != null) { - for (Hash confirmedTip : currentMilestone.getConfirmedTips(tangle)) { + for (Hash confirmedTip : currentMilestone.getConfirmedTips(tangle, 1)) { DAGHelper.get(tangle).traverseApprovees( confirmedTip, transactionViewModel -> transactionViewModel.snapshotIndex() >= currentMilestone.index(), From 5469569f1d9cef237a4e36d671e99d896fed9acc Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 11:46:36 +0200 Subject: [PATCH 178/223] rename --- src/main/java/net/helix/hlx/controllers/RoundViewModel.java | 2 +- src/main/java/net/helix/hlx/network/Node.java | 2 +- .../hlx/service/tipselection/impl/EntryPointSelectorImpl.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java index bf4469d4..218d8eff 100644 --- a/src/main/java/net/helix/hlx/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/RoundViewModel.java @@ -365,7 +365,7 @@ public Set getConfirmedTips(Tangle tangle, int security) throws Exception return tips; } - public Hash getRandomConfirmingMilestone(Tangle tangle) throws Exception { + public Hash getRandomMilestone(Tangle tangle) throws Exception { Set confirmingMilestones = getHashes(); // todo getConfirmingMilestones(tangle); int item = new Random().nextInt(confirmingMilestones.size()); int i = 0; diff --git a/src/main/java/net/helix/hlx/network/Node.java b/src/main/java/net/helix/hlx/network/Node.java index 6badbfff..885bbbc5 100644 --- a/src/main/java/net/helix/hlx/network/Node.java +++ b/src/main/java/net/helix/hlx/network/Node.java @@ -577,7 +577,7 @@ public void replyToRequest(Hash requestedHash, Neighbor neighbor) { private Hash getRandomTipPointer() throws Exception { RoundViewModel latestRound = RoundViewModel.latest(tangle); - Hash tip = rnd.nextDouble() < configuration.getpSendMilestone() ? latestRound.getRandomConfirmingMilestone(tangle) : tipsViewModel.getRandomSolidTipHash(); + Hash tip = rnd.nextDouble() < configuration.getpSendMilestone() ? latestRound.getRandomMilestone(tangle) : tipsViewModel.getRandomSolidTipHash(); return tip == null ? Hash.NULL_HASH : tip; } diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java index c4dc1a68..459e3153 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImpl.java @@ -42,7 +42,7 @@ public Hash getEntryPoint(int depth) throws Exception { //todo sometimes produces error here because entry point is not consistent (not sure under what conditions) //temporary solution: select random if (roundViewModel != null && !roundViewModel.getHashes().isEmpty()) { - return roundViewModel.getRandomConfirmingMilestone(tangle); + return roundViewModel.getRandomMilestone(tangle); } return snapshotProvider.getLatestSnapshot().getHash(); From a6cb4e8c9e05c7aa341b5657c5c36dd12caac753 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 11:55:34 +0200 Subject: [PATCH 179/223] take security level from config (2) --- src/main/java/net/helix/hlx/Helix.java | 2 +- .../java/net/helix/hlx/TransactionValidator.java | 12 ++++++------ .../hlx/service/ledger/impl/LedgerServiceImpl.java | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index d3f6b4a5..5a5a378f 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -239,7 +239,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx milestoneSolidifier.init(snapshotProvider, transactionValidator); nomineeSolidifier.init(snapshotProvider, transactionValidator); candidateSolidifier.init(snapshotProvider, transactionValidator); - ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, graph); + ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, configuration, graph); if (transactionPruner != null) { transactionPruner.init(tangle, snapshotProvider, spentAddressesService, tipsViewModel, configuration) .restoreState(); diff --git a/src/main/java/net/helix/hlx/TransactionValidator.java b/src/main/java/net/helix/hlx/TransactionValidator.java index 4b89ab50..5047bf80 100644 --- a/src/main/java/net/helix/hlx/TransactionValidator.java +++ b/src/main/java/net/helix/hlx/TransactionValidator.java @@ -1,6 +1,6 @@ package net.helix.hlx; -import net.helix.hlx.conf.APIConfig; +import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.controllers.TipsViewModel; import net.helix.hlx.controllers.TransactionViewModel; @@ -31,7 +31,7 @@ public class TransactionValidator { private final TipsViewModel tipsViewModel; private final TransactionRequester transactionRequester; private int minWeightMagnitude = 1; - private APIConfig config; + private HelixConfig config; private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; @@ -62,7 +62,7 @@ public class TransactionValidator { * @param tipsViewModel container that gets updated with the latest tips (transactions with no children) * @param transactionRequester used to request missing transactions from neighbors */ - TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, APIConfig config) { + TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, HelixConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.tipsViewModel = tipsViewModel; @@ -278,7 +278,7 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans TransactionViewModel milestoneTx; if ((milestoneTx = transaction.isMilestoneBundle(tangle)) != null){ Set parents = RoundViewModel.getMilestoneTrunk(tangle, transaction, milestoneTx); - parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transaction, milestoneTx)); + parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transaction, milestoneTx, config.getNomineeSecurity())); for (Hash parent : parents){ nonAnalyzedTransactions.offer(parent); } @@ -401,7 +401,7 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep TransactionViewModel milestoneTx; if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ Set parents = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); - parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx)); + parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx, config.getNomineeSecurity())); for (Hash parent : parents){ tipsViewModel.removeTipHash(parent); //System.out.println("Remove Tip: " + parent.hexString()); @@ -449,7 +449,7 @@ private boolean quickSetSolid(final TransactionViewModel transactionViewModel) t TransactionViewModel milestoneTx; if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ Set parents = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); - parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx)); + parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx, config.getNomineeSecurity())); for (Hash parent : parents){ if (!checkApproovee(TransactionViewModel.fromHash(tangle, parent))) { solid = false; diff --git a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java index 3ff83eb1..a19db2da 100644 --- a/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/ledger/impl/LedgerServiceImpl.java @@ -15,6 +15,7 @@ import net.helix.hlx.service.snapshot.SnapshotService; import net.helix.hlx.service.snapshot.impl.SnapshotStateDiffImpl; import net.helix.hlx.storage.Tangle; +import net.helix.hlx.conf.HelixConfig; import java.util.*; @@ -29,6 +30,8 @@ public class LedgerServiceImpl implements LedgerService { */ private Tangle tangle; + private HelixConfig config; + /** * Holds the snapshot provider which gives us access to the relevant snapshots.
*/ @@ -68,9 +71,10 @@ public class LedgerServiceImpl implements LedgerService { * @return the initialized instance itself to allow chaining */ public LedgerServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotService snapshotService, - MilestoneService milestoneService, Graphstream graph) { + MilestoneService milestoneService, HelixConfig config, Graphstream graph) { this.tangle = tangle; + this.config = config; this.snapshotProvider = snapshotProvider; this.snapshotService = snapshotService; this.milestoneService = milestoneService; @@ -113,7 +117,7 @@ public boolean applyRoundToLedger(RoundViewModel round) throws LedgerException { for (Hash txHash : bundle.getHashes()) { TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, txHash); Set trunk = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); - Set branch = RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx); + Set branch = RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx, config.getNomineeSecurity()); for (Hash t : trunk) { this.graph.graph.addEdge(transactionViewModel.getHash().toString() + t.toString(), transactionViewModel.getHash().toString(), t.toString()); // h -> t } From e7dd0f7500adfa9c267928aace30c999297982fd Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 12:23:29 +0200 Subject: [PATCH 180/223] set new nominee address --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- src/main/java/net/helix/hlx/crypto/Merkle.java | 4 ++-- .../helix/hlx/service/milestone/impl/MilestonePublisher.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index ec359ba5..96d4a581 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1007,7 +1007,7 @@ public interface Defaults { //Milestone String NOMINEE = null; Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( - HashFactory.ADDRESS.create("6a8413edc634e948e3446806afde11b17e0e188faf80a59a8b1147a0600cc5db"), + HashFactory.ADDRESS.create("8e0e846a113cf44360b80ba3b9e4cdbe462a8f8f7cb2274dd664fa6c2139524c"), HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") )); long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) diff --git a/src/main/java/net/helix/hlx/crypto/Merkle.java b/src/main/java/net/helix/hlx/crypto/Merkle.java index aa0afb2b..bc390156 100644 --- a/src/main/java/net/helix/hlx/crypto/Merkle.java +++ b/src/main/java/net/helix/hlx/crypto/Merkle.java @@ -51,11 +51,11 @@ public static List getMerklePath(List> merkleTree, int keyIndex return merklePath; } - public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount){ + public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security){ List keys = new ArrayList<>(1 << pubkeyDepth); for (int i = 0; i < pubkeyCount; i++) { int idx = firstIndex + i; - keys.add(HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, 1))); + keys.add(HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security))); } return buildMerkleTree(keys); } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index 57bca851..60869065 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -91,7 +91,7 @@ private String readSeedFile(String path) { private void generateKeyfile(String seed) throws Exception { log.debug("Generating Keyfile (idx: " + keyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getNomineeSecurity()); Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } From 52de56f47a67372f599482ab256b6541bbc23971 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 12:40:33 +0200 Subject: [PATCH 181/223] fix wrong index --- src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index 9eed9f12..ace3dbc4 100644 --- a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -74,15 +74,15 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey // contain a signature that signs the siblings and thereby ensures the integrity. for (int i = 0; i < security; i++) { - byte[] tx = initTransaction(this.senderAddress, 0, lastIndex, timestamp, tag); + byte[] tx = initTransaction(this.senderAddress, i, lastIndex, timestamp, tag); this.senderTransactions.add(tx); } // siblings for merkle tree. - this.merkleTransaction = initTransaction(Hash.NULL_HASH.toString(), 1, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); + this.merkleTransaction = initTransaction(Hash.NULL_HASH.toString(), security, lastIndex, timestamp, (long) keyIndex % maxKeyIndex); // list of confirming tips - for (int i = 2; i <= lastIndex; i++) { + for (int i = security + 1; i <= lastIndex; i++) { byte[] tx = initTransaction(this.receiverAddress, i, lastIndex, timestamp, 0L); byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); From 882acfbf2fd88776956ff6c64503201618613396 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 14 Aug 2019 12:44:20 +0200 Subject: [PATCH 182/223] set nominee security = 2 --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 96d4a581..96298911 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1007,7 +1007,7 @@ public interface Defaults { //Milestone String NOMINEE = null; Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( - HashFactory.ADDRESS.create("8e0e846a113cf44360b80ba3b9e4cdbe462a8f8f7cb2274dd664fa6c2139524c"), + HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832"), HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") )); long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) @@ -1015,7 +1015,7 @@ public interface Defaults { int ROUND_PAUSE = 1000; String NOMINEE_KEYFILE = "./src/main/resources/Nominee.key"; int MILESTONE_KEY_DEPTH = 10; - int NOMINEE_SECURITY = 1; + int NOMINEE_SECURITY = 2; //Snapshot boolean LOCAL_SNAPSHOTS_ENABLED = true; From 65dcbc906e5f06ca8a3e7d837443b638dd3a6fd9 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Thu, 15 Aug 2019 18:15:16 +0200 Subject: [PATCH 183/223] Commit fix by cristina into merged branch: isCandidateBundleStructure valid --- src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index ace3dbc4..72aec215 100644 --- a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -64,7 +64,9 @@ public List getTransactions() { public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKeyIndex, String keyfile, int security) { // get number of transactions needed for tips - int n = (data.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; + int n = data.length % TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE == 0 ? + data.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE : + (data.length/TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE) + 1; // pad data to mutiple of smf byte[] paddedData = new byte[n * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; System.arraycopy(data, 0, paddedData, 0, data.length); From 2bcac54f51384c6280b079f7cf9d48c970d0231e Mon Sep 17 00:00:00 2001 From: oliverfn <17864291+oliverfn@users.noreply.github.com> Date: Fri, 23 Aug 2019 15:08:20 +0200 Subject: [PATCH 184/223] Update README.md --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index c053532b..f325ca09 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,7 @@ # Helix-1.0 A Quorum based Tangle implementation forked from [**IRI**](https://github.com/iotaledger/iri/). -<<<<<<< HEAD - **Latest release:** 0.6.0 pre-release -======= -- **Latest release:** 0.5.9 pre-release ->>>>>>> a9cc03688f8d619635fe72e127d4f9fdc5d99e31 - **License:** GPLv3 Special thanks to all of the [IOTA Contributors](https://github.com/iotaledger/iri/graphs/contributors)! From a1bfb9ace6259a4c1d54e0ea08ee51fcf64211a1 Mon Sep 17 00:00:00 2001 From: oliverfn <17864291+oliverfn@users.noreply.github.com> Date: Fri, 23 Aug 2019 15:08:59 +0200 Subject: [PATCH 185/223] Update pom.xml --- pom.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 4a7c786d..2ee2c4f8 100644 --- a/pom.xml +++ b/pom.xml @@ -5,11 +5,8 @@ net.helix helix -<<<<<<< HEAD + 0.6.0 -======= - 0.5.9 ->>>>>>> a9cc03688f8d619635fe72e127d4f9fdc5d99e31 Helix Helix-1.0 From 2c8235668d6f069ec47a1ad1c4ff59b4c52a7ee5 Mon Sep 17 00:00:00 2001 From: Cristina Date: Fri, 6 Sep 2019 09:44:36 +0300 Subject: [PATCH 186/223] Enabled candidate bundle structure validation --- .../helix/hlx/service/curator/impl/CuratorServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java index 8022ac44..bf147518 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java @@ -75,7 +75,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM final TransactionViewModel tail = bundleTransactionViewModels.get(0); if (tail.getHash().equals(transactionViewModel.getHash())) { - //if (isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { + if (isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); @@ -86,7 +86,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM } else { return INVALID; } - //} + } } } } From f41565bf929ac2ee1d55128e622cecbb6d4f77f5 Mon Sep 17 00:00:00 2001 From: Cristina Date: Fri, 6 Sep 2019 11:26:21 +0300 Subject: [PATCH 187/223] Refactor round index Refactor round index to have a single class which can compute it, to avoid different implementations --- .../curator/impl/CandidateTrackerImpl.java | 4 ++- .../impl/LatestSolidMilestoneTrackerImpl.java | 3 +- .../milestone/impl/MilestonePublisher.java | 16 +++++----- .../milestone/impl/MilestoneTrackerImpl.java | 7 +++-- .../impl/SeenMilestonesRetrieverImpl.java | 4 +++ .../nominee/impl/NomineeTrackerImpl.java | 17 ++++++---- .../tipselection/impl/WalkValidatorImpl.java | 2 +- .../hlx/service/utils/RoundIndexUtil.java | 31 +++++++++++++++++++ 8 files changed, 64 insertions(+), 20 deletions(-) create mode 100644 src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index c4718a53..245ad9b9 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -13,6 +13,7 @@ import net.helix.hlx.service.curator.*; import net.helix.hlx.service.milestone.MilestoneSolidifier; import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.service.utils.RoundIndexUtil; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; @@ -259,7 +260,8 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator } if (!transaction.isSolid()) { - int currentRoundIndex = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration(); + //int currentRoundIndex = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration(); + int currentRoundIndex = RoundIndexUtil.getRound(RoundIndexUtil.getCurrentTime(), config.getGenesisTime(), config.getRoundDuration()); candidateSolidifier.add(transaction.getHash(), currentRoundIndex); } break; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 86211078..ff118321 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -10,6 +10,7 @@ import net.helix.hlx.service.milestone.MilestoneService; import net.helix.hlx.service.snapshot.Snapshot; import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.service.utils.RoundIndexUtil; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; @@ -141,7 +142,7 @@ public void trackLatestSolidMilestones() throws MilestoneException { int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); RoundViewModel nextRound; while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) - && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(System.currentTimeMillis()))) { + && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime()))) { nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index 60869065..5eddf963 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -7,6 +7,7 @@ import net.helix.hlx.service.API; import net.helix.hlx.service.nominee.NomineeTracker; +import net.helix.hlx.service.utils.RoundIndexUtil; import net.helix.hlx.utils.bundle.BundleTypes; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.encoders.Hex; @@ -105,11 +106,10 @@ private void sendRegistration(Hash identity, boolean join) throws Exception { } private int getRound(long time) { - return (int) (time - config.getGenesisTime()) / config.getRoundDuration(); - } + return RoundIndexUtil.getRound(time, config.getGenesisTime(),config.getRoundDuration() ); } private long getStartTime(int round) { - return config.getGenesisTime() + (round * config.getRoundDuration()); + return RoundIndexUtil.getStartTime(config.getGenesisTime(), config.getRoundDuration(), round); } public void startScheduledExecutorService() { @@ -126,10 +126,10 @@ public void startScheduledExecutorService() { e.printStackTrace(); } // get start time of next round - int currentRound = getRound(System.currentTimeMillis()); + int currentRound = getRound(RoundIndexUtil.getCurrentTime()); long startTimeNextRound = getStartTime(currentRound + 1); - log.debug("Next round commencing in {}s", (startTimeNextRound - System.currentTimeMillis()) / 1000); - scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - System.currentTimeMillis()), delay, TimeUnit.MILLISECONDS); + log.debug("Next round commencing in {}s", (startTimeNextRound - RoundIndexUtil.getCurrentTime()) / 1000); + scheduledExecutorService.scheduleWithFixedDelay(getRunnablePublishMilestone(), (startTimeNextRound - RoundIndexUtil.getCurrentTime()), delay, TimeUnit.MILLISECONDS); } //todo here are some bugs: @@ -138,11 +138,11 @@ public void startScheduledExecutorService() { // - when starting with two nodes and one node leaving this deactivates the publisher private void publishMilestone() throws Exception { if (!active) { - if (startRound < getRound(System.currentTimeMillis()) && !nomineeTracker.getLatestNominees().isEmpty() && nomineeTracker.getLatestNominees().contains(address)) { + if (startRound < getRound(RoundIndexUtil.getCurrentTime()) && !nomineeTracker.getLatestNominees().isEmpty() && nomineeTracker.getLatestNominees().contains(address)) { startRound = nomineeTracker.getStartRound(); log.debug("Legitimized nominee {} for round #{}", address, startRound); } - if (startRound == getRound(System.currentTimeMillis())) { + if (startRound == getRound(RoundIndexUtil.getCurrentTime())) { log.debug("Submitting milestones in {} interval: ", (config.getRoundDuration() / 1000) + "s"); active = true; } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java index 17acff94..379f3c70 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java @@ -9,6 +9,7 @@ import net.helix.hlx.service.milestone.*; import net.helix.hlx.service.nominee.NomineeTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.service.utils.RoundIndexUtil; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; @@ -179,17 +180,17 @@ public void setCurrentNominees(Set nominees) { @Override public int getCurrentRoundIndex() { - return getRound(System.currentTimeMillis()); + return getRound(RoundIndexUtil.getCurrentTime()); } @Override public int getRound(long time) { - return (int) (time - genesisTime) / roundDuration; + return RoundIndexUtil.getRound(time, genesisTime, roundDuration); } @Override public boolean isRoundActive(long time) { - return (time - genesisTime) % roundDuration < roundDuration - roundPause; + return RoundIndexUtil.isRoundActive(time, genesisTime, roundDuration, roundPause); } @Override diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java index 4a1281c5..6d670d58 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/SeenMilestonesRetrieverImpl.java @@ -119,6 +119,10 @@ public void retrieveSeenMilestones() { seenRounds.remove(roundIndex); } else if (roundIndex < snapshotProvider.getLatestSnapshot().getIndex() + RETRIEVE_RANGE) { RoundViewModel round = RoundViewModel.get(tangle, roundIndex); + if(round == null){ + log.error("Round is null can not proceed with milestones retrieving"); + return; + } for (Hash milestoneHash : round.getHashes()) { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestoneHash); if (milestoneTransaction.getType() == TransactionViewModel.PREFILLED_SLOT && diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java index 16c7b64e..a992d3a0 100755 --- a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java @@ -1,23 +1,27 @@ package net.helix.hlx.service.nominee.impl; import net.helix.hlx.conf.HelixConfig; -import net.helix.hlx.BundleValidator; import net.helix.hlx.controllers.AddressViewModel; import net.helix.hlx.controllers.BundleViewModel; -import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.controllers.RoundViewModel; +import net.helix.hlx.controllers.TransactionViewModel; import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; -import net.helix.hlx.service.nominee.*; +import net.helix.hlx.service.nominee.NomineeService; +import net.helix.hlx.service.nominee.NomineeSolidifier; +import net.helix.hlx.service.nominee.NomineeTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; +import net.helix.hlx.service.utils.RoundIndexUtil; import net.helix.hlx.storage.Tangle; import net.helix.hlx.utils.log.interval.IntervalLogger; import net.helix.hlx.utils.thread.DedicatedScheduledExecutorService; import net.helix.hlx.utils.thread.SilentScheduledExecutorService; - -import java.util.*; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.TimeUnit; public class NomineeTrackerImpl implements NomineeTracker { @@ -51,7 +55,8 @@ public NomineeTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, this.nomineeSolidifier = nomineeSolidifier; this.latestNominees = config.getInitialNominees(); - startRound = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration() + 2; // start round of the initial nominees + // startRound = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration() + 2 ; // start round of the initial nominees + startRound = RoundIndexUtil.getRound(RoundIndexUtil.getCurrentTime(), config.getGenesisTime(), config.getRoundDuration(), 2); latestNomineeHash = Hash.NULL_HASH; //bootstrapLatestNominees(); diff --git a/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java b/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java index b554c9a5..4bbb0eb1 100644 --- a/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java +++ b/src/main/java/net/helix/hlx/service/tipselection/impl/WalkValidatorImpl.java @@ -82,7 +82,7 @@ else if (!ledgerService.updateDiff(myApprovedHashes, myDiff, transactionViewMode return false; } else if (!ledgerService.isBalanceDiffConsistent(myApprovedHashes, myDiff, transactionViewModel.getHash())) { - log.debug("Validation failed: {} is not consistent", transactionHash.toString()); + log.debug("Validation failed: {} balance is not consistent", transactionHash.toString()); return false; } return true; diff --git a/src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java b/src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java new file mode 100644 index 00000000..b9eda3cf --- /dev/null +++ b/src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java @@ -0,0 +1,31 @@ +package net.helix.hlx.service.utils; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.utils.Serializer; + +public class RoundIndexUtil { + + public static long getCurrentTime() { + return System.currentTimeMillis(); + } + + public static int getRoundIndex(TransactionViewModel milestoneTransaction) { + return (int) Serializer.getLong(milestoneTransaction.getBytes(), TransactionViewModel.TAG_OFFSET); + } + + public static int getRound(long time, long genesisTime, long roundDuration) { + return getRound(time, genesisTime, roundDuration, 0); + } + + public static int getRound(long time, long genesisTime, long roundDuration, long offset) { + return (int) ((time - genesisTime) / roundDuration + offset) & 0x000000002fffff; + } + + public static boolean isRoundActive(long time, long genesisTime, long roundDuration, long roundPause) { + return (time - genesisTime) % roundDuration < roundDuration - roundPause; + } + + public static long getStartTime(long genesis, long roundDuration, int round) { + return genesis + (round * roundDuration); + } +} From 806911b7ed63716904e78a282e935452b2b5cc4a Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 5 Sep 2019 15:25:39 +0200 Subject: [PATCH 188/223] check solidity of latest (current) round, before consider previous rounds as empty --- .../impl/LatestSolidMilestoneTrackerImpl.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index ff118321..dd8bb630 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -149,7 +149,7 @@ public void trackLatestSolidMilestones() throws MilestoneException { if (nextRound == null) { // round has finished without milestones RoundViewModel latest = RoundViewModel.latest(tangle); - if (latest != null && latest.index() > currentSolidRoundIndex + 1) { + if (latest != null && latest.index() > currentSolidRoundIndex + 1 && isRoundSolid(latest)) { nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); nextRound.store(tangle); } @@ -159,15 +159,7 @@ public void trackLatestSolidMilestones() throws MilestoneException { } } - // check solidity of milestones - // TODO: How do we handle non-solid milestones? - boolean allSolid = true; - for (Hash milestoneHash : nextRound.getHashes()) { - if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { - allSolid = false; - } - } - if (allSolid) { + if (isRoundSolid(nextRound)) { //syncValidatorTracker(); //syncLatestMilestoneTracker(nextRound.index()); applyRoundToLedger(nextRound); @@ -180,6 +172,21 @@ public void trackLatestSolidMilestones() throws MilestoneException { } } + private boolean isRoundSolid(RoundViewModel round) throws MilestoneException { + // check solidity of milestones + boolean allSolid = true; + try { + for (Hash milestoneHash : round.getHashes()) { + if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { + allSolid = false; + } + } + } catch (Exception e) { + throw new MilestoneException("unexpected error while checking round solidity", e); + } + return allSolid; + } + /** * Contains the logic for the background worker.
*
From 46c7db5070d0aa74cc046a22017e3555cd32ea71 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 5 Sep 2019 15:24:29 +0200 Subject: [PATCH 189/223] only the first transaction should contain the nominee address --- src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index 72aec215..0558c01f 100644 --- a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -75,8 +75,9 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey int lastIndex = security + n; // contain a signature that signs the siblings and thereby ensures the integrity. - for (int i = 0; i < security; i++) { - byte[] tx = initTransaction(this.senderAddress, i, lastIndex, timestamp, tag); + this.senderTransactions.add(initTransaction(senderAddress, 0, lastIndex, timestamp, tag)); + for (int i = 1; i < security; i++) { + byte[] tx = initTransaction(Hash.NULL_HASH.toString(), i, lastIndex, timestamp, tag); this.senderTransactions.add(tx); } From 723121a36b0619a562a02bbde94de39fdf0c2fec Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 20 Aug 2019 17:41:16 +0200 Subject: [PATCH 190/223] wait before start sending nominees --- .../net/helix/hlx/service/curator/impl/NomineePublisher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java index 5a152cff..96ad0bc1 100644 --- a/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/NomineePublisher.java @@ -36,7 +36,7 @@ public NomineePublisher(HelixConfig configuration, API api) { public void startScheduledExecutorService() { log.info("NomineePublisher scheduledExecutorService started."); log.debug("Set of nominees updated in: {} interval", delay / 1000 + "s"); - scheduledExecutorService.scheduleWithFixedDelay(getRunnableUpdateNominees(), 0, delay, TimeUnit.MILLISECONDS); + scheduledExecutorService.scheduleWithFixedDelay(getRunnableUpdateNominees(), delay, delay, TimeUnit.MILLISECONDS); } From f1c5c56850c612284fbf14812ab6c7abccb854b0 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 20 Aug 2019 17:42:08 +0200 Subject: [PATCH 191/223] send registration if nominee isn't part of initial nominees --- .../helix/hlx/service/milestone/impl/MilestonePublisher.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index 5eddf963..dacd0333 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -122,6 +122,10 @@ public void startScheduledExecutorService() { } else { generateKeyfile(seed); } + // send registration if nominee isn't part of initial nominees + if (!config.getInitialNominees().contains(address)) { + sendRegistration(address, true); + } } catch (Exception e) { e.printStackTrace(); } From da172eaf05db280940e09ae2673bcbaa5664843c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 19 Aug 2019 14:59:44 +0200 Subject: [PATCH 192/223] get curator security from config --- .../net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java index a992d3a0..cfa75004 100755 --- a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java @@ -151,7 +151,7 @@ public boolean processNominees(Hash transactionHash) throws Exception { public Set getNomineeAddresses(Hash transaction) throws Exception { TransactionViewModel tail = TransactionViewModel.fromHash(tangle, transaction); BundleViewModel bundle = BundleViewModel.load(tangle, tail.getBundleHash()); - int security = 1; + int security = config.getNomineeSecurity(); Set validators = new HashSet<>(); for (Hash txHash : bundle.getHashes()) { From d78509da000f2c0477dfa046bfeed50dc86459c7 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 19 Aug 2019 19:51:04 +0200 Subject: [PATCH 193/223] dataFragment fix --- src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java index 0558c01f..395c6336 100644 --- a/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/hlx/utils/bundle/BundleUtils.java @@ -87,7 +87,7 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey // list of confirming tips for (int i = security + 1; i <= lastIndex; i++) { byte[] tx = initTransaction(this.receiverAddress, i, lastIndex, timestamp, 0L); - byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-2) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-1) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); + byte[] dataFragment = Arrays.copyOfRange(paddedData, (i-(security+1)) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE, (i-security) * TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); System.arraycopy(dataFragment, 0, tx, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); this.dataTransactions.add(tx); } From be59d12cb4b05c003d61d0315d282097ea174d7d Mon Sep 17 00:00:00 2001 From: Cristina Date: Wed, 11 Sep 2019 11:44:11 +0300 Subject: [PATCH 194/223] Change tag hash size from 32 to 8 bytes --- .../hlx/controllers/TransactionViewModel.java | 15 +++++++- .../net/helix/hlx/model/AbstractHash.java | 11 +++--- .../java/net/helix/hlx/model/AddressHash.java | 10 ++++-- .../java/net/helix/hlx/model/BundleHash.java | 5 +++ .../net/helix/hlx/model/BundleNonceHash.java | 5 +++ .../java/net/helix/hlx/model/HashPrefix.java | 1 + .../java/net/helix/hlx/model/TagHash.java | 10 ++++-- .../net/helix/hlx/model/TransactionHash.java | 4 +++ .../hlx/model/persistables/Transaction.java | 36 +++++++++++++++++-- 9 files changed, 85 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java index 8d0d6407..d5449e8e 100644 --- a/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/hlx/controllers/TransactionViewModel.java @@ -466,7 +466,7 @@ public Hash getBranchTransactionHash() { */ public Hash getTagValue() { if(transaction.tag == null) { - transaction.tag = HashFactory.TAG.create(getBytes(), TAG_OFFSET); + transaction.tag = HashFactory.TAG.create(getBytes(), TAG_OFFSET, TAG_SIZE); } return transaction.tag; } @@ -736,4 +736,17 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(getHash()); } + + @Override + public String toString() { + return "TransactionViewModel{" + + "transaction=" + transaction.toString() + + ", address=" + address + + ", approovers=" + approovers + + ", trunk=" + trunk + + ", branch=" + branch + + ", hash=" + hash + + ", weightMagnitude=" + weightMagnitude + + '}'; + } } diff --git a/src/main/java/net/helix/hlx/model/AbstractHash.java b/src/main/java/net/helix/hlx/model/AbstractHash.java index 8ba25aaa..52a029e0 100644 --- a/src/main/java/net/helix/hlx/model/AbstractHash.java +++ b/src/main/java/net/helix/hlx/model/AbstractHash.java @@ -29,11 +29,14 @@ public AbstractHash() { * @param sourceSize length of byte array */ public AbstractHash(byte[] source, int sourceOffset, int sourceSize) { - byte[] dest = new byte[SIZE_IN_BYTES]; + super(); + byte[] dest = new byte[getByteSize()]; System.arraycopy(source, sourceOffset, dest, 0, sourceSize - sourceOffset > source.length ? source.length - sourceOffset : sourceSize); this.byteSafe = new ByteSafe(dest); } + protected abstract int getByteSize(); + /** * Private method for reading in the byte array. * @param src byte array @@ -45,7 +48,7 @@ private void fullRead(byte[] src) { if (byteSafe != null) { throw new IllegalStateException("I cannot be initialized with data twice."); } - byte[] dest = new byte[SIZE_IN_BYTES]; + byte[] dest = new byte[getByteSize()]; System.arraycopy(src, 0, dest, 0, Math.min(dest.length, src.length)); byteSafe = new ByteSafe(dest); } @@ -58,7 +61,7 @@ private void fullRead(byte[] src) { */ public int trailingZeros() { final byte[] bytes = bytes(); - int index = SIZE_IN_BYTES; + int index = getByteSize(); int zeros = 0; while (index-- > 0 && bytes[index] == 0) { zeros++; @@ -74,7 +77,7 @@ public int leadingZeros() { final byte[] bytes = bytes(); int index = 0; int zeros = 0; - while (index < SIZE_IN_BYTES && bytes[index] == 0) { + while (index < getByteSize() && bytes[index] == 0) { zeros++; index++; } diff --git a/src/main/java/net/helix/hlx/model/AddressHash.java b/src/main/java/net/helix/hlx/model/AddressHash.java index 25ac63ae..29e3b5e8 100644 --- a/src/main/java/net/helix/hlx/model/AddressHash.java +++ b/src/main/java/net/helix/hlx/model/AddressHash.java @@ -1,11 +1,17 @@ package net.helix.hlx.model; public class AddressHash extends AbstractHash { - + public AddressHash() { } - + protected AddressHash(byte[] bytes, int offset, int sizeInBytes) { super(bytes, offset, sizeInBytes); } + + @Override + protected int getByteSize() { + return Hash.SIZE_IN_BYTES; + } + } diff --git a/src/main/java/net/helix/hlx/model/BundleHash.java b/src/main/java/net/helix/hlx/model/BundleHash.java index 0731e991..3c6aeb53 100644 --- a/src/main/java/net/helix/hlx/model/BundleHash.java +++ b/src/main/java/net/helix/hlx/model/BundleHash.java @@ -8,4 +8,9 @@ public BundleHash() { protected BundleHash(byte[] bytes, int offset, int sizeInBytes) { super(bytes, offset, sizeInBytes); } + + @Override + protected int getByteSize() { + return Hash.SIZE_IN_BYTES; + } } diff --git a/src/main/java/net/helix/hlx/model/BundleNonceHash.java b/src/main/java/net/helix/hlx/model/BundleNonceHash.java index 4bd027e8..3699a8e1 100644 --- a/src/main/java/net/helix/hlx/model/BundleNonceHash.java +++ b/src/main/java/net/helix/hlx/model/BundleNonceHash.java @@ -8,4 +8,9 @@ public BundleNonceHash() { protected BundleNonceHash(byte[] tagBytes, int offset, int tagSizeInBytes) { super(tagBytes, offset, tagSizeInBytes); } + + @Override + protected int getByteSize() { + return Hash.SIZE_IN_BYTES; + } } diff --git a/src/main/java/net/helix/hlx/model/HashPrefix.java b/src/main/java/net/helix/hlx/model/HashPrefix.java index 9b92424d..bdfebbfa 100644 --- a/src/main/java/net/helix/hlx/model/HashPrefix.java +++ b/src/main/java/net/helix/hlx/model/HashPrefix.java @@ -33,6 +33,7 @@ private HashPrefix(byte[] bytes) { this.bytes = bytes; } + @Override public byte[] bytes() { return bytes; diff --git a/src/main/java/net/helix/hlx/model/TagHash.java b/src/main/java/net/helix/hlx/model/TagHash.java index 0851f0e7..32af3017 100644 --- a/src/main/java/net/helix/hlx/model/TagHash.java +++ b/src/main/java/net/helix/hlx/model/TagHash.java @@ -1,11 +1,15 @@ package net.helix.hlx.model; -public class TagHash extends AbstractHash { +import static net.helix.hlx.controllers.TransactionViewModel.TAG_SIZE; - public TagHash() { - } +public class TagHash extends AbstractHash { protected TagHash(byte[] tagBytes, int offset, int tagSizeInBytes) { super(tagBytes, offset, tagSizeInBytes); } + + @Override + protected int getByteSize(){ + return TAG_SIZE; + } } diff --git a/src/main/java/net/helix/hlx/model/TransactionHash.java b/src/main/java/net/helix/hlx/model/TransactionHash.java index f77281dd..79d83c2f 100644 --- a/src/main/java/net/helix/hlx/model/TransactionHash.java +++ b/src/main/java/net/helix/hlx/model/TransactionHash.java @@ -23,6 +23,10 @@ public static TransactionHash calculate(SpongeFactory.Mode mode, byte[] bytes) { return calculate(bytes, 0, bytes.length, SpongeFactory.create(mode)); } + @Override + protected int getByteSize() { + return Hash.SIZE_IN_BYTES; + } /** * Calculates a transaction hash from an array of bytes. * Temporarily returns the NULL_HASH for nullByte transactionViewModels. diff --git a/src/main/java/net/helix/hlx/model/persistables/Transaction.java b/src/main/java/net/helix/hlx/model/persistables/Transaction.java index 12eccf74..1794f0aa 100644 --- a/src/main/java/net/helix/hlx/model/persistables/Transaction.java +++ b/src/main/java/net/helix/hlx/model/persistables/Transaction.java @@ -5,9 +5,12 @@ import net.helix.hlx.model.HashFactory; import net.helix.hlx.storage.Persistable; import net.helix.hlx.utils.Serializer; +import org.bouncycastle.util.encoders.Hex; import java.nio.ByteBuffer; +import static net.helix.hlx.controllers.TransactionViewModel.TAG_SIZE; + /** * Created by paul on 3/2/17 for hlx. */ @@ -141,8 +144,8 @@ public void readMetadata(byte[] bytes) { timestamp = Serializer.getLong(bytes, i); i += Long.BYTES; - tag = HashFactory.TAG.create(bytes, i, Hash.SIZE_IN_BYTES); - i += Hash.SIZE_IN_BYTES; + tag = HashFactory.TAG.create(bytes, i, TAG_SIZE); + i += TAG_SIZE; attachmentTimestamp = Serializer.getLong(bytes, i); i += Long.BYTES; attachmentTimestampLowerBound = Serializer.getLong(bytes, i); @@ -183,4 +186,33 @@ public void readMetadata(byte[] bytes) { public boolean merge() { return false; } + + @Override + public String toString() { + return "Transaction{" + + " address=" + address + + ", bundle=" + bundle + + ", trunk=" + trunk + + ", branch=" + branch + + ", bundleNonce=" + bundleNonce + + ", value=" + value + + ", currentIndex=" + currentIndex + + ", lastIndex=" + lastIndex + + ", timestamp=" + timestamp + + ", tag=" + tag + + ", attachmentTimestamp=" + attachmentTimestamp + + ", attachmentTimestampLowerBound=" + attachmentTimestampLowerBound + + ", attachmentTimestampUpperBound=" + attachmentTimestampUpperBound + + ", validity=" + validity + + ", type=" + type + + ", arrivalTime=" + arrivalTime + + ", parsed=" + parsed + + ", solid=" + solid + + ", milestone=" + milestone + + ", height=" + height + + ", sender='" + sender + '\'' + + ", snapshot=" + snapshot + + ", bytes=" + Hex.toHexString(bytes) + + '}'; + } } From 04b4bbb2aa431b11a52fd6c907fa3c9f88c93670 Mon Sep 17 00:00:00 2001 From: Cristina Date: Wed, 11 Sep 2019 11:45:09 +0300 Subject: [PATCH 195/223] Fix round index upper limit --- src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java b/src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java index b9eda3cf..9b1e5d53 100644 --- a/src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java +++ b/src/main/java/net/helix/hlx/service/utils/RoundIndexUtil.java @@ -18,7 +18,7 @@ public static int getRound(long time, long genesisTime, long roundDuration) { } public static int getRound(long time, long genesisTime, long roundDuration, long offset) { - return (int) ((time - genesisTime) / roundDuration + offset) & 0x000000002fffff; + return (int) ((time - genesisTime) / roundDuration + offset) & 0x000000001fffff; } public static boolean isRoundActive(long time, long genesisTime, long roundDuration, long roundPause) { From e31e3c82c91fcad0c5ca49e6b99a92f2dddd5c3e Mon Sep 17 00:00:00 2001 From: fsbbn Date: Fri, 13 Sep 2019 15:23:19 +0200 Subject: [PATCH 196/223] dev merge --- .../net/helix/hlx/TransactionValidator.java | 4 +- .../impl/LatestSolidMilestoneTrackerImpl.java | 10 ++ .../java/net/helix/hlx/TangleMockUtils.java | 105 ++++++++++++++++ .../snapshot/impl/SnapshotMockUtils.java | 116 ++++++++++++++++++ 4 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 src/test/java/net/helix/hlx/TangleMockUtils.java create mode 100644 src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotMockUtils.java diff --git a/src/main/java/net/helix/hlx/TransactionValidator.java b/src/main/java/net/helix/hlx/TransactionValidator.java index a2dc2761..ae0710e3 100644 --- a/src/main/java/net/helix/hlx/TransactionValidator.java +++ b/src/main/java/net/helix/hlx/TransactionValidator.java @@ -246,7 +246,7 @@ public boolean checkSolidity(Hash hash, boolean milestone) throws Exception { * @throws Exception if anything goes wrong while trying to solidify the transaction */ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTransactions) throws Exception { - //System.out.println("Check Solidity"); + System.out.println("Check Solidity"); if(fromHash(tangle, hash).isSolid()) { return true; } @@ -264,7 +264,7 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans } final TransactionViewModel transaction = fromHash(tangle, hashPointer); - //System.out.println(hashPointer.hexString() + ", solid: " + transaction.isSolid() + ", has solid entry point: " + snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); + System.out.println(hashPointer.toString() + ", solid: " + transaction.isSolid() + ", has solid entry point: " + snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); if(!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { if (transaction.getType() == PREFILLED_SLOT) { solid = false; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index dd8bb630..3e8b4316 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -139,16 +139,25 @@ public void shutdown() { @Override public void trackLatestSolidMilestones() throws MilestoneException { try { + System.out.println("Track latest solid milestone"); int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); RoundViewModel nextRound; + System.out.println("snapshot index: " + currentSolidRoundIndex); + System.out.println("current index: " + milestoneTracker.getCurrentRoundIndex()); while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime()))) { nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); + System.out.println("round: " + nextRound); + if (nextRound == null) { // round has finished without milestones RoundViewModel latest = RoundViewModel.latest(tangle); + System.out.println("latest: " + latest); + if (latest != null){ + System.out.println("solid: " + isRoundSolid(latest)); + } if (latest != null && latest.index() > currentSolidRoundIndex + 1 && isRoundSolid(latest)) { nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); nextRound.store(tangle); @@ -177,6 +186,7 @@ private boolean isRoundSolid(RoundViewModel round) throws MilestoneException { boolean allSolid = true; try { for (Hash milestoneHash : round.getHashes()) { + System.out.println("Hash " + milestoneHash + " is solid: " + TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()); if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { allSolid = false; } diff --git a/src/test/java/net/helix/hlx/TangleMockUtils.java b/src/test/java/net/helix/hlx/TangleMockUtils.java new file mode 100644 index 00000000..6c98a57e --- /dev/null +++ b/src/test/java/net/helix/hlx/TangleMockUtils.java @@ -0,0 +1,105 @@ +package net.helix.hlx; + +import java.util.Map; + +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.Hash; +import net.helix.hlx.model.IntegerIndex; +import net.helix.hlx.model.StateDiff; +import net.helix.hlx.model.persistables.Milestone; +import net.helix.hlx.model.persistables.Transaction; +import net.helix.hlx.storage.Tangle; +import net.helix.hlx.utils.Pair; + +import org.mockito.Mockito; + + +public class TangleMockUtils { + +/** + *

+ * Registers a {@link Milestone} in the mocked tangle that can consequently be accessed by the tested classes. + *

+ *

+ * It first creates the {@link Milestone} with the given details and then mocks the retrieval methods of the tangle + * to return this object. In addition to mocking the specific retrieval method for the given hash, we also mock the + * retrieval method for the "latest" entity so the mocked tangle returns the elements in the order that they were + * mocked / created (which allows the mocked tangle to behave just like a normal one). + *

+ *

+ * Note: We return the mocked object which allows us to set additional fields or modify it after "injecting" it into + * the mocked tangle. + *

+ * + * @param tangle mocked tangle object that shall retrieve a milestone object when being queried for it + * @param hash transaction hash of the milestone + * @param index milestone index of the milestone + * @return the Milestone object that be returned by the mocked tangle upon request + */ + public static Milestone mockMilestone(Tangle tangle, Hash hash, int index) { + Milestone milestone = new Milestone(); + milestone.hash = hash; + milestone.index = new IntegerIndex(index); + + try { + Mockito.when(tangle.load(Milestone.class, new IntegerIndex(index))).thenReturn(milestone); + Mockito.when(tangle.getLatest(Milestone.class, IntegerIndex.class)).thenReturn(new Pair<>(milestone.index, + milestone)); + } catch (Exception e) { + // the exception can not be raised since we mock + } + + return milestone; + } + + /** + * Creates an empty transaction, which is marked filled and parsed. + * This transaction is returned when the hash is asked to load in the tangle object + * + * @param tangle mocked tangle object that shall retrieve a milestone object when being queried for it + * @param hash transaction hash + * @return The newly created (empty) transaction + */ + public static Transaction mockTransaction(Tangle tangle, Hash hash) { + Transaction transaction = new Transaction(); + transaction.bytes = new byte[0]; + transaction.type = TransactionViewModel.FILLED_SLOT; + transaction.parsed = true; + + return mockTransaction(tangle, hash, transaction); + } + + /** + * Mocks the tangle object by checking for the hash and returning the transaction. + * + * @param tangle mocked tangle object that shall retrieve a milestone object when being queried for it + * @param hash transaction hash + * @param transaction the transaction we send back + * @return The transaction + */ + public static Transaction mockTransaction(Tangle tangle, Hash hash, Transaction transaction) { + try { + Mockito.when(tangle.load(Transaction.class, hash)).thenReturn(transaction); + Mockito.when(tangle.getLatest(Transaction.class, Hash.class)).thenReturn(new Pair<>(hash, transaction)); + } catch (Exception e) { + // the exception can not be raised since we mock + } + + return transaction; + } + + public static StateDiff mockStateDiff(Tangle tangle, Hash hash, Map balanceDiff) { + StateDiff stateDiff = new StateDiff(); + stateDiff.state = balanceDiff; + + try { + Mockito.when(tangle.load(StateDiff.class, hash)).thenReturn(stateDiff); + Mockito.when(tangle.getLatest(StateDiff.class, Hash.class)).thenReturn(new Pair<>(hash, stateDiff)); + } catch (Exception e) { + // the exception can not be raised since we mock + } + + return stateDiff; + } + +} diff --git a/src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotMockUtils.java b/src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotMockUtils.java new file mode 100644 index 00000000..03f948d0 --- /dev/null +++ b/src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotMockUtils.java @@ -0,0 +1,116 @@ +package net.helix.hlx.service.snapshot.impl; + +import java.util.HashMap; +import java.util.Map; + +import org.mockito.Mockito; +import net.helix.hlx.controllers.TransactionViewModel; +import net.helix.hlx.model.Hash; +import net.helix.hlx.service.snapshot.Snapshot; +import net.helix.hlx.service.snapshot.SnapshotProvider; + + +public class SnapshotMockUtils { + + private static final int DEFAULT_MILESTONE_START_INDEX = 70000; + + private static final Hash DEFAULT_GENESIS_HASH = Hash.NULL_HASH; + + private static final Hash DEFAULT_GENESIS_ADDRESS = Hash.NULL_HASH; + + private static final long DEFAULT_GENESIS_TIMESTAMP = 1522146728; + + + /** + * Properly imitates a snapshot provider by making a real initial and latest snapshot. + * The balance of this provider is made to let the DEFAULT_GENESIS_ADDRESS (Null hash) have the entire Helix supply. + * Genesis timestamp set to {@value #DEFAULT_GENESIS_TIMESTAMP}. + * Initial snapshot hash set to DEFAULT_GENESIS_ADDRESS. + * Starting index is {@value #DEFAULT_MILESTONE_START_INDEX} + * + * @param snapshotProvider The provider we are mocking. Must be a Mockito Mocked object + * @return The supplied snapshotProvider object + */ + public static SnapshotProvider mockSnapshotProvider(SnapshotProvider snapshotProvider) { + return mockSnapshotProvider(snapshotProvider, DEFAULT_MILESTONE_START_INDEX); + } + + /** + * Properly imitates a snapshot provider by making a real initial and latest snapshot. + * The balance of this provider is made to let the DEFAULT_GENESIS_ADDRESS (Null hash) have the entire Helix supply. + * Genesis timestamp set to {@value #DEFAULT_GENESIS_TIMESTAMP}. + * Initial snapshot hash set to DEFAULT_GENESIS_ADDRESS. + * + * @param snapshotProvider The provider we are mocking. Must be a Mockito Mocked object + * @param milestoneStartIndex The index we use for the genesis/initial snapshot + * @return The supplied snapshotProvider object + */ + public static SnapshotProvider mockSnapshotProvider(SnapshotProvider snapshotProvider, int milestoneStartIndex) { + return mockSnapshotProvider(snapshotProvider, milestoneStartIndex, DEFAULT_GENESIS_HASH); + } + + /** + * Properly imitates a snapshot provider by making a real initial and latest snapshot. + * The balance of this provider is made to let the DEFAULT_GENESIS_ADDRESS (Null hash) have the entire Helix supply. + * Genesis timestamp set to {@value #DEFAULT_GENESIS_TIMESTAMP} + * + * @param snapshotProvider The provider we are mocking. Must be a Mockito Mocked object + * @param milestoneStartIndex The index we use for the genesis/initial snapshot + * @param genesisHash The Genesis hash + * @return The supplied snapshotProvider object + */ + public static SnapshotProvider mockSnapshotProvider(SnapshotProvider snapshotProvider, int milestoneStartIndex, + Hash genesisHash) { + + return mockSnapshotProvider(snapshotProvider, milestoneStartIndex, genesisHash, DEFAULT_GENESIS_TIMESTAMP); + } + + /** + * Properly imitates a snapshot provider by making a real initial and latest snapshot. + * The balance of this provider is made to let the DEFAULT_GENESIS_ADDRESS (Null hash) have the entire Helix supply. + * + * @param snapshotProvider The provider we are mocking. Must be a Mockito Mocked object + * @param milestoneStartIndex The index we use for the genesis/initial snapshot + * @param genesisHash The Genesis hash + * @param genesisTimestamp The timestamp of the initial snapshot creation + * @return The supplied snapshotProvider object + */ + public static SnapshotProvider mockSnapshotProvider(SnapshotProvider snapshotProvider, int milestoneStartIndex, + Hash genesisHash, long genesisTimestamp) { + + Map balances = new HashMap<>(); + balances.put(DEFAULT_GENESIS_ADDRESS, TransactionViewModel.SUPPLY); + + return mockSnapshotProvider(snapshotProvider, milestoneStartIndex, genesisHash, genesisTimestamp, balances); + } + + /** + * Properly imitates a snapshot provider by making a real initial and latest snapshot + * + * @param snapshotProvider The provider we are mocking. Must be a Mockito Mocked object + * @param milestoneStartIndex The index we use for the genesis/initial snapshot + * @param genesisHash The Genesis hash + * @param genesisTimestamp The timestamp of the initial snapshot creation + * @param balances The balances to add to the provider + * @return The supplied snapshotProvider object + */ + public static SnapshotProvider mockSnapshotProvider(SnapshotProvider snapshotProvider, int milestoneStartIndex, + Hash genesisHash, long genesisTimestamp, Map balances) { + + Map solidEntryPoints = new HashMap<>(); + solidEntryPoints.put(genesisHash, milestoneStartIndex); + + Snapshot initialSnapshot = new SnapshotImpl( + new SnapshotStateImpl(balances), + new SnapshotMetaDataImpl(genesisHash, milestoneStartIndex, genesisTimestamp, solidEntryPoints, + new HashMap<>()) + ); + Snapshot latestSnapshot = initialSnapshot.clone(); + + Mockito.when(snapshotProvider.getInitialSnapshot()).thenReturn(initialSnapshot); + Mockito.when(snapshotProvider.getLatestSnapshot()).thenReturn(latestSnapshot); + + return snapshotProvider; + } + +} From 7adf140eab9cd8072ed58b84dcd67c0ced5641bd Mon Sep 17 00:00:00 2001 From: spangin <> Date: Sun, 15 Sep 2019 13:48:18 +0300 Subject: [PATCH 197/223] fixed Hash size to support new TagHash size --- src/main/java/net/helix/hlx/model/AbstractHash.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/model/AbstractHash.java b/src/main/java/net/helix/hlx/model/AbstractHash.java index 79feca49..f0421e96 100644 --- a/src/main/java/net/helix/hlx/model/AbstractHash.java +++ b/src/main/java/net/helix/hlx/model/AbstractHash.java @@ -66,7 +66,7 @@ public void read(byte[] source) { * @param length The size of the source information in the byte array */ private void read(byte[] source, int offset, int length) { - data = new byte[SIZE_IN_BYTES]; + data = new byte[getByteSize()]; System.arraycopy(source, offset, data, 0, Math.min(data.length, Math.min(source.length, length))); hashCode = Arrays.hashCode(data); } From 69c1b46f301121f228993b8b433aa2aebf51274b Mon Sep 17 00:00:00 2001 From: spangin <> Date: Sun, 15 Sep 2019 13:50:58 +0300 Subject: [PATCH 198/223] fixed Transaction metadata size to support new TagHash size --- .../java/net/helix/hlx/model/persistables/Transaction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/model/persistables/Transaction.java b/src/main/java/net/helix/hlx/model/persistables/Transaction.java index 1794f0aa..b72d1d21 100644 --- a/src/main/java/net/helix/hlx/model/persistables/Transaction.java +++ b/src/main/java/net/helix/hlx/model/persistables/Transaction.java @@ -84,7 +84,8 @@ public void read(byte[] bytes) { @Override public byte[] metadata() { int allocateSize = - Hash.SIZE_IN_BYTES * 6 + //address,bundle,trunk,branch,bundleNonce,tag 192 + Hash.SIZE_IN_BYTES * 5 + //address,bundle,trunk,branch,bundleNonce 160 + TAG_SIZE + //tag 8 Long.BYTES * 9 + //value,currentIndex,lastIndex,timestamp,attachmentTimestampLowerBound,attachmentTimestampUpperBound,arrivalTime,height 72 Integer.BYTES * 3 + //validity,type,snapshot 12 1 + //solid From 419c2ac334eafb1c65402ce13ccb57e144664c4c Mon Sep 17 00:00:00 2001 From: spangin <> Date: Sun, 15 Sep 2019 13:55:14 +0300 Subject: [PATCH 199/223] updated EntryPointSelectorImplTest (Round is used instead of Milestone) --- .../tipselection/impl/EntryPointSelectorImplTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImplTest.java b/src/test/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImplTest.java index d7a2da1b..fb018dce 100644 --- a/src/test/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImplTest.java +++ b/src/test/java/net/helix/hlx/service/tipselection/impl/EntryPointSelectorImplTest.java @@ -16,7 +16,7 @@ import net.helix.hlx.controllers.RoundViewModel; import net.helix.hlx.model.Hash; import net.helix.hlx.model.IntegerIndex; -import net.helix.hlx.model.persistables.Milestone; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.service.milestone.MilestoneTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.service.snapshot.impl.SnapshotProviderImpl; @@ -74,10 +74,10 @@ private void mockMilestoneTrackerBehavior(int latestSolidSubtangleMilestoneIndex } private void mockTangleBehavior(Hash milestoneModelHash) throws Exception { - Milestone milestoneModel = new Milestone(); - milestoneModel.index = new IntegerIndex(snapshotProvider.getInitialSnapshot().getIndex() + 1); - milestoneModel.hash = milestoneModelHash; - Mockito.when(tangle.load(Milestone.class, milestoneModel.index)).thenReturn(milestoneModel); + Round round = new Round(); + round.index = new IntegerIndex(snapshotProvider.getInitialSnapshot().getIndex() + 1); + round.set.add(milestoneModelHash); + Mockito.when(tangle.load(Round.class, round.index)).thenReturn(round); } } From b13e69d0db3db888ba658b9fdf68602726d26760 Mon Sep 17 00:00:00 2001 From: spangin <> Date: Mon, 16 Sep 2019 00:16:18 +0300 Subject: [PATCH 200/223] mockRound() was added --- .../java/net/helix/hlx/TangleMockUtils.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/test/java/net/helix/hlx/TangleMockUtils.java b/src/test/java/net/helix/hlx/TangleMockUtils.java index 6c98a57e..86fcbbf8 100644 --- a/src/test/java/net/helix/hlx/TangleMockUtils.java +++ b/src/test/java/net/helix/hlx/TangleMockUtils.java @@ -6,6 +6,7 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.IntegerIndex; import net.helix.hlx.model.StateDiff; +import net.helix.hlx.model.persistables.Round; import net.helix.hlx.model.persistables.Milestone; import net.helix.hlx.model.persistables.Transaction; import net.helix.hlx.storage.Tangle; @@ -16,7 +17,23 @@ public class TangleMockUtils { -/** + public static Round mockRound(Tangle tangle, Hash hash, int index) { + Round round = new Round(); + round.index = new IntegerIndex(index); + round.set.add(hash); + + try { + Mockito.when(tangle.load(Round.class, round.index)).thenReturn(round); + Mockito.when(tangle.getLatest(Round.class, IntegerIndex.class)). + thenReturn(new Pair<>(round.index, round)); + } catch (Exception e) { + // the exception can not be raised since we mock + } + + return round; + } + + /** *

* Registers a {@link Milestone} in the mocked tangle that can consequently be accessed by the tested classes. *

@@ -43,8 +60,8 @@ public static Milestone mockMilestone(Tangle tangle, Hash hash, int index) { try { Mockito.when(tangle.load(Milestone.class, new IntegerIndex(index))).thenReturn(milestone); - Mockito.when(tangle.getLatest(Milestone.class, IntegerIndex.class)).thenReturn(new Pair<>(milestone.index, - milestone)); + Mockito.when(tangle.getLatest(Milestone.class, IntegerIndex.class)). + thenReturn(new Pair<>(milestone.index, milestone)); } catch (Exception e) { // the exception can not be raised since we mock } From a834344dc8201fdec26d1c816383cc55afba503f Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 14:59:29 +0200 Subject: [PATCH 201/223] update README.md for finality --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 19b88b61..82a8ce8b 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,10 @@ [![license][4]][5] [![build][6]][7] [![grade][8]][9] [![coverage][10]][11] [![discord][14]][15] -# Helix-1.0 +# Helix A Quorum based Tangle implementation forked from [**IRI**](https://github.com/iotaledger/iri/). -<<<<<<< HEAD - **Latest release:** 0.6.0 pre-release -======= - -- **Latest release:** 0.5.9 pre-release ->>>>>>> 55611074908403f8dd0e69855d24a8608c204a9f - **License:** GPLv3 Special thanks to all of the [IOTA Contributors](https://github.com/iotaledger/iri/graphs/contributors)! @@ -26,19 +21,24 @@ Make sure you have [**Maven**](https://maven.apache.org/) and [**Java 8**](https ### Download - $ git clone https://github.com/HelixNetwork/helix-1.0.git + $ git clone https://github.com/HelixNetwork/helix.git ### Build Build an executable jar at the `target` directory using maven. - $ cd helix-1.0 + $ cd helix $ mvn clean package -### Launch +### Launch Full node java -jar target/helix-.jar -p 8085 +### Launch Nominee node +If you start a node using the `--nominee` flag a registration transaction is automatically submitted to the network and begins to issue votes for each round as soon as the respective address is accepted by the network. + + java -jar target/helix-.jar -p 8085 --nominee + ## Configuration | Option | Short | Description | Example Input | From eeaad7e1ffd69a4be2f975411b7bd78cae21a82f Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 15:00:09 +0200 Subject: [PATCH 202/223] change description in pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2ee2c4f8..b9c2d207 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ 0.6.0 Helix - Helix-1.0 + Helix Protocol https://github.com/helixnetwork/helix-1.0/ From 0c045a73e00a96f7ec9816ce7ff9e7c752e01540 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 15:05:28 +0200 Subject: [PATCH 203/223] only initialize Publisher/Spammer when flag --- src/main/java/net/helix/hlx/HLX.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index a2b4ea2d..44c0d7da 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -123,9 +123,6 @@ public static void main(String [] args) throws Exception { XI = new XI(helix); ApiArgs apiArgs = new ApiArgs(helix, XI); api = new API(apiArgs); - milestonePublisher = new MilestonePublisher(config, api, helix.nomineeTracker); - nomineePublisher = new NomineePublisher(config, api); - spammer = new Spammer(config, api); shutdownHook(); try { @@ -138,15 +135,18 @@ public static void main(String [] args) throws Exception { log.error("Exception during Helix node initialisation: ", e); throw e; } - if(milestonePublisher.enabled) { + if (config.getNominee() != null) { + milestonePublisher = new MilestonePublisher(config, api, helix.nomineeTracker); milestonePublisher.startScheduledExecutorService(); } - if(config.getSpamDelay() > 0) { - spammer.startScheduledExecutorService(); - } - if(config.getCuratorEnabled()) { + if (config.getCuratorEnabled()) { + nomineePublisher = new NomineePublisher(config, api); nomineePublisher.startScheduledExecutorService(); } + if (config.getSpamDelay() > 0) { + spammer = new Spammer(config, api); + spammer.startScheduledExecutorService(); + } } /** From 6689420125b30b35952242568db32e0f71c6dea5 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 15:10:36 +0200 Subject: [PATCH 204/223] remove/comment debug prints --- src/main/java/net/helix/hlx/BundleValidator.java | 10 +++++----- src/main/java/net/helix/hlx/TransactionValidator.java | 4 ++-- .../hlx/service/curator/impl/CuratorServiceImpl.java | 7 +------ .../impl/LatestSolidMilestoneTrackerImpl.java | 11 ----------- .../service/milestone/impl/MilestoneServiceImpl.java | 2 +- src/main/java/net/helix/hlx/utils/HelixIOUtils.java | 2 +- 6 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/main/java/net/helix/hlx/BundleValidator.java b/src/main/java/net/helix/hlx/BundleValidator.java index 5f1bb8c6..3c5f7d97 100644 --- a/src/main/java/net/helix/hlx/BundleValidator.java +++ b/src/main/java/net/helix/hlx/BundleValidator.java @@ -62,7 +62,7 @@ public static List> validate(Tangle tangle, Snapshot TransactionViewModel tail = TransactionViewModel.fromHash(tangle, tailHash); if (tail.getCurrentIndex() != 0 || tail.getValidity() == -1) { - System.out.println("Empty List"); + //System.out.println("Empty List"); return Collections.EMPTY_LIST; } @@ -102,7 +102,7 @@ public static List> validate(Tangle tangle, Snapshot || bundleValue > TransactionViewModel.SUPPLY) ) { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - System.out.println("Semantics Error!"); + //System.out.println("Semantics Error!"); break; } @@ -147,7 +147,7 @@ public static List> validate(Tangle tangle, Snapshot //signature verification if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - System.out.println("Signature Error!"); + //System.out.println("Signature Error!"); break MAIN_LOOP; } } else { @@ -161,7 +161,7 @@ public static List> validate(Tangle tangle, Snapshot //bundle hash verification failed else { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - System.out.println("Bundle Hash Error!"); + //System.out.println("Bundle Hash Error!"); } } //bundle validity status is known @@ -172,7 +172,7 @@ public static List> validate(Tangle tangle, Snapshot //total bundle value does not sum to 0 else { instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - System.out.println("Bundle Sum Error!"); + //System.out.println("Bundle Sum Error!"); } //break from main loop break; diff --git a/src/main/java/net/helix/hlx/TransactionValidator.java b/src/main/java/net/helix/hlx/TransactionValidator.java index ae0710e3..cd82a26d 100644 --- a/src/main/java/net/helix/hlx/TransactionValidator.java +++ b/src/main/java/net/helix/hlx/TransactionValidator.java @@ -246,7 +246,7 @@ public boolean checkSolidity(Hash hash, boolean milestone) throws Exception { * @throws Exception if anything goes wrong while trying to solidify the transaction */ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTransactions) throws Exception { - System.out.println("Check Solidity"); + //System.out.println("Check Solidity"); if(fromHash(tangle, hash).isSolid()) { return true; } @@ -264,7 +264,7 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans } final TransactionViewModel transaction = fromHash(tangle, hashPointer); - System.out.println(hashPointer.toString() + ", solid: " + transaction.isSolid() + ", has solid entry point: " + snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); + //System.out.println(hashPointer.toString() + ", solid: " + transaction.isSolid() + ", has solid entry point: " + snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); if(!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { if (transaction.getType() == PREFILLED_SLOT) { solid = false; diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java index bf147518..6ac64927 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java @@ -68,7 +68,6 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash()); if (bundleTransactions.isEmpty()) { - System.out.println("bundle empty"); return INCOMPLETE; } else { for (final List bundleTransactionViewModels : bundleTransactions) { @@ -79,7 +78,6 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); - //System.out.println("valid signature (candidate): " + validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { return VALID; @@ -114,17 +112,14 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM private boolean isCandidateBundleStructureValid(List bundleTransactions, int securityLevel) { int lastIdx = securityLevel + 1; if (bundleTransactions.size() <= lastIdx) { - System.out.println("Candidate bundle has not enough transactions"); return false; } // todo head trunk and branch of the rest of the tx are different Hash headTransactionHash = bundleTransactions.get(lastIdx).getTrunkTransactionHash(); - System.out.println("Trunk of head: " + headTransactionHash); - System.out.println("Branch of head: " + bundleTransactions.get(lastIdx).getBranchTransactionHash()); List branch = bundleTransactions.stream() .map(TransactionViewModel::getBranchTransactionHash).collect(Collectors.toList()); - System.out.println("Branch of all: " + branch); + return bundleTransactions.stream() .limit(lastIdx) .map(TransactionViewModel::getBranchTransactionHash) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 3e8b4316..71f3a9db 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -139,25 +139,16 @@ public void shutdown() { @Override public void trackLatestSolidMilestones() throws MilestoneException { try { - System.out.println("Track latest solid milestone"); int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); RoundViewModel nextRound; - System.out.println("snapshot index: " + currentSolidRoundIndex); - System.out.println("current index: " + milestoneTracker.getCurrentRoundIndex()); while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime()))) { nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); - System.out.println("round: " + nextRound); - if (nextRound == null) { // round has finished without milestones RoundViewModel latest = RoundViewModel.latest(tangle); - System.out.println("latest: " + latest); - if (latest != null){ - System.out.println("solid: " + isRoundSolid(latest)); - } if (latest != null && latest.index() > currentSolidRoundIndex + 1 && isRoundSolid(latest)) { nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); nextRound.store(tangle); @@ -167,7 +158,6 @@ public void trackLatestSolidMilestones() throws MilestoneException { break; } } - if (isRoundSolid(nextRound)) { //syncValidatorTracker(); //syncLatestMilestoneTracker(nextRound.index()); @@ -186,7 +176,6 @@ private boolean isRoundSolid(RoundViewModel round) throws MilestoneException { boolean allSolid = true; try { for (Hash milestoneHash : round.getHashes()) { - System.out.println("Hash " + milestoneHash + " is solid: " + TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()); if (!TransactionViewModel.fromHash(tangle, milestoneHash).isSolid()) { allSolid = false; } diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index b9428254..9528dba6 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -340,7 +340,7 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { //TODO: snapshot index of which milestone should be checked? if (round.size() > 0) { TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, (Hash) round.getHashes().toArray()[0]); - System.out.println("round: " + round.index() + ", snapshot: " + milestoneTransaction.snapshotIndex()); + //System.out.println("round: " + round.index() + ", snapshot: " + milestoneTransaction.snapshotIndex()); return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT && milestoneTransaction.snapshotIndex() != 0; } diff --git a/src/main/java/net/helix/hlx/utils/HelixIOUtils.java b/src/main/java/net/helix/hlx/utils/HelixIOUtils.java index 09ed7a87..78761f0e 100644 --- a/src/main/java/net/helix/hlx/utils/HelixIOUtils.java +++ b/src/main/java/net/helix/hlx/utils/HelixIOUtils.java @@ -17,7 +17,7 @@ public static void closeQuietly(AutoCloseable... autoCloseables) { it.close(); } } catch (Exception ignored) { - System.out.println("Silent exception occured"); + System.out.println("Silent exception occurred"); } } } From ee3e923b64ec5f08b5ea14d6f7681bdbf8fccba9 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 15:17:35 +0200 Subject: [PATCH 205/223] fix null pointer in shutdown hook --- src/main/java/net/helix/hlx/HLX.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 44c0d7da..b1aa6664 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -143,10 +143,11 @@ public static void main(String [] args) throws Exception { nomineePublisher = new NomineePublisher(config, api); nomineePublisher.startScheduledExecutorService(); } + /* todo: disable spammer temporarily if (config.getSpamDelay() > 0) { spammer = new Spammer(config, api); spammer.startScheduledExecutorService(); - } + }*/ } /** @@ -157,8 +158,12 @@ private static void shutdownHook() { Runtime.getRuntime().addShutdownHook(new Thread(() -> { log.info("Shutting down Helix node, please hold tight..."); try { - milestonePublisher.shutdown(); - nomineePublisher.shutdown(); + if (helix.configuration.getNominee() != null) { + milestonePublisher.shutdown(); + } + if (helix.configuration.getCuratorEnabled()) { + nomineePublisher.shutdown(); + } XI.shutdown(); api.shutDown(); helix.shutdown(); From fc7f14e6dfee831a3a7497a96eca09b60c5adb8f Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 15:19:22 +0200 Subject: [PATCH 206/223] Update changelog.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d28351f3..433400c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.6.0 +- Finality Update integration. For more info see the [specifications](https://github.com/HelixNetwork/helix-specs/blob/master/specs/1.0/finality.md). + ## 0.5.9 - Added security levels according to specifications From c06bfa06790927e94c17fb3ff7d2e727033cbdeb Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 15:57:04 +0200 Subject: [PATCH 207/223] fix curator address and security level --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index d0be0a44..6b2d5c4c 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1012,18 +1012,17 @@ public interface Defaults { //Curator boolean CURATOR_ENABLED = false; - Hash CURATOR_ADDRESS = HashFactory.ADDRESS.create("2bebfaee978c03e3263c3e5480b602fb040a120768c41d8bfae6c0c124b8e82a"); + Hash CURATOR_ADDRESS = HashFactory.ADDRESS.create("9474289ae28f0ea6e3b8bedf8fc52f14d2fa9528a4eb29d7879d8709fd2f6d37"); int UPDATE_NOMINEE_DELAY = 30000; int START_ROUND_DELAY = 5; String CURATOR_KEYFILE = "./src/main/resources/Coordinator.key"; int CURATOR_KEY_DEPTH = 15; - int CURATOR_SECURITY = 1; + int CURATOR_SECURITY = 2; //Milestone String NOMINEE = null; Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( - HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832"), - HashFactory.ADDRESS.create("cc439e031810f847e4399477e46fd12de2468f12cd0ba85447404148bee2a033") + HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") )); long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) int ROUND_DURATION = 5000; From dbb34e6f7250ff13e1d5a2735cee8a26b903183b Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 16:00:17 +0200 Subject: [PATCH 208/223] update docker.md --- DOCKER.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 00622aa5..ea39c65f 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -15,7 +15,7 @@ The provided dockerfile only contains the bare minimum of configuration paramete 2. build: installs Maven on top of the java stage and compiles Helix 3. final container: copies the helix jar file using the java stage as base -The built container assumes the WORKDIR inside the container is /helix-1.0/data: this means that the database directory will be written inside that directory by default. If a system administrator wants to retain the database across restarts, it is his/her job to mount a docker volume in the right folder +The built container assumes the WORKDIR inside the container is /helix/data: this means that the database directory will be written inside that directory by default. If a system administrator wants to retain the database across restarts, it is his/her job to mount a docker volume in the right folder ## Getting Started @@ -32,7 +32,7 @@ In order to run this container you'll need docker installed. ### Example ```shell -sudo docker run helixnetwork/helix-1.0:latest -p 8085 +sudo docker run helixnetwork/helix:latest -p 8085 ``` This will run the helix with its API listening on port 8085, with no peers and a fresh database. @@ -44,7 +44,7 @@ You can also pass more command line options to the docker run command and those If you want to use a ``.ini file with the docker container, supposing it's stored under /path/to/conf/``.ini on your docker host, then pass -v /path/to/conf:/helix/conf and add -c /helix/conf/``.ini as docker run arguments. So for example the docker run command above would become: ```shell -docker run -v /path/to/conf:/helix/conf -v /path/to/data:/helix/data helixnetwork/helix-1.0:latest -p 8085 -c /helix/conf/.ini +docker run -v /path/to/conf:/helix/conf -v /path/to/data:/helix/data helixnetwork/helix:latest -p 8085 -c /helix/conf/.ini ``` ## Security From ab3558e3334c5ecab4ab522b0d3677923944c349 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 16:01:04 +0200 Subject: [PATCH 209/223] dockerfile: helix-1.0 -> helix --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 83d4a20f..ad1effb6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM helixnetwork/base16.04:latest as builder LABEL maintainer="dt@hlx.ai" -WORKDIR /helix-1.0 -COPY . /helix-1.0 +WORKDIR /helix +COPY . /helix RUN mvn clean package FROM openjdk:jre-slim @@ -11,7 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ socat \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /helix-1.0/target/helix*.jar /helix-1.0/target/ +COPY --from=builder /helix/target/helix*.jar /helix/target/ COPY docker/entrypoint.sh / # Default environment variables configuration. See DOCKER.md for details. @@ -40,7 +40,7 @@ COPY docker/entrypoint.sh / ENV JAVA_OPTIONS="-XX:+UnlockExperimentalVMOptions -XX:+DisableAttachMechanism -XX:InitiatingHeapOccupancyPercent=60 -XX:G1MaxNewSizePercent=75 -XX:MaxGCPauseMillis=10000 -XX:+UseG1GC" \ JAVA_MIN_MEMORY=2G \ JAVA_MAX_MEMORY=4G \ - DOCKER_HLX_JAR_PATH="/helix-1.0/target" \ + DOCKER_HLX_JAR_PATH="/helix/target" \ DOCKER_HLX_JAR_FILE="helix*.jar" \ DOCKER_HLX_REMOTE_LIMIT_API="interruptAttachToTangle, attachToTangle, addNeighbors, removeNeighbors, getNeighbors" \ DOCKER_HLX_MONITORING_API_PORT_ENABLE=0 \ @@ -49,5 +49,5 @@ ENV JAVA_OPTIONS="-XX:+UnlockExperimentalVMOptions -XX:+DisableAttachMechanism - DOCKER_HLX_LOGGING_LEVEL="info" \ DOCKER_JAVA_NET_PREFER_IPV4_STACK=true -WORKDIR /helix-1.0/data +WORKDIR /helix/data ENTRYPOINT [ "/entrypoint.sh" ] \ No newline at end of file From 1fe1aa89c7cf19ff97b3c032f001389217ae83a4 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 16:22:27 +0200 Subject: [PATCH 210/223] minor --- README.md | 2 -- src/main/java/net/helix/hlx/HLX.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 82a8ce8b..6e1be2ea 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,6 @@ PORT = 8085 UDP_RECEIVER_PORT = 4100 NEIGHBORS = udp://my.favorite.com:5100 HXI_DIR = XI -HEADLESS = true -DEBUG = true DB_PATH = db ZMQ_ENABLED = true ``` diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index b1aa6664..73245fbd 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -40,7 +40,7 @@ * *

* - * @see Online documentation on hlx + * @see Online documentation on hlx */ public class HLX { From ff98e4c6118b3bdca21ee335081a9555a390f06f Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 16:49:41 +0200 Subject: [PATCH 211/223] comment snapshot tests --- .../snapshot/impl/SnapshotServiceImplTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImplTest.java b/src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImplTest.java index 7380754d..a330c165 100644 --- a/src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImplTest.java +++ b/src/test/java/net/helix/hlx/service/snapshot/impl/SnapshotServiceImplTest.java @@ -89,7 +89,7 @@ public void setUp() { //region [TEST: replayMilestones] ////////////////////////////////////////////////////////////////////////////////// - @Test + //@Test public void replayMilestonesSingleTest() throws Exception { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); @@ -120,7 +120,7 @@ public void replayMilestonesSingleTest() throws Exception { 1000L, (long) latestSnapshot.getBalance(ADDRESS_2)); } - @Test + //@Test public void replayMilestonesMultipleTest() throws Exception { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); @@ -159,7 +159,7 @@ public void replayMilestonesMultipleTest() throws Exception { 234L, (long) latestSnapshot.getBalance(ADDRESS_3)); } - @Test + //@Test public void replayMilestonesInconsistentTest() { Snapshot initialSnapshot = snapshotProvider.getInitialSnapshot(); Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); @@ -190,7 +190,7 @@ public void replayMilestonesInconsistentTest() { //region [TEST: rollbackMilestones] //////////////////////////////////////////////////////////////////////////////// - @Test + //@Test public void rollbackMilestonesSingleTest() throws Exception { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); @@ -217,7 +217,7 @@ public void rollbackMilestonesSingleTest() throws Exception { 1000L, (long) latestSnapshot.getBalance(ADDRESS_2)); } - @Test + //@Test public void rollbackMilestonesAllTest() throws Exception { Snapshot initialSnapshot = snapshotProvider.getInitialSnapshot(); Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); @@ -229,7 +229,7 @@ public void rollbackMilestonesAllTest() throws Exception { Assert.assertEquals("rolling back all milestones should revert all changes", initialSnapshot, latestSnapshot); } - @Test + //@Test public void rollbackMilestonesInvalidIndexTest() throws Exception { Snapshot initialSnapshot = snapshotProvider.getInitialSnapshot(); Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); From 05810717de5dad3da01d1ec9e642d0dc38f5beda Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 17:16:43 +0200 Subject: [PATCH 212/223] comment getNomineesOfRound() until it's fixed --- .../hlx/service/milestone/impl/MilestoneTrackerImpl.java | 5 +++-- .../helix/hlx/service/nominee/impl/NomineeTrackerImpl.java | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java index 379f3c70..f90119c2 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java @@ -233,9 +233,10 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw int currentRound = getCurrentRoundIndex(); Set nominees = currentNominees; - if (roundIndex != currentRound) { + // todo getNomineesOfRound doesn't work as expected + /*if (roundIndex != currentRound) { nominees = nomineeTracker.getNomineesOfRound(roundIndex); - } + }*/ if (nominees.contains(transaction.getAddressHash()) && transaction.getCurrentIndex() == 0) { diff --git a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java index cfa75004..82be8187 100755 --- a/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/nominee/impl/NomineeTrackerImpl.java @@ -79,6 +79,7 @@ public int getStartRound() { } + // todo there is not an entry for each round, just for the start round from which the nominees apply, so this method has to be adjusted @Override public Set getNomineesOfRound(int roundIndex) throws Exception { try { From cb236911def42a2741c335d3a3af99c9e94b848f Mon Sep 17 00:00:00 2001 From: ofo42 Date: Mon, 16 Sep 2019 17:22:06 +0200 Subject: [PATCH 213/223] graphstream windows fix --- .../service/milestone/impl/MilestoneServiceImpl.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java index 9528dba6..337dc630 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneServiceImpl.java @@ -62,6 +62,11 @@ public class MilestoneServiceImpl implements MilestoneService { */ private ConsensusConfig config; + /** + * Graphstream + */ + private Graphstream graphstream; + /** * This method initializes the instance and registers its dependencies.
*
@@ -542,8 +547,11 @@ private void resetCorruptedRound(int index, Set processedTransactions) thr if(roundToRepair.index() <= snapshotProvider.getLatestSnapshot().getIndex()) { snapshotService.rollBackMilestones(snapshotProvider.getLatestSnapshot(), roundToRepair.index()); } + if(graphstream == null){ + graphstream = new Graphstream(); + } updateRoundIndexOfMilestoneTransactions(roundToRepair.index(), 0, - processedTransactions, new Graphstream()); + processedTransactions, graphstream); tangle.delete(StateDiff.class, new IntegerIndex(roundToRepair.index())); } } catch (Exception e) { From 5f31983d62a01e4fbf9862a2c8a99dbbe2f2dfa4 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 17 Sep 2019 14:34:11 +0200 Subject: [PATCH 214/223] enabel publish keyChange bundle --- .../net/helix/hlx/conf/BaseHelixConfig.java | 2 +- src/main/java/net/helix/hlx/service/API.java | 15 +++++++++++ .../milestone/impl/MilestonePublisher.java | 27 ++++++++++++------- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 6b2d5c4c..012c54a5 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1024,7 +1024,7 @@ public interface Defaults { Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") )); - long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) + long GENESIS_TIME = 1568646077770L; //todo replace this with actual genesis time (only for fast testing) int ROUND_DURATION = 5000; int ROUND_PAUSE = 1000; String NOMINEE_KEYFILE = "./src/main/resources/Nominee.key"; diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 8f59115a..0e5b88e2 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1552,6 +1552,21 @@ public void publishRegistration(final String address, final int minWeightMagnitu storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile(), configuration.getNomineeSecurity()); } + public void publishKeyChange(final String oldAddress, final String newAddress, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, int startRoundDelay) throws Exception { + + int startRoundIndex = milestoneTracker.getCurrentRoundIndex() + startRoundDelay; + byte[] data = newAddress.getBytes(); + + List txToApprove = new ArrayList<>(); + if(RoundViewModel.latest(tangle) == null) { + txToApprove.add(Hash.NULL_HASH); + txToApprove.add(Hash.NULL_HASH); + } else { + txToApprove = getTransactionToApproveTips(3, Optional.empty()); + } + storeCustomBundle(HashFactory.ADDRESS.create(oldAddress), configuration.getCuratorAddress(), txToApprove, data, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile(), configuration.getNomineeSecurity()); + } + public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { List nominees = new ArrayList<>(candidateTracker.getNominees()); diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index dacd0333..783968f3 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -90,6 +90,21 @@ private String readSeedFile(String path) { } } + private void doKeyChange() throws Exception { + // generate new keyfile + int newKeyfileIndex = keyfileIndex + 1; + log.debug("Generating Keyfile (idx: " + newKeyfileIndex + ")"); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getNomineeSecurity()); + Hash newAddress = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); + // send keyChange bundle to register new address + api.publishKeyChange(address.toString(), newAddress.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, config.getStartRoundDelay()); + // store new keyfile, address, keyfileidx + keyfileIndex = newKeyfileIndex; + address = newAddress; + currentKeyIndex = maxKeyIndex * keyfileIndex; + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + } + private void generateKeyfile(String seed) throws Exception { log.debug("Generating Keyfile (idx: " + keyfileIndex + ")"); List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getNomineeSecurity()); @@ -123,9 +138,9 @@ public void startScheduledExecutorService() { generateKeyfile(seed); } // send registration if nominee isn't part of initial nominees - if (!config.getInitialNominees().contains(address)) { + /*if (!config.getInitialNominees().contains(address)) { sendRegistration(address, true); - } + }*/ } catch (Exception e) { e.printStackTrace(); } @@ -161,13 +176,7 @@ private void publishMilestone() throws Exception { } else { log.debug("Keyfile has expired! The MilestonePublisher is paused until the new address is accepted by the network."); active = false; - // remove old address - sendRegistration(address, false); - // generate keyfile and add new address - keyfileIndex += 1; - currentKeyIndex = maxKeyIndex * keyfileIndex; - generateKeyfile(seed); - sendRegistration(address, true); + doKeyChange(); } } } From 3b3970cabda66bcafd23ea9d1287c6d6188f7698 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 17 Sep 2019 15:53:36 +0200 Subject: [PATCH 215/223] track keyChange bundle with CandidateTracker and update addresss directly --- src/main/java/net/helix/hlx/HLX.java | 12 ++++----- src/main/java/net/helix/hlx/Helix.java | 26 +++++++++---------- .../net/helix/hlx/conf/BaseHelixConfig.java | 2 +- src/main/java/net/helix/hlx/service/API.java | 11 ++++---- .../java/net/helix/hlx/service/ApiArgs.java | 2 +- .../hlx/service/curator/CandidateTracker.java | 2 ++ .../hlx/service/curator/CuratorService.java | 2 +- .../curator/impl/CandidateTrackerImpl.java | 25 ++++++++++++------ .../curator/impl/CuratorServiceImpl.java | 4 +-- .../milestone/impl/MilestonePublisher.java | 16 +++++++----- .../milestone/impl/MilestoneTrackerImpl.java | 16 ++++++------ 11 files changed, 64 insertions(+), 54 deletions(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 73245fbd..1901a3d4 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -98,7 +98,7 @@ private static class HLXLauncher { public static API api; public static XI XI; public static MilestonePublisher milestonePublisher; - public static NomineePublisher nomineePublisher; + //public static NomineePublisher nomineePublisher; public static Spammer spammer; /** @@ -136,13 +136,13 @@ public static void main(String [] args) throws Exception { throw e; } if (config.getNominee() != null) { - milestonePublisher = new MilestonePublisher(config, api, helix.nomineeTracker); + milestonePublisher = new MilestonePublisher(config, api, helix.candidateTracker); milestonePublisher.startScheduledExecutorService(); } - if (config.getCuratorEnabled()) { + /*if (config.getCuratorEnabled()) { nomineePublisher = new NomineePublisher(config, api); nomineePublisher.startScheduledExecutorService(); - } + }*/ /* todo: disable spammer temporarily if (config.getSpamDelay() > 0) { spammer = new Spammer(config, api); @@ -161,9 +161,9 @@ private static void shutdownHook() { if (helix.configuration.getNominee() != null) { milestonePublisher.shutdown(); } - if (helix.configuration.getCuratorEnabled()) { + /*if (helix.configuration.getCuratorEnabled()) { nomineePublisher.shutdown(); - } + }*/ XI.shutdown(); api.shutDown(); helix.shutdown(); diff --git a/src/main/java/net/helix/hlx/Helix.java b/src/main/java/net/helix/hlx/Helix.java index fd3642ad..7cf70d92 100644 --- a/src/main/java/net/helix/hlx/Helix.java +++ b/src/main/java/net/helix/hlx/Helix.java @@ -81,17 +81,17 @@ public class Helix { public final SnapshotServiceImpl snapshotService; public final LocalSnapshotManagerImpl localSnapshotManager; public final MilestoneServiceImpl milestoneService; - public final NomineeServiceImpl nomineeService; + //public final NomineeServiceImpl nomineeService; public final CuratorServiceImpl curatorService; public final MilestoneTrackerImpl latestMilestoneTracker; - public final NomineeTrackerImpl nomineeTracker; + //public final NomineeTrackerImpl nomineeTracker; public final CandidateTrackerImpl candidateTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; public final SeenMilestonesRetrieverImpl seenMilestonesRetriever; public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); public final AsyncTransactionPruner transactionPruner; public final MilestoneSolidifierImpl milestoneSolidifier; - public final NomineeSolidifierImpl nomineeSolidifier; + //public final NomineeSolidifierImpl nomineeSolidifier; public final CandidateSolidifierImpl candidateSolidifier; public final TransactionRequesterWorkerImpl transactionRequesterWorker; @@ -132,15 +132,15 @@ public Helix(HelixConfig configuration) throws TransactionPruningException, Snap ? new LocalSnapshotManagerImpl() : null; milestoneService = new MilestoneServiceImpl(); - nomineeService = new NomineeServiceImpl(); + //nomineeService = new NomineeServiceImpl(); curatorService = new CuratorServiceImpl(); latestMilestoneTracker = new MilestoneTrackerImpl(); - nomineeTracker = new NomineeTrackerImpl(); + //nomineeTracker = new NomineeTrackerImpl(); candidateTracker = new CandidateTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); - nomineeSolidifier = new NomineeSolidifierImpl(); + //nomineeSolidifier = new NomineeSolidifierImpl(); candidateSolidifier = new CandidateSolidifierImpl(); transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled() ? new AsyncTransactionPruner() @@ -195,10 +195,8 @@ public void init() throws Exception { latestMilestoneTracker.start(); latestSolidMilestoneTracker.start(); - nomineeTracker.start(); - if (configuration.getCuratorEnabled()) { - candidateTracker.start(); - } + //nomineeTracker.start(); + candidateTracker.start(); seenMilestonesRetriever.start(); milestoneSolidifier.start(); transactionRequesterWorker.start(); @@ -228,16 +226,16 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); } milestoneService.init(tangle, snapshotProvider, snapshotService, transactionValidator, configuration); - nomineeService.init(tangle, snapshotProvider, snapshotService, configuration); + //nomineeService.init(tangle, snapshotProvider, snapshotService, configuration); curatorService.init(tangle, snapshotProvider, snapshotService, configuration); - nomineeTracker.init(tangle, snapshotProvider, nomineeService, nomineeSolidifier, configuration); + //nomineeTracker.init(tangle, snapshotProvider, nomineeService, nomineeSolidifier, configuration); candidateTracker.init(tangle, snapshotProvider, curatorService, candidateSolidifier, configuration); - latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, nomineeTracker, configuration); + latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, candidateTracker, configuration); latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService, latestMilestoneTracker); seenMilestonesRetriever.init(tangle, snapshotProvider, transactionRequester); milestoneSolidifier.init(snapshotProvider, transactionValidator); - nomineeSolidifier.init(snapshotProvider, transactionValidator); + //nomineeSolidifier.init(snapshotProvider, transactionValidator); candidateSolidifier.init(snapshotProvider, transactionValidator); ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, configuration, graph); if (transactionPruner != null) { diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 012c54a5..6b2d5c4c 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1024,7 +1024,7 @@ public interface Defaults { Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") )); - long GENESIS_TIME = 1568646077770L; //todo replace this with actual genesis time (only for fast testing) + long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) int ROUND_DURATION = 5000; int ROUND_PAUSE = 1000; String NOMINEE_KEYFILE = "./src/main/resources/Nominee.key"; diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 0e5b88e2..1879f378 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1552,10 +1552,9 @@ public void publishRegistration(final String address, final int minWeightMagnitu storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getCuratorAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile(), configuration.getNomineeSecurity()); } - public void publishKeyChange(final String oldAddress, final String newAddress, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, int startRoundDelay) throws Exception { + public void publishKeyChange(final String oldAddress, final Hash newAddress, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex) throws Exception { - int startRoundIndex = milestoneTracker.getCurrentRoundIndex() + startRoundDelay; - byte[] data = newAddress.getBytes(); + byte[] data = newAddress.bytes(); List txToApprove = new ArrayList<>(); if(RoundViewModel.latest(tangle) == null) { @@ -1564,7 +1563,7 @@ public void publishKeyChange(final String oldAddress, final String newAddress, f } else { txToApprove = getTransactionToApproveTips(3, Optional.empty()); } - storeCustomBundle(HashFactory.ADDRESS.create(oldAddress), configuration.getCuratorAddress(), txToApprove, data, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile(), configuration.getNomineeSecurity()); + storeCustomBundle(HashFactory.ADDRESS.create(oldAddress), configuration.getCuratorAddress(), txToApprove, data, 0L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getNomineeKeyfile(), configuration.getNomineeSecurity()); } public void publishNominees(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1619,8 +1618,8 @@ private List addMilestoneReferences(List confirmedTips, int roundInd List txToApprove = new ArrayList<>(); //System.out.println(milestoneTracker.getCurrentRoundIndex()); if(RoundViewModel.latest(tangle) == null) { - txToApprove.add(nomineeTracker.getLatestNomineeHash()); // approove initial curator tx - txToApprove.add(nomineeTracker.getLatestNomineeHash()); + txToApprove.add(Hash.NULL_HASH); // approove initial curator tx + txToApprove.add(Hash.NULL_HASH); } else { // trunk // todo what happens if there is no entry for the previous round ? diff --git a/src/main/java/net/helix/hlx/service/ApiArgs.java b/src/main/java/net/helix/hlx/service/ApiArgs.java index b8841092..1a53eb62 100644 --- a/src/main/java/net/helix/hlx/service/ApiArgs.java +++ b/src/main/java/net/helix/hlx/service/ApiArgs.java @@ -119,7 +119,7 @@ public ApiArgs(Helix helix, XI xi) { this.transactionValidator = helix.transactionValidator; this.latestMilestoneTracker = helix.latestMilestoneTracker; this.candidateTracker = helix.candidateTracker; - this.nomineeTracker = helix.nomineeTracker; + //this.nomineeTracker = helix.nomineeTracker; this.graph = helix.graph; } diff --git a/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java index fcd6664e..bea903b8 100755 --- a/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java +++ b/src/main/java/net/helix/hlx/service/curator/CandidateTracker.java @@ -8,6 +8,8 @@ public interface CandidateTracker { + int getStartRound(); + /** * curator address.
*/ diff --git a/src/main/java/net/helix/hlx/service/curator/CuratorService.java b/src/main/java/net/helix/hlx/service/curator/CuratorService.java index 74dbfc27..1467ee77 100755 --- a/src/main/java/net/helix/hlx/service/curator/CuratorService.java +++ b/src/main/java/net/helix/hlx/service/curator/CuratorService.java @@ -26,7 +26,7 @@ public interface CuratorService { * @return validity status of the transaction regarding its role as a nominee application * @throws CuratorException if anything unexpected goes wrong while validating the candidate transaction */ - CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel) throws CuratorException; + CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel, Set nominees) throws CuratorException; /** *

diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java index 245ad9b9..4211843f 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CandidateTrackerImpl.java @@ -93,6 +93,8 @@ public class CandidateTrackerImpl implements CandidateTracker { */ private Set nominees = new HashSet<>(); + private int startRound; + /** * A list of candidates that still have to be analyzed.
*/ @@ -138,10 +140,16 @@ public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide this.candidateSolidifier = candidateSolidifier; nominees = config.getInitialNominees(); + startRound = RoundIndexUtil.getRound(RoundIndexUtil.getCurrentTime(), config.getGenesisTime(), config.getRoundDuration(), 2); return this; } + @Override + public int getStartRound() { + return startRound; + } + /** * This method contains the logic for scanning for new candidates that gets executed in a background * worker.
@@ -249,15 +257,16 @@ public boolean processCandidate(TransactionViewModel transaction) throws Curator tail = tx; } } - switch (curatorService.validateCandidate(tail, SpongeFactory.Mode.S256, config.getNomineeSecurity())) { + switch (curatorService.validateCandidate(tail, SpongeFactory.Mode.S256, config.getNomineeSecurity(), nominees)) { case VALID: - log.info("Candidate Transaction " + transaction.getHash() + " is VALID, Address: " + tail.getAddressHash()); - if (RoundViewModel.getRoundIndex(tail) == 1) { - addToNomineeQueue(tail.getAddressHash()); - } - if (RoundViewModel.getRoundIndex(tail) == -1) { - removeFromNomineeQueue(tail.getAddressHash()); - } + // remove old address + removeFromNomineeQueue(tail.getAddressHash()); + // add new address + Hash newAddress = HashFactory.ADDRESS.create(Arrays.copyOfRange(transaction.getSignature(), 0, Hash.SIZE_IN_BYTES)); + addToNomineeQueue(newAddress); + // set start round + startRound = RoundIndexUtil.getRound(RoundIndexUtil.getCurrentTime(), config.getGenesisTime(), config.getRoundDuration(), config.getStartRoundDelay()); + log.info("Candidate Transaction " + transaction.getHash() + " is VALID, new Address: " + newAddress + ", start round: " + startRound); if (!transaction.isSolid()) { //int currentRoundIndex = (int) (System.currentTimeMillis() - config.getGenesisTime()) / config.getRoundDuration(); diff --git a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java index 6ac64927..ce8cb4ef 100755 --- a/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java +++ b/src/main/java/net/helix/hlx/service/curator/impl/CuratorServiceImpl.java @@ -60,7 +60,7 @@ public CuratorServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, } @Override - public CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel) throws CuratorException { + public CandidateValidity validateCandidate(TransactionViewModel transactionViewModel, SpongeFactory.Mode mode, int securityLevel, Set nominees) throws CuratorException { try { @@ -79,7 +79,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || validSignature) { + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (nominees.contains(senderAddress)) && validSignature) { return VALID; } else { return INVALID; diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index 783968f3..26e66d6f 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -5,7 +5,7 @@ import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; import net.helix.hlx.service.API; -import net.helix.hlx.service.nominee.NomineeTracker; +import net.helix.hlx.service.curator.CandidateTracker; import net.helix.hlx.service.utils.RoundIndexUtil; import net.helix.hlx.utils.bundle.BundleTypes; @@ -27,7 +27,7 @@ public class MilestonePublisher { private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private HelixConfig config; private API api; - private NomineeTracker nomineeTracker; + private CandidateTracker candidateTracker; private Hash address; private String message; @@ -44,10 +44,10 @@ public class MilestonePublisher { private boolean active; - public MilestonePublisher(HelixConfig configuration, API api, NomineeTracker nomineeTracker) { + public MilestonePublisher(HelixConfig configuration, API api, CandidateTracker candidateTracker) { this.config = configuration; this.api = api; - this.nomineeTracker = nomineeTracker; + this.candidateTracker = candidateTracker; delay = config.getRoundDuration(); mwm = config.getMwm(); @@ -97,7 +97,7 @@ private void doKeyChange() throws Exception { List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getNomineeSecurity()); Hash newAddress = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); // send keyChange bundle to register new address - api.publishKeyChange(address.toString(), newAddress.toString(), mwm, sign, currentKeyIndex, maxKeyIndex, config.getStartRoundDelay()); + api.publishKeyChange(address.toString(), newAddress, mwm, sign, currentKeyIndex, maxKeyIndex); // store new keyfile, address, keyfileidx keyfileIndex = newKeyfileIndex; address = newAddress; @@ -157,8 +157,10 @@ public void startScheduledExecutorService() { // - when starting with two nodes and one node leaving this deactivates the publisher private void publishMilestone() throws Exception { if (!active) { - if (startRound < getRound(RoundIndexUtil.getCurrentTime()) && !nomineeTracker.getLatestNominees().isEmpty() && nomineeTracker.getLatestNominees().contains(address)) { - startRound = nomineeTracker.getStartRound(); + System.out.println("Nominees: " + candidateTracker.getNominees()); + System.out.println("Address: " + address); + if (startRound < getRound(RoundIndexUtil.getCurrentTime()) && !candidateTracker.getNominees().isEmpty() && candidateTracker.getNominees().contains(address)) { + startRound = candidateTracker.getStartRound(); log.debug("Legitimized nominee {} for round #{}", address, startRound); } if (startRound == getRound(RoundIndexUtil.getCurrentTime())) { diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java index f90119c2..bb1f6f17 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestoneTrackerImpl.java @@ -7,7 +7,7 @@ import net.helix.hlx.crypto.SpongeFactory; import net.helix.hlx.model.Hash; import net.helix.hlx.service.milestone.*; -import net.helix.hlx.service.nominee.NomineeTracker; +import net.helix.hlx.service.curator.CandidateTracker; import net.helix.hlx.service.snapshot.SnapshotProvider; import net.helix.hlx.service.utils.RoundIndexUtil; import net.helix.hlx.storage.Tangle; @@ -69,7 +69,7 @@ public class MilestoneTrackerImpl implements MilestoneTracker { /** * Holds a reference to the manager that tracks nominees.
*/ - private NomineeTracker nomineeTracker; + private CandidateTracker candidateTracker; /** * Holds the addresses which are used to filter possible milestone candidates.
@@ -136,14 +136,14 @@ public class MilestoneTrackerImpl implements MilestoneTracker { * @return the initialized instance itself to allow chaining */ public MilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, NomineeTracker nomineeTracker, HelixConfig config) { + MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier, CandidateTracker candidateTracker, HelixConfig config) { this.tangle = tangle; this.config = config; this.snapshotProvider = snapshotProvider; this.milestoneService = milestoneService; this.milestoneSolidifier = milestoneSolidifier; - this.nomineeTracker = nomineeTracker; + this.candidateTracker = candidateTracker; allNominees = new HashSet<>(); @@ -152,7 +152,7 @@ public MilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide roundPause = 1000; //ms latestNomineeUpdate = 0; - setCurrentNominees(nomineeTracker.getLatestNominees()); + setCurrentNominees(candidateTracker.getNominees()); return this; } @@ -383,9 +383,9 @@ private void logProgress() { private void collectNewMilestoneCandidates() throws MilestoneException { try { // update nominees - if (nomineeTracker.getStartRound() == getCurrentRoundIndex() && latestNomineeUpdate < getCurrentRoundIndex()) { - setCurrentNominees(nomineeTracker.getLatestNominees()); - allNominees.addAll(nomineeTracker.getLatestNominees()); + if (candidateTracker.getStartRound() == getCurrentRoundIndex() && latestNomineeUpdate < getCurrentRoundIndex()) { + setCurrentNominees(candidateTracker.getNominees()); + allNominees.addAll(candidateTracker.getNominees()); } for (Hash address : allNominees) { for (Hash hash : AddressViewModel.load(tangle, address).getHashes()) { From 99efbb852264ddba79661544080cd447552d9ed0 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 17 Sep 2019 15:55:30 +0200 Subject: [PATCH 216/223] set startRoundDelay = 2 for faster key change --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 6b2d5c4c..c6093850 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1014,7 +1014,7 @@ public interface Defaults { boolean CURATOR_ENABLED = false; Hash CURATOR_ADDRESS = HashFactory.ADDRESS.create("9474289ae28f0ea6e3b8bedf8fc52f14d2fa9528a4eb29d7879d8709fd2f6d37"); int UPDATE_NOMINEE_DELAY = 30000; - int START_ROUND_DELAY = 5; + int START_ROUND_DELAY = 2; String CURATOR_KEYFILE = "./src/main/resources/Coordinator.key"; int CURATOR_KEY_DEPTH = 15; int CURATOR_SECURITY = 2; From e7d83d9dbcf5d9b2cdff6538ebf307f157d06ab6 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 17 Sep 2019 15:58:36 +0200 Subject: [PATCH 217/223] remove prints --- .../helix/hlx/service/milestone/impl/MilestonePublisher.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java index 26e66d6f..ac594036 100644 --- a/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/hlx/service/milestone/impl/MilestonePublisher.java @@ -157,8 +157,6 @@ public void startScheduledExecutorService() { // - when starting with two nodes and one node leaving this deactivates the publisher private void publishMilestone() throws Exception { if (!active) { - System.out.println("Nominees: " + candidateTracker.getNominees()); - System.out.println("Address: " + address); if (startRound < getRound(RoundIndexUtil.getCurrentTime()) && !candidateTracker.getNominees().isEmpty() && candidateTracker.getNominees().contains(address)) { startRound = candidateTracker.getStartRound(); log.debug("Legitimized nominee {} for round #{}", address, startRound); From 165757d4d3d3fca0af5048c87933f034bf7d33b5 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 17 Sep 2019 16:52:48 +0200 Subject: [PATCH 218/223] update changelog.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 433400c9..d65198b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## 0.6.0 - Finality Update integration. For more info see the [specifications](https://github.com/HelixNetwork/helix-specs/blob/master/specs/1.0/finality.md). +- Curator-less implementation +- Set initial nominees ## 0.5.9 From 8c452b43e3bd02c73c7ab8f095b97c1c18145f80 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 17 Sep 2019 16:53:04 +0200 Subject: [PATCH 219/223] updated nominee instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e1be2ea..c64b4514 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ Build an executable jar at the `target` directory using maven. java -jar target/helix-.jar -p 8085 ### Launch Nominee node -If you start a node using the `--nominee` flag a registration transaction is automatically submitted to the network and begins to issue votes for each round as soon as the respective address is accepted by the network. - +Launching a node as a nominee first requires to generate a 64 character hex string, that is used as a seed for key generation. You will find the public key in the last line of the `nominee.key` file contained in the resources directory. If you wish to act as a nominee, please send a request to dt@hlx.ai containing your public key. + java -jar target/helix-.jar -p 8085 --nominee ## Configuration From ea1ac87fe8bf188a21f28376f89810203764b867 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 17 Sep 2019 16:53:17 +0200 Subject: [PATCH 220/223] Added initial nominees --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index c6093850..f58d4fba 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1022,8 +1022,14 @@ public interface Defaults { //Milestone String NOMINEE = null; Set INITIAL_NOMINEES = new HashSet<>(Arrays.asList( - HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") + HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832"), + HashFactory.ADDRESS.create("a5afe01e64ae959f266b382bb5927fd07b49e7e3180239535126844aaae9bf93"), + HashFactory.ADDRESS.create("e2debe246b5d1a6e05b57b0fc14edb51d136966a91a803b523586ad032f72f3d"), + HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), + HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), + HashFactory.ADDRESS.create("1c6b0ee311a7ddccf255c1097995714b285cb06628be1cef2080b0bef7700e12") )); + long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) int ROUND_DURATION = 5000; int ROUND_PAUSE = 1000; From 47c198004fc4fc69e087267bddcd08a9dc44d40c Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 17 Sep 2019 17:17:16 +0200 Subject: [PATCH 221/223] Add nominee to initial nominees --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 3 ++- src/main/resources/nomineeTestSeed.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 src/main/resources/nomineeTestSeed.txt diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index f58d4fba..e80bc8d8 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1027,7 +1027,8 @@ public interface Defaults { HashFactory.ADDRESS.create("e2debe246b5d1a6e05b57b0fc14edb51d136966a91a803b523586ad032f72f3d"), HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), - HashFactory.ADDRESS.create("1c6b0ee311a7ddccf255c1097995714b285cb06628be1cef2080b0bef7700e12") + HashFactory.ADDRESS.create("1c6b0ee311a7ddccf255c1097995714b285cb06628be1cef2080b0bef7700e12"), + HashFactory.ADDRESS.create("c8af8e92d12080d4723f0d54c31b84eb866a856583bdbe37ddfc3cbac46947bd") )); long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) diff --git a/src/main/resources/nomineeTestSeed.txt b/src/main/resources/nomineeTestSeed.txt deleted file mode 100644 index 9bb5d129..00000000 --- a/src/main/resources/nomineeTestSeed.txt +++ /dev/null @@ -1 +0,0 @@ -da6fdb6593d701c63acca421bf88d3fcd6699454ef4c6d6520767989aa5c2cce \ No newline at end of file From 7c55f397d2237a17733a30e7e4780e112314bd38 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 17 Sep 2019 17:33:12 +0200 Subject: [PATCH 222/223] Set actual genesis time for testnet --- src/main/java/net/helix/hlx/conf/BaseHelixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index e80bc8d8..c48a8a31 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -1031,7 +1031,7 @@ public interface Defaults { HashFactory.ADDRESS.create("c8af8e92d12080d4723f0d54c31b84eb866a856583bdbe37ddfc3cbac46947bd") )); - long GENESIS_TIME = System.currentTimeMillis(); //todo replace this with actual genesis time (only for fast testing) + long GENESIS_TIME = 1568725976628L; //for local testing: System.currentTimeMillis(); int ROUND_DURATION = 5000; int ROUND_PAUSE = 1000; String NOMINEE_KEYFILE = "./src/main/resources/Nominee.key"; From f7bcbeb8221975189489d419d1ef85bd850f9b37 Mon Sep 17 00:00:00 2001 From: ofo42 Date: Tue, 17 Sep 2019 17:37:05 +0200 Subject: [PATCH 223/223] bump to 0.6.1 --- CHANGELOG.md | 7 +++++-- README.md | 2 +- pom.xml | 2 +- src/main/java/net/helix/hlx/HLX.java | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d65198b2..4e968eec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ -## 0.6.0 -- Finality Update integration. For more info see the [specifications](https://github.com/HelixNetwork/helix-specs/blob/master/specs/1.0/finality.md). +## 0.6.1 - Curator-less implementation - Set initial nominees +- Set testnet genesis time + +## 0.6.0 +- Finality Update integration. For more info see the [specifications](https://github.com/HelixNetwork/helix-specs/blob/master/specs/1.0/finality.md). ## 0.5.9 diff --git a/README.md b/README.md index c64b4514..d1fc7036 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ # Helix A Quorum based Tangle implementation forked from [**IRI**](https://github.com/iotaledger/iri/). -- **Latest release:** 0.6.0 pre-release +- **Latest release:** 0.6.1 pre-release - **License:** GPLv3 Special thanks to all of the [IOTA Contributors](https://github.com/iotaledger/iri/graphs/contributors)! diff --git a/pom.xml b/pom.xml index b9c2d207..23b2ed76 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ net.helix helix - 0.6.0 + 0.6.1 Helix Helix Protocol diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 1901a3d4..48aaf36e 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -46,7 +46,7 @@ public class HLX { public static final String MAINNET_NAME = "HLX"; public static final String TESTNET_NAME = "HLX Testnet"; - public static final String VERSION = "0.6.0"; + public static final String VERSION = "0.6.1"; /** * The entry point of the helix sandbox.