From b1b3e231dab5b2e2e4104f23f043519e2439667c Mon Sep 17 00:00:00 2001 From: fsbbn Date: Tue, 11 Jun 2019 14:41:35 +0200 Subject: [PATCH 1/6] internal spammer --- src/main/java/net/helix/hlx/HLX.java | 4 ++++ src/main/java/net/helix/hlx/service/API.java | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 7ece6c7d..662d3337 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -6,6 +6,7 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.service.API; import net.helix.hlx.service.milestone.MSS; +import net.helix.hlx.service.Spammer; import com.beust.jcommander.JCommander; import com.beust.jcommander.ParameterException; @@ -96,6 +97,7 @@ private static class HLXLauncher { public static API api; public static HXI hxi; public static MSS mss; + public static Spammer spammer; /** * Starts hlx. Setup is as follows: @@ -123,6 +125,7 @@ public static void main(String [] args) throws Exception { helix.tipsViewModel, helix.transactionValidator, helix.latestMilestoneTracker, helix.graph); mss = new MSS(config, api); + spammer = new Spammer(api); shutdownHook(); try { @@ -138,6 +141,7 @@ public static void main(String [] args) throws Exception { if(config.getMsDelay() > 0) { mss.startScheduledExecutorService(); } + spammer.startScheduledExecutorService(); } /** diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 04b35bbf..129c5681 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -1419,12 +1419,12 @@ public void shutDown() { * @param message The message to store **/ private synchronized AbstractResponse storeMessageStatement(final String address, final String message) throws Exception { - final List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - attachStoreAndBroadcast(address, message, txToApprove); + attachStoreAndBroadcast(address, message); return AbstractResponse.createEmptyResponse(); } - private void attachStoreAndBroadcast(final String address, final String message, final List txToApprove) throws Exception { + public void attachStoreAndBroadcast(final String address, final String message) throws Exception { + final List txToApprove = getTransactionToApproveTips(3, Optional.empty()); attachStoreAndBroadcast(address, message, txToApprove, 0, 1, false); } From 8bc4373ab03f85b0e7b0d5679be703ba4e0b4eac Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 12 Jun 2019 13:47:09 +0200 Subject: [PATCH 2/6] add spammer --- .../java/net/helix/hlx/service/Spammer.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/main/java/net/helix/hlx/service/Spammer.java diff --git a/src/main/java/net/helix/hlx/service/Spammer.java b/src/main/java/net/helix/hlx/service/Spammer.java new file mode 100644 index 00000000..93e5f653 --- /dev/null +++ b/src/main/java/net/helix/hlx/service/Spammer.java @@ -0,0 +1,52 @@ +package net.helix.hlx.service; + +import net.helix.hlx.model.Hash; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class Spammer { + private static final Logger log = LoggerFactory.getLogger(Spammer.class); + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + private API api; + + private String address; + private String message; + private int delay; + + public Spammer(API api) { + this.api = api; + this.message = StringUtils.repeat('0', 1024*2); + this.address = Hash.NULL_HASH.hexString(); + this.delay = 50; + } + + public void startScheduledExecutorService() { + log.info("Spammer scheduledExecutorService started."); + log.info("Submitting Tx every: " + this.delay + "ms."); + this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnableSendTx(), 10000, this.delay, TimeUnit.MILLISECONDS); + } + + private void sendTx() throws Exception { + this.api.attachStoreAndBroadcast(this.address, this.message); + } + + private Runnable getRunnableSendTx() { + return () -> { + try { + sendTx(); + } catch (Exception e) { + e.printStackTrace(); + } + }; + } + + public void shutdown() { + log.info("Shutting down Spammer Thread"); + scheduledExecutorService.shutdown(); + } +} From 227b46b013b3756bc11383af3d3439f153bd4367 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 12 Jun 2019 13:47:59 +0200 Subject: [PATCH 3/6] reverse transactions before attaching --- src/main/java/net/helix/hlx/service/API.java | 6 ++- .../service/restserver/resteasy/RestEasy.java | 40 +++++++++---------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/API.java b/src/main/java/net/helix/hlx/service/API.java index 129c5681..a6e804ab 100644 --- a/src/main/java/net/helix/hlx/service/API.java +++ b/src/main/java/net/helix/hlx/service/API.java @@ -7,9 +7,12 @@ import net.helix.hlx.conf.HelixConfig; import net.helix.hlx.controllers.*; import net.helix.hlx.crypto.*; +import net.helix.hlx.model.BundleHash; import net.helix.hlx.model.Hash; import net.helix.hlx.model.HashFactory; +import net.helix.hlx.model.TransactionHash; import net.helix.hlx.model.persistables.Transaction; +import net.helix.hlx.model.persistables.Bundle; import net.helix.hlx.network.Neighbor; import net.helix.hlx.network.Node; import net.helix.hlx.network.TransactionRequester; @@ -1439,7 +1442,7 @@ private void attachStoreAndBroadcast(final String address, final String message, byte[] currentIndexBytes = new byte[TransactionViewModel.CURRENT_INDEX_SIZE]; final byte[] lastIndexBytes = new byte[TransactionViewModel.LAST_INDEX_SIZE]; - final String lastIndexHex = isMilestone ? Hex.toHexString(lastIndexBytes) : Integer.toHexString(txCount); // TODO: lastIndex has to be 0 for milestones and based on txCount for other tx. + final String lastIndexHex = isMilestone ? Hex.toHexString(lastIndexBytes) : Integer.toHexString(txCount-1); // TODO: lastIndex has to be 0 for milestones and based on txCount for other tx. final byte[] newMilestoneIndex = Serializer.serialize(index); final String tagHex = Hex.toHexString(newMilestoneIndex); // milestoneTracker index is parsed from tag @@ -1479,6 +1482,7 @@ private void attachStoreAndBroadcast(final String address, final String message, sponge.squeeze(essenceHash, 0, essenceHash.length); final String bundleHash = Hex.toHexString(essenceHash); transactions = transactions.stream().map(tx -> StringUtils.rightPad(tx + bundleHash + StringUtils.repeat('0', 128) + tagHex, BYTES_SIZE, '0')).collect(Collectors.toList()); + Collections.reverse(transactions); List powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions); storeTransactionsStatement(powResult); broadcastTransactionsStatement(powResult); diff --git a/src/main/java/net/helix/hlx/service/restserver/resteasy/RestEasy.java b/src/main/java/net/helix/hlx/service/restserver/resteasy/RestEasy.java index 22bb3a72..2a718f8e 100644 --- a/src/main/java/net/helix/hlx/service/restserver/resteasy/RestEasy.java +++ b/src/main/java/net/helix/hlx/service/restserver/resteasy/RestEasy.java @@ -139,28 +139,28 @@ public void init(ApiProcessor processFunction) { info.setDeploymentName("Helix Realm"); info.setContextPath("/"); /** TODO: check credentials here instead of in process request - info.addSecurityWrapper(new HandlerWrapper() { - @Override - public HttpHandler wrap(HttpHandler toWrap) { - String credentials = remoteAuth; - if (credentials == null || credentials.isEmpty()) { - return toWrap; - } + info.addSecurityWrapper(new HandlerWrapper() { + @Override + public HttpHandler wrap(HttpHandler toWrap) { + String credentials = remoteAuth; + if (credentials == null || credentials.isEmpty()) { + return toWrap; + } - final Map users = new HashMap<>(2); - users.put(credentials.split(":")[0], credentials.split(":")[1].toCharArray()); + final Map users = new HashMap<>(2); + users.put(credentials.split(":")[0], credentials.split(":")[1].toCharArray()); - IdentityManager identityManager = new MapIdentityManager(users); - HttpHandler handler = toWrap; - handler = new AuthenticationCallHandler(handler); - handler = new AuthenticationConstraintHandler(handler); - final List mechanisms = - Collections.singletonList(new BasicAuthenticationMechanism("Helix Realm")); + IdentityManager identityManager = new MapIdentityManager(users); + HttpHandler handler = toWrap; + handler = new AuthenticationCallHandler(handler); + handler = new AuthenticationConstraintHandler(handler); + final List mechanisms = + Collections.singletonList(new BasicAuthenticationMechanism("Helix Realm")); - handler = new AuthenticationMechanismsHandler(handler, mechanisms); - handler = new SecurityInitialHandler(AuthenticationMode.PRO_ACTIVE, identityManager, handler); - return handler; - } + handler = new AuthenticationMechanismsHandler(handler, mechanisms); + handler = new SecurityInitialHandler(AuthenticationMode.PRO_ACTIVE, identityManager, handler); + return handler; + } });*/ info.addInnerHandlerChainWrapper(handler -> { @@ -318,4 +318,4 @@ private static void setupResponseHeaders(HttpServerExchange exchange) { headerMap.add(new HttpString("Access-Control-Allow-Origin"),"*"); headerMap.add(new HttpString("Keep-Alive"), "timeout=500, max=100"); } -} +} \ No newline at end of file From 1b08eb8204719793382f167fe460cb89fcbfc069 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 12 Jun 2019 16:00:06 +0200 Subject: [PATCH 4/6] logging level --- src/main/java/net/helix/hlx/HLX.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index 662d3337..d7df2396 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -68,7 +68,7 @@ public static void main(String[] args) throws Exception { private static void configureLogging() { HelixIOUtils.saveLogs(); // TODO: Find a solution, that allows to save the logs under the condition of `config.isSaveLogEnabled()`. String config = System.getProperty("logback.configurationFile"); - String level = System.getProperty("logging-level", "debug").toUpperCase(); + String level = System.getProperty("logging-level", "info").toUpperCase(); switch (level) { case "OFF": case "ERROR": From b9b11c83a11507907d5ca55891df10c8ac12984b Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 12 Jun 2019 16:00:29 +0200 Subject: [PATCH 5/6] fixing confirmed tps --- .../stats/TimeWindowedApproveeCounter.java | 22 ++++++++++++++----- .../stats/TransactionStatsPublisher.java | 8 +++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/helix/hlx/service/stats/TimeWindowedApproveeCounter.java b/src/main/java/net/helix/hlx/service/stats/TimeWindowedApproveeCounter.java index efec0a0b..d46c6658 100644 --- a/src/main/java/net/helix/hlx/service/stats/TimeWindowedApproveeCounter.java +++ b/src/main/java/net/helix/hlx/service/stats/TimeWindowedApproveeCounter.java @@ -31,6 +31,10 @@ private boolean isOlderThanMaxAge(Instant now, TransactionViewModel transaction) return age <= maxTransactionAgeSeconds; } + private boolean isConfirmed(Instant now, TransactionViewModel transaction) { + return transaction.snapshotIndex() > 0; + } + /** * Checks whether the transaction is in the validity time window. * @@ -56,14 +60,14 @@ boolean isInTimeWindow(Instant now, TransactionViewModel transaction) { * processed hashes. * @throws TraversalException if anything goes wrong while traversing the graph and processing the transactions */ - long getCount(Instant now, Hash startingTransactionHash, HashSet processedTransactions) + long getCount(Instant now, Hash startingTransactionHash, HashSet processedTransactions, boolean onlyConfirmed) throws TraversalException { DAGHelper helper = DAGHelper.get(tangle); - ValidTransactionCounter transactionCounter = new ValidTransactionCounter(now); + ValidTransactionCounter transactionCounter = new ValidTransactionCounter(now, onlyConfirmed); helper.traverseApprovees(startingTransactionHash, t -> isOlderThanMaxAge(now, t), transactionCounter::count, - processedTransactions); + processedTransactions); return transactionCounter.getCount(); } @@ -76,16 +80,24 @@ long getCount(Instant now, Hash startingTransactionHash, HashSet processed private final class ValidTransactionCounter { final Instant now; + boolean confirmed; long count = 0; - private ValidTransactionCounter(Instant now) { + private ValidTransactionCounter(Instant now, boolean onlyConfirmed) { this.now = now; + this.confirmed = onlyConfirmed; } private void count(TransactionViewModel transactionViewModel) { if (isInTimeWindow(now, transactionViewModel)) { - count++; + if (confirmed) { + if (isConfirmed(now, transactionViewModel)){ + count++; + } + } else { + count++; + } } } diff --git a/src/main/java/net/helix/hlx/service/stats/TransactionStatsPublisher.java b/src/main/java/net/helix/hlx/service/stats/TransactionStatsPublisher.java index 44456f7b..415c1256 100644 --- a/src/main/java/net/helix/hlx/service/stats/TransactionStatsPublisher.java +++ b/src/main/java/net/helix/hlx/service/stats/TransactionStatsPublisher.java @@ -26,7 +26,7 @@ */ public class TransactionStatsPublisher { - private static final long PUBLISH_INTERVAL = Duration.ofSeconds(30).toMillis(); + private static final long PUBLISH_INTERVAL = Duration.ofSeconds(10).toMillis(); private static final String CONFIRMED_TRANSACTIONS_TOPIC = "ct5s2m"; private static final String TOTAL_TRANSACTIONS_TOPIC = "t5s2m"; @@ -94,7 +94,7 @@ private Hash getSuperTip() throws Exception { private long getConfirmedTransactionsCount(Instant now) throws Exception { - return approveeCounter.getCount(now, getSuperTip(), new HashSet<>()); + return approveeCounter.getCount(now, getSuperTip(), new HashSet<>(), true); } private long getAllTransactionsCount(Instant now) throws Exception { @@ -106,10 +106,10 @@ private long getAllTransactionsCount(Instant now) throws Exception { // count the tip, if it is the valid time window log.debug("DZMQ: {}", tip.hexString()); if (approveeCounter.isInTimeWindow(now, TransactionViewModel.fromHash(tangle, tip))) { - count += 1 + approveeCounter.getCount(now, tip, processedTransactions); + count += 1 + approveeCounter.getCount(now, tip, processedTransactions, false); } else { // even if the tip is not in the time window, count approvees that might be older - count += approveeCounter.getCount(now, tip, processedTransactions); + count += approveeCounter.getCount(now, tip, processedTransactions, false); } } From 7359bd55cac042471992777b8b2e43252529d5b0 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 12 Jun 2019 21:09:27 +0200 Subject: [PATCH 6/6] spam flag --- src/main/java/net/helix/hlx/HLX.java | 6 ++++-- .../java/net/helix/hlx/conf/BaseHelixConfig.java | 14 ++++++++++++++ src/main/java/net/helix/hlx/conf/HelixConfig.java | 2 +- src/main/java/net/helix/hlx/conf/SpamConfig.java | 12 ++++++++++++ src/main/java/net/helix/hlx/service/Spammer.java | 7 +++++-- 5 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 src/main/java/net/helix/hlx/conf/SpamConfig.java diff --git a/src/main/java/net/helix/hlx/HLX.java b/src/main/java/net/helix/hlx/HLX.java index d7df2396..ea832f63 100644 --- a/src/main/java/net/helix/hlx/HLX.java +++ b/src/main/java/net/helix/hlx/HLX.java @@ -125,7 +125,7 @@ public static void main(String [] args) throws Exception { helix.tipsViewModel, helix.transactionValidator, helix.latestMilestoneTracker, helix.graph); mss = new MSS(config, api); - spammer = new Spammer(api); + spammer = new Spammer(config, api); shutdownHook(); try { @@ -141,7 +141,9 @@ public static void main(String [] args) throws Exception { if(config.getMsDelay() > 0) { mss.startScheduledExecutorService(); } - spammer.startScheduledExecutorService(); + if(config.getSpamDelay() > 0) { + spammer.startScheduledExecutorService(); + } } /** diff --git a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java index 0e553db3..66f1fc4e 100644 --- a/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/BaseHelixConfig.java @@ -112,6 +112,9 @@ public abstract class BaseHelixConfig implements HelixConfig { protected int minDelay = Defaults.MS_MIN_DELAY; protected String cooAddress = Defaults.COORDINATOR_ADDRESS; + //Spammer + protected int spamDelay = Defaults.SPAM_DELAY; + public BaseHelixConfig() { //empty constructor } @@ -804,6 +807,14 @@ public String getSaveLogXMLFile() { @Parameter(names = {"--savelog-xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE) protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; } + @Override + public int getSpamDelay() { + return spamDelay; + } + @JsonProperty + @Parameter(names = {"--spam"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE) + protected void setSpamDelay(int spamDelay) { this.spamDelay = spamDelay; } + public interface Defaults { //API int API_PORT = 14700; @@ -896,5 +907,8 @@ public interface Defaults { boolean SAVELOG_ENABLED = false; String SAVELOG_BASE_PATH = "logs/"; String SAVELOG_XML_FILE = "/logback-save.xml"; + + //Spammer + int SPAM_DELAY = 0; } } diff --git a/src/main/java/net/helix/hlx/conf/HelixConfig.java b/src/main/java/net/helix/hlx/conf/HelixConfig.java index 883df99c..6de1e1e9 100644 --- a/src/main/java/net/helix/hlx/conf/HelixConfig.java +++ b/src/main/java/net/helix/hlx/conf/HelixConfig.java @@ -8,7 +8,7 @@ * A container for all possible configuration parameters of SBX. * In charge of how we parse the configuration from given inputs. */ -public interface HelixConfig extends APIConfig, NodeConfig, HXIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, GraphConfig, LoggingConfig { +public interface HelixConfig extends APIConfig, NodeConfig, HXIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, GraphConfig, LoggingConfig, SpamConfig { File CONFIG_FILE = new File("hlx.ini"); /** * Parses the args to populate the configuration object diff --git a/src/main/java/net/helix/hlx/conf/SpamConfig.java b/src/main/java/net/helix/hlx/conf/SpamConfig.java new file mode 100644 index 00000000..cd599fb9 --- /dev/null +++ b/src/main/java/net/helix/hlx/conf/SpamConfig.java @@ -0,0 +1,12 @@ +package net.helix.hlx.conf; + +public interface SpamConfig { + /** + * @return {@value Descriptions#GET_SPAM_DELAY} + */ + int getSpamDelay(); + + interface Descriptions { + String GET_SPAM_DELAY = "Delay of spam."; + } +} diff --git a/src/main/java/net/helix/hlx/service/Spammer.java b/src/main/java/net/helix/hlx/service/Spammer.java index 93e5f653..5128082f 100644 --- a/src/main/java/net/helix/hlx/service/Spammer.java +++ b/src/main/java/net/helix/hlx/service/Spammer.java @@ -1,5 +1,6 @@ package net.helix.hlx.service; +import net.helix.hlx.conf.SpamConfig; import net.helix.hlx.model.Hash; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -13,16 +14,18 @@ public class Spammer { private static final Logger log = LoggerFactory.getLogger(Spammer.class); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private API api; + private SpamConfig config; private String address; private String message; private int delay; - public Spammer(API api) { + public Spammer(SpamConfig config, API api) { this.api = api; + this.config = config; this.message = StringUtils.repeat('0', 1024*2); this.address = Hash.NULL_HASH.hexString(); - this.delay = 50; + this.delay = config.getSpamDelay(); } public void startScheduledExecutorService() {