Spammer and confirmed TPS fix

src/main/java/net/helix/hlx/HLX.java

@@ -6,6 +6,7 @@
 import net.helix.hlx.conf.HelixConfig;
 import net.helix.hlx.service.API;
 import net.helix.hlx.service.milestone.MSS;
+import net.helix.hlx.service.Spammer;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.ParameterException;
@@ -67,7 +68,7 @@ public static void main(String[] args) throws Exception {
     private static void configureLogging() {
         HelixIOUtils.saveLogs(); // TODO: Find a solution, that allows to save the logs under the condition of `config.isSaveLogEnabled()`.
         String config = System.getProperty("logback.configurationFile");
-        String level = System.getProperty("logging-level", "debug").toUpperCase();
+        String level = System.getProperty("logging-level", "info").toUpperCase();
         switch (level) {
             case "OFF":
             case "ERROR":
@@ -96,6 +97,7 @@ private static class HLXLauncher {
         public static API api;
         public static HXI hxi;
         public static MSS mss;
+        public static Spammer spammer;
 
         /**
          * Starts hlx. Setup is as follows:
@@ -123,6 +125,7 @@ public static void main(String [] args) throws Exception {
                     helix.tipsViewModel, helix.transactionValidator,
                     helix.latestMilestoneTracker, helix.graph);
             mss = new MSS(config, api);
+            spammer = new Spammer(config, api);
             shutdownHook();
 
             try {
@@ -138,6 +141,9 @@ public static void main(String [] args) throws Exception {
             if(config.getMsDelay() > 0) {
                 mss.startScheduledExecutorService();
             }
+            if(config.getSpamDelay() > 0) {
+                spammer.startScheduledExecutorService();
+            }
         }
 
         /**

src/main/java/net/helix/hlx/conf/BaseHelixConfig.java

@@ -112,6 +112,9 @@ public abstract class BaseHelixConfig implements HelixConfig {
     protected int minDelay = Defaults.MS_MIN_DELAY;
     protected String cooAddress = Defaults.COORDINATOR_ADDRESS;
 
+    //Spammer
+    protected int spamDelay = Defaults.SPAM_DELAY;
+
     public BaseHelixConfig() {
         //empty constructor
     }
@@ -804,6 +807,14 @@ public String getSaveLogXMLFile() {
     @Parameter(names = {"--savelog-xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE)
     protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; }
 
+    @Override
+    public int getSpamDelay() {
+        return spamDelay;
+    }
+    @JsonProperty
+    @Parameter(names = {"--spam"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE)
+    protected void setSpamDelay(int spamDelay) { this.spamDelay = spamDelay; }
+
     public interface Defaults {
         //API
         int API_PORT = 14700;
@@ -896,5 +907,8 @@ public interface Defaults {
         boolean SAVELOG_ENABLED = false;
         String SAVELOG_BASE_PATH = "logs/";
         String SAVELOG_XML_FILE = "/logback-save.xml";
+
+        //Spammer
+        int SPAM_DELAY = 0;
     }
 }

src/main/java/net/helix/hlx/conf/HelixConfig.java

@@ -8,7 +8,7 @@
  *  A container for all possible configuration parameters of SBX.
  *  In charge of how we parse the configuration from given inputs.
  */
-public interface HelixConfig extends APIConfig, NodeConfig, HXIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, GraphConfig, LoggingConfig {
+public interface HelixConfig extends APIConfig, NodeConfig, HXIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, GraphConfig, LoggingConfig, SpamConfig {
     File CONFIG_FILE = new File("hlx.ini");
     /**
      * Parses the args to populate the configuration object

src/main/java/net/helix/hlx/conf/SpamConfig.java

@@ -0,0 +1,12 @@
+package net.helix.hlx.conf;
+
+public interface SpamConfig {
+    /**
+     * @return {@value Descriptions#GET_SPAM_DELAY}
+     */
+    int getSpamDelay();
+
+    interface Descriptions {
+        String GET_SPAM_DELAY = "Delay of spam.";
+    }
+}

src/main/java/net/helix/hlx/service/API.java

@@ -7,9 +7,12 @@
 import net.helix.hlx.conf.HelixConfig;
 import net.helix.hlx.controllers.*;
 import net.helix.hlx.crypto.*;
+import net.helix.hlx.model.BundleHash;
 import net.helix.hlx.model.Hash;
 import net.helix.hlx.model.HashFactory;
+import net.helix.hlx.model.TransactionHash;
 import net.helix.hlx.model.persistables.Transaction;
+import net.helix.hlx.model.persistables.Bundle;
 import net.helix.hlx.network.Neighbor;
 import net.helix.hlx.network.Node;
 import net.helix.hlx.network.TransactionRequester;
@@ -1419,12 +1422,12 @@ public void shutDown() {
      * @param message The message to store
      **/
     private synchronized AbstractResponse storeMessageStatement(final String address, final String message) throws Exception {
-        final List<Hash> txToApprove = getTransactionToApproveTips(3, Optional.empty());
-        attachStoreAndBroadcast(address, message, txToApprove);
+        attachStoreAndBroadcast(address, message);
         return AbstractResponse.createEmptyResponse();
     }
 
-    private void attachStoreAndBroadcast(final String address, final String message, final List<Hash> txToApprove) throws Exception {
+    public void attachStoreAndBroadcast(final String address, final String message) throws Exception {
+        final List<Hash> txToApprove = getTransactionToApproveTips(3, Optional.empty());
         attachStoreAndBroadcast(address, message, txToApprove, 0, 1, false);
     }
 
@@ -1439,7 +1442,7 @@ private void attachStoreAndBroadcast(final String address, final String message,
         byte[] currentIndexBytes = new byte[TransactionViewModel.CURRENT_INDEX_SIZE];
 
         final byte[] lastIndexBytes = new byte[TransactionViewModel.LAST_INDEX_SIZE];
-        final String lastIndexHex = isMilestone ? Hex.toHexString(lastIndexBytes) : Integer.toHexString(txCount); // TODO: lastIndex has to be 0 for milestones and based on txCount for other tx.
+        final String lastIndexHex = isMilestone ? Hex.toHexString(lastIndexBytes) : Integer.toHexString(txCount-1); // TODO: lastIndex has to be 0 for milestones and based on txCount for other tx.
 
         final byte[] newMilestoneIndex = Serializer.serialize(index);
         final String tagHex = Hex.toHexString(newMilestoneIndex);  // milestoneTracker index is parsed from tag
@@ -1479,6 +1482,7 @@ private void attachStoreAndBroadcast(final String address, final String message,
         sponge.squeeze(essenceHash, 0, essenceHash.length);
         final String bundleHash = Hex.toHexString(essenceHash);
         transactions = transactions.stream().map(tx -> StringUtils.rightPad(tx + bundleHash + StringUtils.repeat('0', 128) + tagHex, BYTES_SIZE, '0')).collect(Collectors.toList());
+        Collections.reverse(transactions);
         List<String> powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions);
         storeTransactionsStatement(powResult);
         broadcastTransactionsStatement(powResult);

src/main/java/net/helix/hlx/service/Spammer.java

@@ -0,0 +1,55 @@
+package net.helix.hlx.service;
+
+import net.helix.hlx.conf.SpamConfig;
+import net.helix.hlx.model.Hash;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class Spammer {
+    private static final Logger log = LoggerFactory.getLogger(Spammer.class);
+    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+    private API api;
+    private SpamConfig config;
+
+    private String address;
+    private String message;
+    private int delay;
+
+    public Spammer(SpamConfig config, API api) {
+        this.api = api;
+        this.config = config;
+        this.message = StringUtils.repeat('0', 1024*2);
+        this.address = Hash.NULL_HASH.hexString();
+        this.delay = config.getSpamDelay();
+    }
+
+    public void startScheduledExecutorService() {
+        log.info("Spammer scheduledExecutorService started.");
+        log.info("Submitting Tx every: " + this.delay + "ms.");
+        this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnableSendTx(), 10000, this.delay,  TimeUnit.MILLISECONDS);
+    }
+
+    private void sendTx() throws Exception {
+        this.api.attachStoreAndBroadcast(this.address, this.message);
+    }
+
+    private Runnable getRunnableSendTx() {
+        return () -> {
+            try {
+                sendTx();
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        };
+    }
+
+    public void shutdown() {
+        log.info("Shutting down Spammer Thread");
+        scheduledExecutorService.shutdown();
+    }
+}

src/main/java/net/helix/hlx/service/restserver/resteasy/RestEasy.java

@@ -139,28 +139,28 @@ public void init(ApiProcessor processFunction) {
         info.setDeploymentName("Helix Realm");
         info.setContextPath("/");
         /** TODO: check credentials here instead of in process request
-        info.addSecurityWrapper(new HandlerWrapper() {
-            @Override
-            public HttpHandler wrap(HttpHandler toWrap) {
-                String credentials = remoteAuth;
-                if (credentials == null || credentials.isEmpty()) {
-                    return toWrap;
-                }
+         info.addSecurityWrapper(new HandlerWrapper() {
+        @Override
+        public HttpHandler wrap(HttpHandler toWrap) {
+        String credentials = remoteAuth;
+        if (credentials == null || credentials.isEmpty()) {
+        return toWrap;
+        }
 
-                final Map<String, char[]> users = new HashMap<>(2);
-                users.put(credentials.split(":")[0], credentials.split(":")[1].toCharArray());
+        final Map<String, char[]> users = new HashMap<>(2);
+        users.put(credentials.split(":")[0], credentials.split(":")[1].toCharArray());
 
-                IdentityManager identityManager = new MapIdentityManager(users);
-                HttpHandler handler = toWrap;
-                handler = new AuthenticationCallHandler(handler);
-                handler = new AuthenticationConstraintHandler(handler);
-                final List<AuthenticationMechanism> mechanisms =
-                        Collections.singletonList(new BasicAuthenticationMechanism("Helix Realm"));
+        IdentityManager identityManager = new MapIdentityManager(users);
+        HttpHandler handler = toWrap;
+        handler = new AuthenticationCallHandler(handler);
+        handler = new AuthenticationConstraintHandler(handler);
+        final List<AuthenticationMechanism> mechanisms =
+        Collections.singletonList(new BasicAuthenticationMechanism("Helix Realm"));
 
-                handler = new AuthenticationMechanismsHandler(handler, mechanisms);
-                handler = new SecurityInitialHandler(AuthenticationMode.PRO_ACTIVE, identityManager, handler);
-                return handler;
-            }
+        handler = new AuthenticationMechanismsHandler(handler, mechanisms);
+        handler = new SecurityInitialHandler(AuthenticationMode.PRO_ACTIVE, identityManager, handler);
+        return handler;
+        }
         });*/
 
         info.addInnerHandlerChainWrapper(handler -> {
@@ -318,4 +318,4 @@ private static void setupResponseHeaders(HttpServerExchange exchange) {
         headerMap.add(new HttpString("Access-Control-Allow-Origin"),"*");
         headerMap.add(new HttpString("Keep-Alive"), "timeout=500, max=100");
     }
-}
+}

src/main/java/net/helix/hlx/service/stats/TimeWindowedApproveeCounter.java

@@ -31,6 +31,10 @@ private boolean isOlderThanMaxAge(Instant now, TransactionViewModel transaction)
         return age <= maxTransactionAgeSeconds;
     }
 
+    private boolean isConfirmed(Instant now, TransactionViewModel transaction) {
+        return transaction.snapshotIndex() > 0;
+    }
+
     /**
      * Checks whether the transaction is in the validity time window.
      *
@@ -56,14 +60,14 @@ boolean isInTimeWindow(Instant now, TransactionViewModel transaction) {
      *                                processed hashes.
      * @throws TraversalException if anything goes wrong while traversing the graph and processing the transactions
      */
-    long getCount(Instant now, Hash startingTransactionHash, HashSet<Hash> processedTransactions)
+    long getCount(Instant now, Hash startingTransactionHash, HashSet<Hash> processedTransactions, boolean onlyConfirmed)
             throws TraversalException {
 
         DAGHelper helper = DAGHelper.get(tangle);
 
-        ValidTransactionCounter transactionCounter = new ValidTransactionCounter(now);
+        ValidTransactionCounter transactionCounter = new ValidTransactionCounter(now, onlyConfirmed);
         helper.traverseApprovees(startingTransactionHash, t -> isOlderThanMaxAge(now, t), transactionCounter::count,
-                processedTransactions);
+                    processedTransactions);
 
         return transactionCounter.getCount();
     }
@@ -76,16 +80,24 @@ long getCount(Instant now, Hash startingTransactionHash, HashSet<Hash> processed
     private final class ValidTransactionCounter {
 
         final Instant now;
+        boolean confirmed;
 
         long count = 0;
 
-        private ValidTransactionCounter(Instant now) {
+        private ValidTransactionCounter(Instant now, boolean onlyConfirmed) {
             this.now = now;
+            this.confirmed = onlyConfirmed;
         }
 
         private void count(TransactionViewModel transactionViewModel) {
             if (isInTimeWindow(now, transactionViewModel)) {
-                count++;
+                if (confirmed) {
+                    if (isConfirmed(now, transactionViewModel)){
+                        count++;
+                    }
+                } else {
+                    count++;
+                }
             }
         }
 

src/main/java/net/helix/hlx/service/stats/TransactionStatsPublisher.java

@@ -26,7 +26,7 @@
  */
 public class TransactionStatsPublisher {
 
-    private static final long PUBLISH_INTERVAL = Duration.ofSeconds(30).toMillis();
+    private static final long PUBLISH_INTERVAL = Duration.ofSeconds(10).toMillis();
 
     private static final String CONFIRMED_TRANSACTIONS_TOPIC = "ct5s2m";
     private static final String TOTAL_TRANSACTIONS_TOPIC = "t5s2m";
@@ -94,7 +94,7 @@ private Hash getSuperTip() throws Exception {
 
     private long getConfirmedTransactionsCount(Instant now) throws Exception {
 
-        return approveeCounter.getCount(now, getSuperTip(), new HashSet<>());
+        return approveeCounter.getCount(now, getSuperTip(), new HashSet<>(), true);
     }
 
     private long getAllTransactionsCount(Instant now) throws Exception {
@@ -106,10 +106,10 @@ private long getAllTransactionsCount(Instant now) throws Exception {
             // count the tip, if it is the valid time window
             log.debug("DZMQ: {}", tip.hexString());
             if (approveeCounter.isInTimeWindow(now, TransactionViewModel.fromHash(tangle, tip))) {
-                count += 1 + approveeCounter.getCount(now, tip, processedTransactions);
+                count += 1 + approveeCounter.getCount(now, tip, processedTransactions, false);
             } else {
                 // even if the tip is not in the time window, count approvees that might be older
-                count += approveeCounter.getCount(now, tip, processedTransactions);
+                count += approveeCounter.getCount(now, tip, processedTransactions, false);
             }
         }