merge dev

Dockerfile

@@ -14,4 +14,4 @@ EXPOSE 14700/udp
 EXPOSE 14700
 EXPOSE 14600/udp
 
-CMD ["/usr/bin/java", "-XX:+DisableAttachMechanism", "-Xmx8g", "-Xms256m", "-Dlogback.configurationFile=/testnet-1.0/conf/logback.xml", "-Djava.net.preferIPv4Stack=true", "-jar", "helix*.jar", "-p", "14700", "-u", "14600", "--remote", "$@"]
+ENTRYPOINT ["/usr/bin/java", "-XX:+DisableAttachMechanism", "-Xmx8g", "-Xms256m", "-Dlogback.configurationFile=/testnet-1.0/conf/logback.xml", "-Djava.net.preferIPv4Stack=true", "-jar", "helix*.jar", "-p", "14700", "-u", "14600", "--remote", "$@"]

README.md

@@ -4,36 +4,24 @@
 
 # Helix Protocol
 This is the 1.0 implementation of the Helix Protocol based on [**IRI**](https://github.com/iotaledger/iri/).
-* **Latest release:** 0.5.3 pre-release
+* **Latest release:** 0.5.4 pre-release
 * **License:** GPLv3
 
 ## Developers
 
 - Please see the [CONTRIBUTING.md](https://github.com/HelixNetwork/testnet-1.0/blob/dev/CONTRIBUTING.md) and [STYLEGUIDE.md](https://github.com/HelixNetwork/testnet-1.0/blob/dev/STYLEGUIDE.md) if you wish to contribute to this repository!
 - Please read and update the [testnet-1.0-specifications](https://github.com/HelixNetwork/helix-specs/blob/master/specs/testnet-1.0.md).
 - You may enable auto-submission of milestones by passing the `-m`-flag and an integer for the delay.
-- Test-Balance:
-```
-Seed#0: df36d3a5c687106be8c8880ce06117a302bd09fe88355cd4102b901ad9f76ec2
-Addr#0: 556a2431d03e57e92b7d4d4d37f98332fce5427d8167e16c0a5cfbe20899d261
-
-Seed#1: 7b6cc72ce82f3e1369b3e62bfc9607853ae607d352de4110a93645d575898bc6
-Addr#1: 196a2095205189ad2aa77c1125fc9e5d9c4888fb307a4a16caca6f6d311036e7
+- You may enable auto-submission of spam by passing: `--spam <spamIntervalInMillis>` and `--pow-disabled`
 
-Seed#2: 462813e2e99aeb25e94fba849af07bf8927e3a81911c16359f87a6cef1a960c2
-Addr#2: 9ac84e8c4df3e51e78f088b2f51408c97333ad982313101cac14ccb03f137e1f
-
-Seed#3: e2ccb4dbaffc70b02f0d1c14bc1214cc2833d157ddcddb8179014bd593861aa1
-Addr#3: b662e011dae0a5a554281efc7d858894d797fe92b292c5a29c8d3b2ce648aae0
-```
 These addresses hold value, you may use the corresponding seeds to issue value-transfers.
 
 ## Installing   
 Make sure you have [**Maven**](https://maven.apache.org/) and [**Java 8**](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) installed on your computer.
 
 ### Download
 ```
-$ git clone https://github.com/HelixNetwork/sbx.git
+$ git clone https://github.com/HelixNetwork/testnet-1.0.git
 ```
 ### Compile
 ```
@@ -47,15 +35,13 @@ This will create a `target` directory in which you will find the executable jar
 ### Launch
 
 ```
-java -jar target/testnet-<VERSION>.jar -p 14700
+java -jar target/helix-<VERSION>.jar -p 14700
 ```
 
 With auto-milestone submission:
 ```
-java -jar target/testnet-<VERSION>.jar -p 14700 -m 30
+java -jar target/helix-<VERSION>.jar -p 14700 -m 30
 ```
-**NOTICE**: The `-m` flag is only temporary and will be removed in future updates. Its current purpose is to ease the process of submitting milestones as a developer. spment purposes
-If you are running a node with the milestone flag, you should consider limiting the api.
 
 ## CLI
 

changelog.md

@@ -3,6 +3,19 @@
         - API constructor only needs specific objects from helix instance
         - Separate http server from api backend to improve readability, maintainability and enable multiple impl.
         - Resteasy undertow integration
+    - Fixed subseed() in Winternitz class
+    - Added Unit Tests:
+        - RocksDB
+        - Tangle
+        - Winternitz
+    - Fixed: Timestamp conversion in API deviates from Node
+    - Stats Publisher: 
+        - MIN_TRANSACTION_AGE_THRESHOLD set to 5 seconds
+        - MAX_TRANSACTION_AGE_THRESHOLD set to 2 minutes
+    - Fix: Timestamp conversion in Node / API
+    - Fix SignedFiles: line seperator should not be part of digest
+    - New Resource files
+    - Added configurable spammer for testing/experiments 
 
 # 0.5.3
     - PoW Integration (Replace divepearler with GreedyMiner)

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>net.helix</groupId>
     <artifactId>helix</artifactId>
-    <version>0.5.3</version>
+    <version>0.5.4</version>
 
     <name>Helix</name>
     <description>Helix-1.0</description>

src/main/java/net/helix/hlx/HLX.java

@@ -6,6 +6,7 @@
 import net.helix.hlx.conf.HelixConfig;
 import net.helix.hlx.service.API;
 import net.helix.hlx.service.milestone.MSS;
+import net.helix.hlx.service.Spammer;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.ParameterException;
@@ -45,7 +46,7 @@ public class HLX {
 
     public static final String MAINNET_NAME = "HLX";
     public static final String TESTNET_NAME = "HLX Testnet";
-    public static final String VERSION = "0.5.3";
+    public static final String VERSION = "0.5.4";
 
     /**
      * The entry point of the helix sandbox.
@@ -67,7 +68,7 @@ public static void main(String[] args) throws Exception {
     private static void configureLogging() {
         HelixIOUtils.saveLogs(); // TODO: Find a solution, that allows to save the logs under the condition of `config.isSaveLogEnabled()`.
         String config = System.getProperty("logback.configurationFile");
-        String level = System.getProperty("logging-level", "debug").toUpperCase();
+        String level = System.getProperty("logging-level", "info").toUpperCase();
         switch (level) {
             case "OFF":
             case "ERROR":
@@ -96,6 +97,7 @@ private static class HLXLauncher {
         public static API api;
         public static HXI hxi;
         public static MSS mss;
+        public static Spammer spammer;
 
         /**
          * Starts hlx. Setup is as follows:
@@ -123,6 +125,7 @@ public static void main(String [] args) throws Exception {
                     helix.tipsViewModel, helix.transactionValidator,
                     helix.latestMilestoneTracker, helix.graph);
             mss = new MSS(config, api);
+            spammer = new Spammer(config, api);
             shutdownHook();
 
             try {
@@ -138,6 +141,9 @@ public static void main(String [] args) throws Exception {
             if(config.getMsDelay() > 0) {
                 mss.startScheduledExecutorService();
             }
+            if(config.getSpamDelay() > 0) {
+                spammer.startScheduledExecutorService();
+            }
         }
 
         /**

src/main/java/net/helix/hlx/SignedFiles.java

@@ -60,7 +60,7 @@ private static byte[] digestFile(String filename, Sponge sha3) throws IOExceptio
             // building snapshot message
             StringBuilder sb = new StringBuilder();
             reader.lines().forEach(line -> {
-                String hex = line + System.lineSeparator(); // can return a null
+                String hex = line; // can return a null
                 if (hex == null) {
                     throw new IllegalArgumentException("BYTES ARE NULL. INPUT= '" + line + "'");
                 }

src/main/java/net/helix/hlx/conf/BaseHelixConfig.java

@@ -112,6 +112,9 @@ public abstract class BaseHelixConfig implements HelixConfig {
     protected int minDelay = Defaults.MS_MIN_DELAY;
     protected String cooAddress = Defaults.COORDINATOR_ADDRESS;
 
+    //Spammer
+    protected int spamDelay = Defaults.SPAM_DELAY;
+
     public BaseHelixConfig() {
         //empty constructor
     }
@@ -804,6 +807,14 @@ public String getSaveLogXMLFile() {
     @Parameter(names = {"--savelog-xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE)
     protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; }
 
+    @Override
+    public int getSpamDelay() {
+        return spamDelay;
+    }
+    @JsonProperty
+    @Parameter(names = {"--spam"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE)
+    protected void setSpamDelay(int spamDelay) { this.spamDelay = spamDelay; }
+
     public interface Defaults {
         //API
         int API_PORT = 14700;
@@ -896,5 +907,8 @@ public interface Defaults {
         boolean SAVELOG_ENABLED = false;
         String SAVELOG_BASE_PATH = "logs/";
         String SAVELOG_XML_FILE = "/logback-save.xml";
+
+        //Spammer
+        int SPAM_DELAY = 0;
     }
 }

src/main/java/net/helix/hlx/conf/HelixConfig.java

@@ -8,7 +8,7 @@
  *  A container for all possible configuration parameters of SBX.
  *  In charge of how we parse the configuration from given inputs.
  */
-public interface HelixConfig extends APIConfig, NodeConfig, HXIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, GraphConfig, LoggingConfig {
+public interface HelixConfig extends APIConfig, NodeConfig, HXIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, GraphConfig, LoggingConfig, SpamConfig {
     File CONFIG_FILE = new File("hlx.ini");
     /**
      * Parses the args to populate the configuration object

src/main/java/net/helix/hlx/conf/SpamConfig.java

@@ -0,0 +1,12 @@
+package net.helix.hlx.conf;
+
+public interface SpamConfig {
+    /**
+     * @return {@value Descriptions#GET_SPAM_DELAY}
+     */
+    int getSpamDelay();
+
+    interface Descriptions {
+        String GET_SPAM_DELAY = "Delay of spam.";
+    }
+}

src/main/java/net/helix/hlx/crypto/Winternitz.java

@@ -1,10 +1,7 @@
 package net.helix.hlx.crypto;
 
-import net.helix.hlx.utils.Serializer;
-import org.bouncycastle.util.encoders.Hex;
-
-import java.math.BigInteger;
 import java.util.Arrays;
+import org.bouncycastle.util.encoders.Hex;
 
 
 public class Winternitz {
@@ -21,15 +18,25 @@ public class Winternitz {
      * Generate subseed by adding seed and index.
      * @param mode hashing mode
      * @param seed privately generated random
-     * @param index subseed index
+     * @param index subseed index has to be in [1..(Integer.MAX_VALUE-255)]
      * @return <code> byte[] </code> subseed
      */
     public static byte[] subseed(SpongeFactory.Mode mode, final byte[] seed, int index) {
-        if (index < 0) {
+        if (index < 0 || index > Integer.MAX_VALUE - 255) {
             throw new RuntimeException("Invalid subseed index: " + index);
         }
-        byte[] indexInBytes = Serializer.serialize(index);
-        final byte[] subseedPreimage = new BigInteger(seed).add(new BigInteger(indexInBytes)).toByteArray();
+        if (seed.length % Sha3.HASH_LENGTH != 0){
+            throw new RuntimeException("Invalid seed length: " + seed.length);
+        }
+        final byte[] subseedPreimage = seed.clone();
+        for (int i = subseedPreimage.length - 1; i >= 0; i--) {
+            index += (subseedPreimage[i] & 0xFF);
+            subseedPreimage[i] = (byte)index;
+            index >>= 8;
+            if (index == 0) {
+                break;
+            }
+        }
         final byte[] subseed = new byte[Sha3.HASH_LENGTH];
         final Sponge hash = SpongeFactory.create(mode);
         hash.absorb(subseedPreimage, 0, subseedPreimage.length);

src/main/java/net/helix/hlx/network/Node.java

@@ -472,7 +472,7 @@ public void processReceivedData(TransactionViewModel receivedTransactionViewMode
 
         //if new, then broadcast to all neighbors
         if (stored) {
-            receivedTransactionViewModel.setArrivalTime(System.currentTimeMillis());
+            receivedTransactionViewModel.setArrivalTime(System.currentTimeMillis()/1000L);
             try {
                 transactionValidator.updateStatus(receivedTransactionViewModel);
                 receivedTransactionViewModel.updateSender(neighbor.getAddress().toString());

src/main/java/net/helix/hlx/service/API.java

@@ -7,9 +7,12 @@
 import net.helix.hlx.conf.HelixConfig;
 import net.helix.hlx.controllers.*;
 import net.helix.hlx.crypto.*;
+import net.helix.hlx.model.BundleHash;
 import net.helix.hlx.model.Hash;
 import net.helix.hlx.model.HashFactory;
+import net.helix.hlx.model.TransactionHash;
 import net.helix.hlx.model.persistables.Transaction;
+import net.helix.hlx.model.persistables.Bundle;
 import net.helix.hlx.network.Neighbor;
 import net.helix.hlx.network.Node;
 import net.helix.hlx.network.TransactionRequester;
@@ -1419,12 +1422,12 @@ public void shutDown() {
      * @param message The message to store
      **/
     private synchronized AbstractResponse storeMessageStatement(final String address, final String message) throws Exception {
-        final List<Hash> txToApprove = getTransactionToApproveTips(3, Optional.empty());
-        attachStoreAndBroadcast(address, message, txToApprove);
+        attachStoreAndBroadcast(address, message);
         return AbstractResponse.createEmptyResponse();
     }
 
-    private void attachStoreAndBroadcast(final String address, final String message, final List<Hash> txToApprove) throws Exception {
+    public void attachStoreAndBroadcast(final String address, final String message) throws Exception {
+        final List<Hash> txToApprove = getTransactionToApproveTips(3, Optional.empty());
         attachStoreAndBroadcast(address, message, txToApprove, 0, 1, false);
     }
 
@@ -1439,7 +1442,7 @@ private void attachStoreAndBroadcast(final String address, final String message,
         byte[] currentIndexBytes = new byte[TransactionViewModel.CURRENT_INDEX_SIZE];
 
         final byte[] lastIndexBytes = new byte[TransactionViewModel.LAST_INDEX_SIZE];
-        final String lastIndexHex = isMilestone ? Hex.toHexString(lastIndexBytes) : Integer.toHexString(txCount); // TODO: lastIndex has to be 0 for milestones and based on txCount for other tx.
+        final String lastIndexHex = isMilestone ? Hex.toHexString(lastIndexBytes) : Integer.toHexString(txCount-1); // TODO: lastIndex has to be 0 for milestones and based on txCount for other tx.
 
         final byte[] newMilestoneIndex = Serializer.serialize(index);
         final String tagHex = Hex.toHexString(newMilestoneIndex);  // milestoneTracker index is parsed from tag
@@ -1479,6 +1482,7 @@ private void attachStoreAndBroadcast(final String address, final String message,
         sponge.squeeze(essenceHash, 0, essenceHash.length);
         final String bundleHash = Hex.toHexString(essenceHash);
         transactions = transactions.stream().map(tx -> StringUtils.rightPad(tx + bundleHash + StringUtils.repeat('0', 128) + tagHex, BYTES_SIZE, '0')).collect(Collectors.toList());
+        Collections.reverse(transactions);
         List<String> powResult = attachToTangleStatement(txToApprove.get(0), txToApprove.get(1), minWeightMagnitude, transactions);
         storeTransactionsStatement(powResult);
         broadcastTransactionsStatement(powResult);

src/main/java/net/helix/hlx/service/Spammer.java

@@ -0,0 +1,55 @@
+package net.helix.hlx.service;
+
+import net.helix.hlx.conf.SpamConfig;
+import net.helix.hlx.model.Hash;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class Spammer {
+    private static final Logger log = LoggerFactory.getLogger(Spammer.class);
+    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+    private API api;
+    private SpamConfig config;
+
+    private String address;
+    private String message;
+    private int delay;
+
+    public Spammer(SpamConfig config, API api) {
+        this.api = api;
+        this.config = config;
+        this.message = StringUtils.repeat('0', 1024*2);
+        this.address = Hash.NULL_HASH.hexString();
+        this.delay = config.getSpamDelay();
+    }
+
+    public void startScheduledExecutorService() {
+        log.info("Spammer scheduledExecutorService started.");
+        log.info("Submitting Tx every: " + this.delay + "ms.");
+        this.scheduledExecutorService.scheduleWithFixedDelay(this.getRunnableSendTx(), 10000, this.delay,  TimeUnit.MILLISECONDS);
+    }
+
+    private void sendTx() throws Exception {
+        this.api.attachStoreAndBroadcast(this.address, this.message);
+    }
+
+    private Runnable getRunnableSendTx() {
+        return () -> {
+            try {
+                sendTx();
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        };
+    }
+
+    public void shutdown() {
+        log.info("Shutting down Spammer Thread");
+        scheduledExecutorService.shutdown();
+    }
+}