Merge branch 'master' into sip2

snowblossomcoin · Jun 27, 2018 · 96f9841 · 96f9841
2 parents d0e056d + ba84169
commit 96f9841
Show file tree

Hide file tree

Showing 21 changed files with 327 additions and 29 deletions.
diff --git a/example/deployment/snowblossom/client.sh b/example/deployment/snowblossom/client.sh
@@ -1,2 +1,3 @@
 #!/bin/bash
+
 java -jar SnowBlossomClient_deploy.jar configs/client.conf $1 $2 $3
diff --git a/example/deployment/snowblossom/miner.sh b/example/deployment/snowblossom/miner.sh
@@ -1,2 +1,3 @@
 #!/bin/bash
+
 java -jar SnowBlossomMiner_deploy.jar configs/miner.conf
diff --git a/example/deployment/snowblossom/node.sh b/example/deployment/snowblossom/node.sh
@@ -1,2 +1,3 @@
 #!/bin/bash
+
 java -jar SnowBlossomNode_deploy.jar configs/node.conf
diff --git a/example/systemd/snowblossom-explorer.service b/example/systemd/snowblossom-explorer.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Snowblossom Explorer
+After=network.target
+
+[Service]
+User=snowblossom
+WorkingDirectory=/var/snowblossom
+ExecStart=/var/snowblossom/source/bazel-bin/ShackletonExplorer configs/explorer.conf
+
+[Install]
+WantedBy=multi-user.target
diff --git a/lib/src/Globals.java b/lib/src/Globals.java
@@ -44,6 +44,8 @@ public class Globals
 
   public static final int BLOCK_CHUNK_HEADER_DOWNLOAD_SIZE = 500;
 
+  public static final int ADDRESS_HISTORY_MAX_REPLY = 10000;
+
   public static void addCryptoProvider()
   {
     Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider());

diff --git a/lib/src/db/DB.java b/lib/src/db/DB.java
@@ -28,10 +28,14 @@ public abstract class DB implements DBFace
   protected DBMap block_height_map;
   protected DBMap special_map;
   protected ProtoDBMap<Transaction> tx_map;
+  protected DBMapMutationSet address_history_map;
+
+  private Config config;
 
   public DB(Config config)
   {
     Runtime.getRuntime().addShutdownHook(new DBShutdownThread());
+    this.config = config;
   }
 
 
@@ -49,6 +53,11 @@ public void open()
     utxo_node_map = openMap("u");
     block_height_map = openMap("height");
     special_map = openMap("special");
+
+    if (config.getBoolean("addr_index"))
+    {
+      address_history_map = openMutationMapSet("addr_hist_2");
+    }
   }
 
   @Override
@@ -66,6 +75,9 @@ public void open()
   @Override
   public DBMap getUtxoNodeMap() { return utxo_node_map; }
 
+  @Override
+  public DBMapMutationSet getAddressHistoryMap() { return address_history_map; }
+
   @Override
   public ChainHash getBlockHashAtHeight(int height)
   {

diff --git a/lib/src/db/DBFace.java b/lib/src/db/DBFace.java
@@ -10,6 +10,7 @@ public interface DBFace
   public ProtoDBMap<Block> getBlockMap();
   public ProtoDBMap<BlockSummary> getBlockSummaryMap();
   public ProtoDBMap<Transaction> getTransactionMap();
+  public DBMapMutationSet getAddressHistoryMap();
 
   public DBMap getUtxoNodeMap();
 

diff --git a/lib/src/db/DBMapMutationSet.java b/lib/src/db/DBMapMutationSet.java
@@ -6,23 +6,32 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.Set;
+import java.util.List;
+import com.google.common.collect.TreeMultimap;
+import snowblossom.lib.trie.ByteStringComparator;
 
 public abstract class DBMapMutationSet
 {
   public abstract void add(ByteString key, ByteString value);
 
+  public static TreeMultimap<ByteString, ByteString> createMap()
+  {
+    return TreeMultimap.create(new ByteStringComparator(), new ByteStringComparator());
+  }
+
   /** Override this if the DB can do something better */
-  public void addAll(Collection<Map.Entry<ByteString, ByteString> > lst)
+  public void addAll(TreeMultimap<ByteString, ByteString> map)
   {
     long t1 = System.nanoTime();
-    for(Map.Entry<ByteString, ByteString> me : lst)
+
+    for(Map.Entry<ByteString, ByteString> me : map.entries())
     {
       add(me.getKey(), me.getValue());
     }
     TimeRecord.record(t1, "db_putmutset_seq");
   }
 
-  public abstract Set<ByteString> getSet(ByteString key, int max_reply);
+  public abstract List<ByteString> getSet(ByteString key, int max_reply);
 
   public abstract void remove(ByteString key, ByteString value);
 

diff --git a/lib/src/db/rocksdb/RocksDBMapMutationSet.java b/lib/src/db/rocksdb/RocksDBMapMutationSet.java
@@ -1,14 +1,19 @@
 package snowblossom.lib.db.rocksdb;
 
 import com.google.protobuf.ByteString;
+import com.google.common.collect.TreeMultimap;
 import snowblossom.lib.db.DBMapMutationSet;
 import snowblossom.lib.db.DBTooManyResultsException;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksDBException;
+import org.rocksdb.WriteBatch;
 import org.rocksdb.RocksIterator;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.LinkedList;
+import java.util.List;
 
 public class RocksDBMapMutationSet extends DBMapMutationSet
 {
@@ -78,6 +83,29 @@ public void add(ByteString key, ByteString value)
 
   }
 
+  @Override
+  public void addAll(TreeMultimap<ByteString, ByteString> map)
+  {
+    try
+    {
+      WriteBatch batch = new WriteBatch();
+      byte b[]=new byte[0];
+
+      for(Map.Entry<ByteString, ByteString> me : map.entries())
+      {
+        ByteString w = getDBKey(me.getKey(), me.getValue());
+        batch.put(w.toByteArray(), b);
+      }
+
+      db.write(jdb.getWriteOption(), batch);
+    }
+    catch(RocksDBException e)
+    {
+      throw new RuntimeException(e);
+    }
+
+  }
+
   @Override
   public void remove(ByteString key, ByteString value)
   {
@@ -93,11 +121,11 @@ public void remove(ByteString key, ByteString value)
   }
 
   @Override
-  public Set<ByteString> getSet(ByteString key, int max_reply)
+  public List<ByteString> getSet(ByteString key, int max_reply)
   {
 		ByteString dbKey = getDBKey(key);
 
-    HashSet<ByteString> set = new HashSet<>();
+    LinkedList<ByteString> set = new LinkedList<>();
     int count = 0;
     RocksIterator it = db.newIterator();
 

diff --git a/miner/src/MrPlow.java b/miner/src/MrPlow.java
@@ -305,19 +305,6 @@ public void printStats()
 
     last_stats_time = now;
 
-    if (count == 0)
-    {
-      logger.info("we seem to be stalled, reconnecting to node");
-      try
-      {
-        subscribe();
-      }
-      catch (Throwable t)
-      {
-        logger.info("Exception in subscribe: " + t);
-      }
-    }
-
     if (config.getBoolean("display_timerecord"))
     {
 

diff --git a/miner/src/ShareManager.java b/miner/src/ShareManager.java
@@ -68,9 +68,7 @@ public synchronized void prune(long keep)
       {
         share_map.put(address, map_shares);
       }
-
       total_shares -= shares;
-
     }
   }
   public synchronized PPLNSState getState()

diff --git a/node/src/AddressHistoryUtil.java b/node/src/AddressHistoryUtil.java
@@ -0,0 +1,90 @@
+package snowblossom.node;
+import snowblossom.lib.db.DB;
+import snowblossom.proto.*;
+import com.google.protobuf.ByteString;
+import com.google.common.collect.TreeMultimap;
+import snowblossom.lib.db.DBMapMutationSet;
+import snowblossom.lib.*;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public class AddressHistoryUtil
+{
+  public static void saveAddressHistory(Block blk, DB db)
+  {
+    TreeMultimap<ByteString, ByteString> map = DBMapMutationSet.createMap();
+    int height = blk.getHeader().getBlockHeight();
+
+    ChainHash blk_id = new ChainHash(blk.getHeader().getSnowHash());
+
+    for(Transaction tx : blk.getTransactionsList())
+    {
+      ChainHash tx_id = new ChainHash(tx.getTxHash());
+      ByteString val = getValue(blk_id, tx_id, height);
+      TransactionInner inner = TransactionUtil.getInner(tx);
+
+      for(TransactionInput in : inner.getInputsList())
+      {
+        ByteString addr = in.getSpecHash();
+        map.put(addr, val);
+      }
+      for(TransactionOutput out : inner.getOutputsList())
+      {
+        ByteString addr = out.getRecipientSpecHash();
+        map.put(addr, val);
+      }
+
+    }
+    db.getAddressHistoryMap().addAll(map);
+  }
+
+  public static ByteString getValue(ChainHash blk_id, ChainHash tx_id, int height)
+  {
+    byte[] buff = new byte[4 + Globals.BLOCKCHAIN_HASH_LEN * 2];
+
+    ByteBuffer bb = ByteBuffer.wrap(buff);
+    bb.putInt(height);
+    bb.put(blk_id.toByteArray());
+    bb.put(tx_id.toByteArray());
+
+    return ByteString.copyFrom(buff);
+
+  }
+
+
+  public static HistoryList getHistory(AddressSpecHash spec_hash, DB db, BlockHeightCache cache)
+  {
+    List<ByteString> value_set = db.getAddressHistoryMap().getSet(spec_hash.getBytes(), Globals.ADDRESS_HISTORY_MAX_REPLY);
+
+    HistoryList.Builder hist_list = HistoryList.newBuilder();
+
+    for(ByteString val : value_set)
+    {
+      ByteBuffer bb = ByteBuffer.wrap(val.toByteArray());
+      int height = bb.getInt();
+
+      byte[] b = new byte[Globals.BLOCKCHAIN_HASH_LEN];
+
+      bb.get(b);
+      ChainHash blk_hash = new ChainHash(b);
+
+      bb.get(b);
+      ChainHash tx_hash = new ChainHash(b);
+
+      if (blk_hash.equals(cache.getHash(height)))
+      {
+        hist_list.addEntries(HistoryEntry
+          .newBuilder()
+          .setBlockHeight(height)
+          .setTxHash(tx_hash.getBytes())
+          .setBlockHash(blk_hash.getBytes())
+          .build());
+      }
+    }
+
+    return hist_list.build();
+
+  }
+
+
+}
diff --git a/node/src/BlockHeightCache.java b/node/src/BlockHeightCache.java
@@ -0,0 +1,33 @@
+package snowblossom.node;
+
+
+import java.util.TreeMap;
+import snowblossom.lib.ChainHash;
+
+public class BlockHeightCache
+{
+  private TreeMap<Integer, ChainHash> map = new TreeMap<>();
+  private SnowBlossomNode node;
+
+  public BlockHeightCache(SnowBlossomNode node)
+  {
+    this.node = node;
+  }
+
+  public synchronized void setHash(int height, ChainHash hash)
+  {
+    map.put(height, hash);
+  }
+
+  public synchronized ChainHash getHash(int height)
+  {
+    ChainHash hash = map.get(height);
+    if (hash == null)
+    {
+      hash = node.getDB().getBlockHashAtHeight(height);
+      map.put(height, hash);
+    }
+    return hash;
+  }
+
+}
diff --git a/node/src/BlockIngestor.java b/node/src/BlockIngestor.java
@@ -39,6 +39,7 @@ public class BlockIngestor
   private TimeRecord time_record;
 
   private boolean tx_index=false;
+  private boolean addr_index=false;
 
   public BlockIngestor(SnowBlossomNode node)
     throws Exception
@@ -63,6 +64,7 @@ public BlockIngestor(SnowBlossomNode node)
     }
 
     tx_index = node.getConfig().getBoolean("tx_index");
+    addr_index = node.getConfig().getBoolean("addr_index");
 
   }
 
@@ -121,6 +123,13 @@ public boolean ingestBlock(Block blk)
           db.getTransactionMap().putAll(tx_map);
         }
       }
+      if (addr_index)
+      {
+        try(TimeRecordAuto tra_tx = TimeRecord.openAuto("BlockIngestor.saveAddrHist"))
+        {
+          AddressHistoryUtil.saveAddressHistory(blk, db);
+        }
+      }
 
 
       try(TimeRecordAuto tra_tx = TimeRecord.openAuto("BlockIngestor.blockSave"))
@@ -211,6 +220,7 @@ private void updateHeights(BlockSummary summary)
       if ((found == null) || (!found.equals(hash)))
       {
         db.setBlockHashAtHeight(height, hash);
+        node.getBlockHeightCache().setHash(height, hash);
         if (height == 0) return;
         summary = db.getBlockSummaryMap().get(summary.getHeader().getPrevBlockHash());
       }

diff --git a/node/src/Peerage.java b/node/src/Peerage.java
@@ -127,7 +127,10 @@ private PeerChainTip getTip()
     }
     for(int i=0; i<10; i++)
     {
-      tip.addPeers(shuffled_peer_list.poll());
+      if (shuffled_peer_list.size() > 0)
+      {
+        tip.addPeers(shuffled_peer_list.poll());
+      }
     }
 
     return tip.build();