Version 505. Added verifier performance rating.

src/main/java/co/nyzo/verifier/MeshListener.java

@@ -329,17 +329,23 @@ public static Message response(Message message) {
                     response = new Message(MessageType.BlacklistStatusResponse417,
                             new BlacklistStatusResponse(message));
 
+                } else if (messageType == MessageType.PerformanceScoreStatusRequest418) {
+
+                    response = new Message(MessageType.PerformanceScoreStatusResponse419,
+                            new PerformanceScoreStatusResponse(message));
+
                 } else if (messageType == MessageType.ResetRequest500) {
 
-                    boolean success = ByteUtil.arraysAreEqual(message.getSourceNodeIdentifier(), Block.genesisVerifier);
+                    boolean success = ByteUtil.arraysAreEqual(message.getSourceNodeIdentifier(),
+                            Verifier.getIdentifier());
                     String responseMessage;
                     if (success) {
                         responseMessage = "reset request accepted";
                         UpdateUtil.reset();
                     } else {
                         responseMessage = "source node identifier, " +
                                 PrintUtil.compactPrintByteArray(message.getSourceNodeIdentifier()) + ", is not the " +
-                                "Genesis verifier, " + PrintUtil.compactPrintByteArray(Block.genesisVerifier);
+                                "local verifier, " + PrintUtil.compactPrintByteArray(Verifier.getIdentifier());
                     }
 
                     response = new Message(MessageType.ResetResponse501, new BooleanMessageResponse(success,

src/main/java/co/nyzo/verifier/Message.java

@@ -436,6 +436,8 @@ private static MessageObject processContent(MessageType type, ByteBuffer buffer)
             content = NewVerifierTallyStatusResponse.fromByteBuffer(buffer);
         } else if (type == MessageType.BlacklistStatusResponse417) {
             content = BlacklistStatusResponse.fromByteBuffer(buffer);
+        } else if (type == MessageType.PerformanceScoreStatusResponse419) {
+            content = PerformanceScoreStatusResponse.fromByteBuffer(buffer);
         } else if (type == MessageType.ResetResponse501) {
             content = BooleanMessageResponse.fromByteBuffer(buffer);
         } else if (type == MessageType.Error65534) {

src/main/java/co/nyzo/verifier/MessageType.java

@@ -73,9 +73,11 @@ public enum MessageType {
     NewVerifierTallyStatusResponse415(415),
     BlacklistStatusRequest416(416),
     BlacklistStatusResponse417(417),
+    PerformanceScoreStatusRequest418(418),
+    PerformanceScoreStatusResponse419(419),
 
     // bootstrapping messages
-    ResetRequest500(500),   // resets the blockchain   TODO: key this to the local verifier before release
+    ResetRequest500(500),   // resets the blockchain
     ResetResponse501(501),
 
     // the highest allowable message number is 65535

src/main/java/co/nyzo/verifier/Verifier.java

@@ -535,6 +535,12 @@ private static void verifierMain() {
                         // activity.
                         NodeManager.sendNodeJoinRequests(10);
 
+                        // Update scores with the verifier performance manager.
+                        long scoreUpdateHeight = BlockManager.getFrozenEdgeHeight();
+                        Block scoreUpdateBlock = BlockManager.frozenBlockForHeight(scoreUpdateHeight);
+                        VerifierPerformanceManager.updateScoresForFrozenBlock(scoreUpdateBlock,
+                                BlockVoteManager.votesForHeight(scoreUpdateHeight));
+
                         // Perform blacklist and unfrozen block maintenance.
                         BlacklistManager.performMaintenance();
                         UnfrozenBlockManager.performMaintenance();

src/main/java/co/nyzo/verifier/VerifierPerformanceManager.java

@@ -0,0 +1,145 @@
+package co.nyzo.verifier;
+
+import co.nyzo.verifier.messages.BlockVote;
+import co.nyzo.verifier.util.FileUtil;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
+
+public class VerifierPerformanceManager {
+
+    // All scores start at zero. For consistency with chain scores, a lower score is considered superior to a higher
+    // score. The 3:6 increment-to-decrement ratio means that, with 75% consensus, the cycle will see score reductions
+    // on average for each block frozen.
+    private static final int perBlockIncrement = 3;
+    private static final int perVoteDecrement = -6;
+    private static final int removalThresholdScore = 12343 * 2 * perBlockIncrement;  // two days from 0
+    private static final int minimumScore = -removalThresholdScore;  // up to two additional days for good performance
+
+    private static final Map<ByteBuffer, Integer> verifierScoreMap = new ConcurrentHashMap<>();
+    private static AtomicInteger blocksSinceWritingFile = new AtomicInteger();
+
+    public static final File scoreFile = new File(Verifier.dataRootDirectory, "performance_scores");
+
+    private static final BiFunction<Integer, Integer, Integer> mergeFunction =
+            new BiFunction<Integer, Integer, Integer>() {
+                @Override
+                public Integer apply(Integer integer0, Integer integer1) {
+                    int value0 = integer0 == null ? 0 : integer0;
+                    int value1 = integer1 == null ? 0 : integer1;
+                    return Math.max(minimumScore, value0 + value1);
+                }
+            };
+
+    static {
+        loadPersistedScores();
+    }
+
+    public static void updateScoresForFrozenBlock(Block block, Map<ByteBuffer, BlockVote> votes) {
+
+        // Only proceed if the block is not null. It is rare or maybe impossible for the block to be null, but it is
+        // still a reasonable precaution in an environment such as this.
+        if (block != null) {
+
+            // Add for each in-cycle verifier. Each time a block is frozen, a verifier's score increases, but it then
+            // decreases for each vote received.
+            Set<ByteBuffer> inCycleVerifiers = BlockManager.verifiersInCurrentCycleSet();
+            for (ByteBuffer verifierIdentifier : inCycleVerifiers) {
+                verifierScoreMap.merge(verifierIdentifier, perBlockIncrement, mergeFunction);
+            }
+
+            // Subtract for each vote for hash of the block that was frozen. These are the votes that helped the
+            // blockchain reach consensus.
+            for (ByteBuffer verifierIdentifier : votes.keySet()) {
+                BlockVote vote = votes.get(verifierIdentifier);
+                if (ByteUtil.arraysAreEqual(vote.getHash(), block.getHash())) {
+                    verifierScoreMap.merge(verifierIdentifier, perVoteDecrement, mergeFunction);
+                }
+            }
+
+            // Every 30 blocks, clean up the map and write it to file. We only track scores for in-cycle verifiers.
+            // This increment/check/reset of the count is not strictly thread-safe, but failure would only cause the
+            // file to be written an extra time. Checking that the cycle is complete ensures we don't clear in-cycle
+            // verifiers from the map due to missing cycle information in the block manager.
+            if (blocksSinceWritingFile.incrementAndGet() >= 30 && BlockManager.isCycleComplete()) {
+                blocksSinceWritingFile.set(0);
+
+                // Remove all out-of-cycle verifiers from the map.
+                for (ByteBuffer verifierIdentifier : new HashSet<>(verifierScoreMap.keySet())) {
+                    if (!BlockManager.verifierInCurrentCycle(verifierIdentifier)) {
+                        verifierScoreMap.remove(verifierIdentifier);
+                    }
+                }
+
+                // Write the map to the file.
+                persistScores();
+            }
+        }
+    }
+
+    private static void persistScores() {
+
+        List<String> lines = printScores();
+        Path path = Paths.get(scoreFile.getAbsolutePath());
+        FileUtil.writeFile(path, lines);
+    }
+
+    private static void loadPersistedScores() {
+
+        // This method is called in the class's static block. We load any scores that were previously saved to disk so
+        // that scores do not reset each time the verifier is reloaded.
+        Path path = Paths.get(scoreFile.getAbsolutePath());
+        try {
+            List<String> lines = Files.readAllLines(path);
+            for (String line : lines) {
+                line = line.trim();
+                int indexOfHash = line.indexOf("#");
+                if (indexOfHash >= 0) {
+                    line = line.substring(0, indexOfHash).trim();
+                }
+                String[] split = line.split(",");
+                if (split.length == 2) {
+                    try {
+                        byte[] identifier = ByteUtil.byteArrayFromHexString(split[0].trim(), FieldByteSize.identifier);
+                        int score = Integer.parseInt(split[1].trim());
+                        verifierScoreMap.put(ByteBuffer.wrap(identifier), score);
+                    } catch (Exception ignored) { }
+                }
+            }
+
+            System.out.println("loaded " + verifierScoreMap.size() + " scores from file");
+        } catch (Exception ignored) { }
+    }
+
+    public static List<String> printScores() {
+
+        // Scores are written one per line: verifier, followed by identifier. For ease of reading, they are sorted
+        // high (bad) to low (good) so that the verifiers that are most in danger of penalties are at the top of the
+        // list.
+        List<ByteBuffer> identifiers = new ArrayList<>(verifierScoreMap.keySet());
+        Collections.sort(identifiers, new Comparator<ByteBuffer>() {
+            @Override
+            public int compare(ByteBuffer identifier1, ByteBuffer identifier2) {
+                Integer score1 = verifierScoreMap.getOrDefault(identifier1, 0);
+                Integer score2 = verifierScoreMap.getOrDefault(identifier2, 0);
+                return score2.compareTo(score1);
+            }
+        });
+
+        List<String> lines = new ArrayList<>();
+        for (ByteBuffer identifier : identifiers) {
+            int score = verifierScoreMap.getOrDefault(identifier, 0);
+            lines.add(String.format("%s, %5d  # %s", ByteUtil.arrayAsStringWithDashes(identifier.array()), score,
+                    NicknameManager.get(identifier.array())));
+        }
+
+        return lines;
+    }
+}

src/main/java/co/nyzo/verifier/Version.java

@@ -2,7 +2,7 @@
 
 public class Version {
 
-    private static final int version = 504;
+    private static final int version = 505;
 
     public static int getVersion() {
 

src/main/java/co/nyzo/verifier/messages/debug/PerformanceScoreStatusResponse.java

@@ -0,0 +1,88 @@
+package co.nyzo.verifier.messages.debug;
+
+import co.nyzo.verifier.*;
+import co.nyzo.verifier.messages.MultilineTextResponse;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class PerformanceScoreStatusResponse implements MessageObject, MultilineTextResponse {
+
+    private List<String> lines;
+
+    public PerformanceScoreStatusResponse(Message request) {
+
+        // This is a debug request, so it must be signed by the local verifier.
+        if (ByteUtil.arraysAreEqual(request.getSourceNodeIdentifier(), Verifier.getIdentifier())) {
+
+            this.lines = VerifierPerformanceManager.printScores();
+        } else {
+            this.lines = Arrays.asList("*** Unauthorized ***");
+        }
+    }
+
+    public PerformanceScoreStatusResponse(List<String> lines) {
+
+        this.lines = lines;
+    }
+
+    public List<String> getLines() {
+        return lines;
+    }
+
+    @Override
+    public int getByteSize() {
+
+        int byteSize = FieldByteSize.unnamedShort;  // list length
+        for (String line : lines) {
+            byteSize += FieldByteSize.string(line);
+        }
+
+        return byteSize;
+    }
+
+    @Override
+    public byte[] getBytes() {
+
+        byte[] result = new byte[getByteSize()];
+        ByteBuffer buffer = ByteBuffer.wrap(result);
+
+        buffer.putShort((short) lines.size());
+        for (String line : lines) {
+            byte[] lineBytes = line.getBytes(StandardCharsets.UTF_8);
+            buffer.putShort((short) lineBytes.length);
+            buffer.put(lineBytes);
+        }
+
+        return result;
+    }
+
+    public static PerformanceScoreStatusResponse fromByteBuffer(ByteBuffer buffer) {
+
+        PerformanceScoreStatusResponse result = null;
+
+        try {
+            int numberOfLines = buffer.getShort() & 0xffff;
+            List<String> lines = new ArrayList<>();
+            for (int i = 0; i < numberOfLines; i++) {
+                short lineByteLength = buffer.getShort();
+                byte[] lineBytes = new byte[lineByteLength];
+                buffer.get(lineBytes);
+                lines.add(new String(lineBytes, StandardCharsets.UTF_8));
+            }
+
+            result = new PerformanceScoreStatusResponse(lines);
+
+        } catch (Exception ignored) { }
+
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "[PerformanceScoreStatusResponse(lines=" + lines.size() + ")]";
+    }
+}

src/main/java/co/nyzo/verifier/scripts/ConsensusTallyStatusRequestScript.java

@@ -1,76 +1,11 @@
 package co.nyzo.verifier.scripts;
 
 import co.nyzo.verifier.*;
-import co.nyzo.verifier.messages.debug.ConsensusTallyStatusResponse;
-import co.nyzo.verifier.util.IpUtil;
-import co.nyzo.verifier.util.UpdateUtil;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
 
 public class ConsensusTallyStatusRequestScript {
 
     public static void main(String[] args) {
 
-        // Get the private seed. If none is provided, we will send a request to the loopback address.
-        byte[] privateSeed;
-        if (args.length < 1) {
-            System.out.println("*** no in-cycle verifier seed provided; using local verifier seed ***");
-            privateSeed = null;
-        } else {
-            privateSeed = ByteUtil.byteArrayFromHexString(args[0], FieldByteSize.seed);
-        }
-
-        // Get the corresponding identifier.
-        byte[] inCycleVerifierIdentifier = privateSeed == null ? null : KeyUtil.identifierForSeed(privateSeed);
-
-        // Get the IP addresses of the verifier.
-        List<byte[]> ipAddresses = inCycleVerifierIdentifier == null ?
-                Collections.singletonList(IpUtil.addressFromString("127.0.0.1")) :
-                ScriptUtil.ipAddressesForVerifier(inCycleVerifierIdentifier);
-        if (ipAddresses.isEmpty()) {
-            System.out.println("unable to find IP address of " +
-                    ByteUtil.arrayAsStringWithDashes(inCycleVerifierIdentifier));
-        }
-
-        // Send the request to our verifier instances.
-        AtomicInteger numberOfResponsesNotYetReceived = new AtomicInteger(ipAddresses.size());
-        Message message = new Message(MessageType.ConsensusTallyStatusRequest412, null);
-        if (privateSeed != null) {
-            message.sign(privateSeed);
-        }
-        for (byte[] ipAddress : ipAddresses) {
-            Message.fetch(IpUtil.addressAsString(ipAddress), MeshListener.standardPort, message, new MessageCallback() {
-                @Override
-                public void responseReceived(Message message) {
-
-                    System.out.println("response message: " + message);
-                    if (message != null) {
-                        if (message.getContent() instanceof ConsensusTallyStatusResponse) {
-                            ConsensusTallyStatusResponse response = (ConsensusTallyStatusResponse) message.getContent();
-                            System.out.println("response number of lines: " + response.getLines().size());
-                            for (String line : response.getLines()) {
-                                System.out.println(line);
-                            }
-                        } else {
-                            System.out.println("content is incorrect type: " + message.getContent());
-                        }
-                    }
-
-                    numberOfResponsesNotYetReceived.decrementAndGet();
-                }
-            });
-        }
-
-        // Wait for the responses to return.
-        while (numberOfResponsesNotYetReceived.get() > 0) {
-            try {
-                Thread.sleep(300L);
-            } catch (Exception ignored) { }
-        }
-
-        // Terminate the application.
-        UpdateUtil.terminate();
+        ScriptUtil.fetchMultilineStatus(MessageType.ConsensusTallyStatusRequest412, args);
     }
 }

src/main/java/co/nyzo/verifier/scripts/PerformanceScoreStatusRequestScript.java

@@ -0,0 +1,11 @@
+package co.nyzo.verifier.scripts;
+
+import co.nyzo.verifier.MessageType;
+
+public class PerformanceScoreStatusRequestScript {
+
+    public static void main(String[] args) {
+
+        ScriptUtil.fetchMultilineStatus(MessageType.PerformanceScoreStatusRequest418, args);
+    }
+}