Discv5 refactor (#148)

* refactor discv5 constructs and test it with an eth2 client

* happy with code paths

* spotless, refactor tests

* Fix all tests

* Fix compilation issue

* spotless

* WIP

* don't double hash signature input

* Fix expectation of findnode request id being present

* Add test for Ticket

* spotless

* remove the duplicate code identifier of messages

* remove unused class

bytes/src/main/java/org/apache/tuweni/bytes/Bytes.java

@@ -29,6 +29,7 @@
 import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.List;
 import java.util.Random;
 
 import io.netty.buffer.ByteBuf;
@@ -102,6 +103,34 @@ static Bytes wrap(Bytes... values) {
     return ConcatenatedBytes.wrap(values);
   }
 
+  /**
+   * Create a value containing the concatenation of the values provided.
+   *
+   * @param values The values to copy and concatenate.
+   * @return A value containing the result of concatenating the value from {@code values} in their provided order.
+   * @throws IllegalArgumentException if the result overflows an int.
+   */
+  static Bytes concatenate(List<Bytes> values) {
+    if (values.size() == 0) {
+      return EMPTY;
+    }
+
+    int size;
+    try {
+      size = values.stream().mapToInt(Bytes::size).reduce(0, Math::addExact);
+    } catch (ArithmeticException e) {
+      throw new IllegalArgumentException("Combined length of values is too long (> Integer.MAX_VALUE)");
+    }
+
+    MutableBytes result = MutableBytes.create(size);
+    int offset = 0;
+    for (Bytes value : values) {
+      value.copyTo(result, offset);
+      offset += value.size();
+    }
+    return result;
+  }
+
   /**
    * Create a value containing the concatenation of the values provided.
    *

crypto/src/main/java/org/apache/tuweni/crypto/SECP256K1.java

@@ -161,7 +161,7 @@ private static ECPoint decompressKey(BigInteger xBN, boolean yBit) {
    * Given the above two points, a correct usage of this method is inside a for loop from 0 to 3, and if the output is
    * null OR a key that is not the one you expect, you try again with the next recovery id.
    *
-   * @param v Which possible key to recover.
+   * @param v Which possible key to recover - can be null if either key can be attempted.
    * @param r The R component of the signature.
    * @param s The S component of the signature.
    * @param messageHash Hash of the data that was signed.
@@ -378,6 +378,12 @@ public static Bytes32 calculateKeyAgreement(SecretKey privKey, PublicKey theirPu
     return UInt256.valueOf(agreement.calculateAgreement(pubKeyP)).toBytes();
   }
 
+  public static Bytes deriveECDHKeyAgreement(Bytes srcPrivKey, Bytes destPubKey) {
+    ECPoint pudDestPoint = SECP256K1.PublicKey.fromBytes(destPubKey).asEcPoint();
+    ECPoint mult = pudDestPoint.multiply(srcPrivKey.toUnsignedBigInteger());
+    return Bytes.wrap(mult.getEncoded(true));
+  }
+
   /**
    * A SECP256K1 private key.
    */

devp2p/build.gradle

@@ -35,4 +35,5 @@ dependencies {
   testImplementation 'org.junit.jupiter:junit-jupiter-params'
 
   testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
+  testRuntimeOnly 'ch.qos.logback:logback-classic'
 }

devp2p/src/integrationTest/java/org/apache/tuweni/devp2p/v5/NodeDiscoveryServiceTest.java → devp2p/src/integrationTest/java/org/apache/tuweni/devp2p/v5/DiscoveryV5ServiceTest.java

@@ -23,12 +23,12 @@
 
 @Timeout(10)
 @ExtendWith(BouncyCastleExtension.class)
-class NodeDiscoveryServiceTest {
+class DiscoveryV5ServiceTest {
 
   @Test
   void testStartAndStop() throws InterruptedException {
-    NodeDiscoveryService service =
-        DiscoveryService.open(SECP256K1.KeyPair.random(), 10000, new InetSocketAddress("localhost", 10000));
+    DiscoveryV5Service service =
+        DiscoveryService.open(SECP256K1.KeyPair.random(), 0, new InetSocketAddress("localhost", 10000));
     service.startAsync().join();
     service.terminateAsync().join();
   }

devp2p/src/integrationTest/kotlin/org/apache/tuweni/devp2p/v5/ConnectTwoServersTest.kt

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tuweni.devp2p.v5
+
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.runBlocking
+import org.apache.tuweni.bytes.Bytes
+import org.apache.tuweni.concurrent.coroutines.await
+import org.apache.tuweni.crypto.Hash
+import org.apache.tuweni.crypto.SECP256K1
+import org.apache.tuweni.devp2p.EthereumNodeRecord
+import org.apache.tuweni.junit.BouncyCastleExtension
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertNotNull
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import java.util.concurrent.ConcurrentHashMap
+
+internal class SimpleTestENRStorage : ENRStorage {
+
+  val storage: MutableMap<Bytes, EthereumNodeRecord> = ConcurrentHashMap()
+
+  override fun find(nodeId: Bytes): EthereumNodeRecord? = storage[nodeId]
+
+  override fun put(nodeId: Bytes, enr: EthereumNodeRecord) { storage.put(nodeId, enr) }
+}
+
+@ExtendWith(BouncyCastleExtension::class)
+class ConnectTwoServersTest {
+
+  @Test
+  fun testConnectTwoServers() = runBlocking {
+    val storage = SimpleTestENRStorage()
+    val service = DiscoveryService.open(
+      SECP256K1.KeyPair.random(),
+      localPort = 40000,
+      bootstrapENRList = emptyList(),
+      enrStorage = storage
+    )
+    service.start().await()
+
+    val otherStorage = SimpleTestENRStorage()
+    val otherService = DiscoveryService.open(
+      SECP256K1.KeyPair.random(),
+      localPort = 40001,
+      bootstrapENRList = emptyList(),
+      enrStorage = otherStorage
+    )
+    otherService.start().await()
+    otherService.addPeer(service.enr()).await()
+    delay(500)
+    assertEquals(1, storage.storage.size)
+    assertEquals(1, otherStorage.storage.size)
+    assertNotNull(otherStorage.find(Hash.sha2_256(service.enr().toRLP())))
+    assertNotNull(storage.find(Hash.sha2_256(otherService.enr().toRLP())))
+  }
+}

devp2p/src/integrationTest/kotlin/org/apache/tuweni/devp2p/v5/LighthouseTest.kt

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tuweni.devp2p.v5
+
+import kotlinx.coroutines.runBlocking
+import org.apache.tuweni.crypto.SECP256K1
+import org.apache.tuweni.junit.BouncyCastleExtension
+import org.junit.jupiter.api.Disabled
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import java.net.InetSocketAddress
+
+/**
+ * Test a developer can run from their machine to contact a remote server.
+ */
+@ExtendWith(BouncyCastleExtension::class)
+class LighthouseTest {
+
+  @Disabled
+  @Test
+  fun testConnect() = runBlocking {
+    val enrRec =
+    "-Iu4QHtMAII7O9sQHpBQ-eNvZIi_f_M5f-JZWTr_PUHiLgZ3ZRd2CkGFYL_fONOVTRw0GL2dMo4yzQP2eBcu0sM5C0IB" +
+      "gmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQIJk7MTrqCvOqk7mysZ6A3F19HDc6ebOOzqSoxVuJbsrYN0Y3CCIyiDdWRwgiMo"
+
+    val service = DiscoveryService.open(
+      SECP256K1.KeyPair.random(),
+      localPort = 0,
+      bindAddress = InetSocketAddress("0.0.0.0", 10000),
+      bootstrapENRList = listOf(enrRec)
+    )
+    service.start().join()
+    kotlinx.coroutines.delay(50000)
+  }
+}

devp2p/src/integrationTest/resources/logback.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations under the License.
+-->
+<configuration>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- encoders are assigned the type
+             ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="org.apache.tuweni.devp2p.v5" level="TRACE" />
+    <root level="debug">
+        <appender-ref ref="STDOUT" />
+    </root>
+</configuration>

devp2p/src/main/kotlin/org/apache/tuweni/devp2p/DiscoveryService.kt

@@ -644,7 +644,7 @@ internal class CoroutineDiscoveryService(
     pending.complete(VerificationResult(peer, endpoint))
 
     if (packet.enrSeq != null) {
-      if (peer.enr == null || peer.enr!!.seq < packet.enrSeq) {
+      if (peer.enr == null || peer.enr!!.seq() < packet.enrSeq) {
         val now = timeSupplier()
         withTimeoutOrNull(ENR_REQUEST_TIMEOUT_MS) { enrRequest(endpoint, peer).verify(now) }
       }