Merge pull request #85 from HorizenOfficial/dev

2.3.0 to main

build.sbt

@@ -25,7 +25,7 @@ lazy val commonSettings = Seq(
     Wart.OptionPartial),
   organization := "io.horizen",
   organizationName := "Zen Blockchain Foundation",
-  version := "2.2.0",
+  version := "2.3.0",
   licenses := Seq("CC0" -> url("https://creativecommons.org/publicdomain/zero/1.0/legalcode")),
   homepage := Some(url("https://github.com/HorizenOfficial/Sparkz")),
   pomExtra :=
@@ -46,18 +46,17 @@ lazy val commonSettings = Seq(
   fork := true // otherwise, "java.net.SocketException: maximum number of DatagramSockets reached"
 )
 
-val circeVersion = "0.14.2"
-val akkaVersion = "2.7.0"
-val akkaHttpVersion = "10.4.0"
+val circeVersion = "0.14.5"
+val akkaVersion = "2.8.5"
+val akkaHttpVersion = "10.5.3"
 
 val networkDependencies = Seq(
   "com.typesafe.akka" %% "akka-actor" % akkaVersion,
   "com.typesafe.akka" %% "akka-http-core" % akkaHttpVersion,
   "com.typesafe.akka" %% "akka-http" % akkaHttpVersion,
   "com.typesafe.akka" %% "akka-parsing" % akkaHttpVersion,
-  "com.typesafe.akka" %% "akka-protobuf" % akkaVersion,
   "com.typesafe.akka" %% "akka-stream" % akkaVersion,
-  "commons-net" % "commons-net" % "3.9.0"
+  "commons-net" % "commons-net" % "3.10.0"
 )
 
 val apiDependencies = Seq(
@@ -68,16 +67,16 @@ val apiDependencies = Seq(
 )
 
 val loggingDependencies = Seq(
-  "ch.qos.logback" % "logback-classic" % "1.3.0-alpha16"
+  "ch.qos.logback" % "logback-classic" % "1.4.14"
 )
 
 
 val testingDependencies = Seq(
   "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test",
   "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % "test",
-  "org.scalactic" %% "scalactic" % "3.2.12" % "test",
-  "org.scalatest" %% "scalatest" % "3.2.12" % "test",
-  "org.scalacheck" %% "scalacheck" % "1.16.0",
+  "org.scalactic" %% "scalactic" % "3.2.15" % "test",
+  "org.scalatest" %% "scalatest" % "3.2.15" % "test",
+  "org.scalacheck" %% "scalacheck" % "1.17.0",
   "org.scalatestplus" %% "scalatestplus-scalacheck" % "3.1.0.0-RC2" % Test,
   "org.mockito" %% "mockito-scala" % "1.17.12" % Test
 )

release-notes.md

@@ -1,3 +1,9 @@
+2.3.0
+---------
+* Updated dependencies
+* [Bugfix] /node/removeFromBlackList now don't require explicitly the port number
+* [Bugfix] Node was not able to get new peers other than the known ones under some configurations
+
 2.2.0
 ---------
 * Improved banning mechanism for deprecated nodes (nodes not updated after an hardfork will start to ban updated nodes because receiving newer blocks without a known parent - this will prevent unnecessary network traffic)

scrypto/build.sbt

@@ -12,7 +12,7 @@ libraryDependencies ++= Seq(
   "org.rudogma" %% "supertagged" % "1.5",
   "com.google.guava" % "guava" % "23.0",
   "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2",
-  "org.bouncycastle" % "bcprov-jdk15on" % "1.70"
+  "org.bouncycastle" % "bcprov-jdk18on" % "1.77"
 
 )
 

src/main/scala/sparkz/core/api/http/PeersApiRoute.scala

@@ -8,7 +8,7 @@ import io.circe.generic.semiauto._
 import io.circe.syntax._
 import io.circe.{Decoder, Encoder, Json}
 import sparkz.core.api.http.PeersApiRoute.PeerApiRequest.AddToBlacklistBodyRequest
-import sparkz.core.api.http.PeersApiRoute.Request.ConnectBodyRequest
+import sparkz.core.api.http.PeersApiRoute.Request.AddressBodyRequest
 import sparkz.core.api.http.PeersApiRoute.{BlacklistedPeers, PeerInfoResponse, PeersStatusResponse}
 import sparkz.core.network.ConnectedPeer
 import sparkz.core.network.NetworkController.ReceivableMessages.{ConnectTo, DisconnectFromNode, GetConnectedPeers, GetPeersStatus}
@@ -20,6 +20,7 @@ import sparkz.core.utils.NetworkTimeProvider
 
 import java.net.{InetAddress, InetSocketAddress}
 import scala.concurrent.ExecutionContext
+import scala.util.{Failure, Success, Try}
 
 case class PeersApiRoute(peerManager: ActorRef,
                          networkController: ActorRef,
@@ -117,7 +118,7 @@ case class PeersApiRoute(peerManager: ActorRef,
 
   def connect: Route = (path("connect") & post & withBasicAuth) {
     _ => {
-      entity(as[ConnectBodyRequest]) { bodyRequest =>
+      entity(as[AddressBodyRequest]) { bodyRequest =>
         val peerAddress = bodyRequest.address
 
         val maybeAddress = addressAndPortRegexp.findFirstMatchIn(peerAddress)
@@ -187,16 +188,14 @@ case class PeersApiRoute(peerManager: ActorRef,
 
   def removeFromBlacklist: Route = (path("blacklist") & delete & withBasicAuth) {
     _ => {
-      entity(as[Json]) { json =>
-        val maybeAddress = json.asString.flatMap(addressAndPortRegexp.findFirstMatchIn)
-
-        maybeAddress match {
-          case None => ApiError(StatusCodes.BadRequest, s"address $maybeAddress is not well formatted")
+      entity(as[AddressBodyRequest]) { bodyRequest =>
+        val peerAddress = bodyRequest.address
+        Try(InetAddress.getByName(peerAddress)) match {
+          case Failure(exception) =>
+            ApiError(StatusCodes.BadRequest, s"address $peerAddress is not well formatted: ${exception.getMessage}")
 
-          case Some(addressAndPort) =>
-            val host = InetAddress.getByName(addressAndPort.group(1))
-            val port = addressAndPort.group(2).toInt
-            peerManager ! RemoveFromBlacklist(new InetSocketAddress(host, port))
+          case Success(address) =>
+            peerManager ! RemoveFromBlacklist(address)
             ApiResponse.OK
         }
       }
@@ -224,7 +223,7 @@ object PeersApiRoute {
   }
 
   object Request {
-    case class ConnectBodyRequest(address: String)
+    case class AddressBodyRequest(address: String)
   }
 
   object PeerApiRequest {
@@ -247,6 +246,6 @@ object PeersApiRoute {
   implicit val encodePeersStatusResponse: Encoder[PeersStatusResponse] = deriveEncoder
 
   @SuppressWarnings(Array("org.wartremover.warts.PublicInference"))
-  implicit val decodeConnectBodyRequest: Decoder[ConnectBodyRequest] = deriveDecoder
+  implicit val decodeConnectBodyRequest: Decoder[AddressBodyRequest] = deriveDecoder
 }
 

src/main/scala/sparkz/core/network/NetworkController.scala

@@ -317,7 +317,7 @@ class NetworkController(settings: NetworkSettings,
 
     val peersAlreadyTriedFewTimeBefore = getPeersWeAlreadyTriedToConnectFewTimeAgo
 
-    val randomPeerF = peerManagerRef ? RandomPeerForConnectionExcluding(peersAddresses ++ peersAlreadyTriedFewTimeBefore)
+    val randomPeerF = peerManagerRef ? RandomPeerForConnectionExcluding(peersAddresses ++ peersAlreadyTriedFewTimeBefore, settings.onlyConnectToKnownPeers)
     randomPeerF.mapTo[Option[PeerInfo]].foreach {
       case Some(peerInfo) =>
         peerInfo.peerSpec.address.foreach(address => {

src/main/scala/sparkz/core/network/peer/InMemoryPeerDatabase.scala

@@ -104,13 +104,10 @@ final class InMemoryPeerDatabase(sparkzSettings: SparkzSettings, sparkzContext:
   }
 
   override def allPeers: Map[InetSocketAddress, PeerDatabaseValue] =
-    if (settings.onlyConnectToKnownPeers)
-      knownPeers
-    else
-      knownPeers ++ bucketManager.getTriedPeers ++ bucketManager.getNewPeers
+    knownPeers ++ bucketManager.getTriedPeers ++ bucketManager.getNewPeers
 
   override def blacklistedPeers: Seq[InetAddress] = blacklist
-    .map { case (address, bannedTill) if checkBanned(address, bannedTill) =>
+    .collect { case (address, bannedTill) if checkBanned(address, bannedTill) =>
       address
     }
     .toSeq
@@ -174,10 +171,7 @@ final class InMemoryPeerDatabase(sparkzSettings: SparkzSettings, sparkzContext:
     }
 
   override def randomPeersSubset: Map[InetSocketAddress, PeerDatabaseValue] =
-    if (settings.onlyConnectToKnownPeers)
-      knownPeers
-    else
-      knownPeers ++ bucketManager.getRandomPeers
+    knownPeers ++ bucketManager.getRandomPeers
 
   override def updatePeer(peerDatabaseValue: PeerDatabaseValue): Unit = {
     if (peerIsNotBlacklistedAndNotKnownPeer(peerDatabaseValue)) {

src/main/scala/sparkz/core/network/peer/PeerManager.scala

@@ -3,6 +3,7 @@ package sparkz.core.network.peer
 import akka.actor.{Actor, ActorRef, ActorSystem, Props}
 import sparkz.core.app.SparkzContext
 import sparkz.core.network._
+import sparkz.core.network.peer.PeerDatabase.PeerConfidence.PeerConfidence
 import sparkz.core.network.peer.PeerDatabase.{PeerConfidence, PeerDatabaseValue}
 import sparkz.core.settings.SparkzSettings
 import sparkz.core.utils.NetworkUtils
@@ -88,7 +89,7 @@ class PeerManager(
       }
 
     case RemoveFromBlacklist(address) =>
-      peerDatabase.removeFromBlacklist(address.getAddress)
+      peerDatabase.removeFromBlacklist(address)
 
     case RemovePeer(address) =>
       peerDatabase.remove(address)
@@ -139,7 +140,7 @@ object PeerManager {
 
     case class AddToBlacklist(remote: InetSocketAddress, penalty: Option[PenaltyType] = None)
 
-    case class RemoveFromBlacklist(remote: InetSocketAddress)
+    case class RemoveFromBlacklist(remote: InetAddress)
 
     // peerListOperations messages
 
@@ -174,9 +175,10 @@ object PeerManager {
                           blacklistedPeers: Seq[InetAddress],
                           sparkzContext: SparkzContext): Seq[PeerInfo] = {
         val recentlySeenNonBlacklisted = peers.values.toSeq
-          .filter { p =>
-            (p.peerInfo.connectionType.isDefined || p.peerInfo.lastHandshake > 0) &&
-              !blacklistedPeers.contains(p.address.getAddress)
+          .filterNot(peer => blacklistedPeers.contains(peer.address.getAddress))
+          .filter { p => p.peerInfo.connectionType.isDefined ||
+            p.peerInfo.lastHandshake > 0 ||
+            p.confidence == PeerConfidence.High
           }
         Random.shuffle(recentlySeenNonBlacklisted).take(howMany).map(_.peerInfo)
       }
@@ -189,34 +191,32 @@ object PeerManager {
                           sparkzContext: SparkzContext): Map[InetSocketAddress, PeerInfo] = peers.map(p => p._1 -> p._2.peerInfo)
     }
 
-    case class RandomPeerForConnectionExcluding(excludedPeers: Seq[Option[InetSocketAddress]]) extends GetPeers[Option[PeerInfo]] {
+    case class RandomPeerForConnectionExcluding(excludedPeers: Seq[Option[InetSocketAddress]], onlyKnownPeers: Boolean = false) extends GetPeers[Option[PeerInfo]] {
       private val secureRandom = new SecureRandom()
 
       override def choose(peers: Map[InetSocketAddress, PeerDatabaseValue],
                           blacklistedPeers: Seq[InetAddress],
                           sparkzContext: SparkzContext): Option[PeerInfo] = {
-        var response: Option[PeerInfo] = None
+        val candidates: Map[PeerConfidence, Seq[PeerDatabaseValue]] = peers.values.toSeq
+          .filterNot(goodCandidateFilter(excludedPeers, blacklistedPeers, _))
+          .groupBy(_.confidence)
 
-        val forgerPeers = peers.filter(_._2.confidence == PeerConfidence.Forger)
-        val forgerCandidates = forgerPeers.values.filterNot(goodCandidateFilter(excludedPeers, blacklistedPeers, _)).toSeq
+        val knownPeersCandidates = candidates.getOrElse(PeerConfidence.High, Seq())
+        val forgerCandidates = candidates.getOrElse(PeerConfidence.Forger, Seq())
 
-        if (forgerCandidates.nonEmpty) {
-          response = Some(forgerCandidates(secureRandom.nextInt(forgerCandidates.size)).peerInfo)
+        if (onlyKnownPeers) {
+          if (knownPeersCandidates.nonEmpty)
+            Some(knownPeersCandidates(secureRandom.nextInt(knownPeersCandidates.size)).peerInfo)
+          else None
         } else {
-          val highConfidencePeers = peers.filter(_._2.confidence == PeerConfidence.High)
-          val highConfidenceCandidates = highConfidencePeers.values.filterNot(goodCandidateFilter(excludedPeers, blacklistedPeers, _)).toSeq
-
-          if (highConfidenceCandidates.nonEmpty) {
-            response = Some(highConfidenceCandidates(secureRandom.nextInt(highConfidenceCandidates.size)).peerInfo)
-          } else {
-            val candidates = peers.values.filterNot(goodCandidateFilter(excludedPeers, blacklistedPeers, _)).toSeq
-
-            if (candidates.nonEmpty)
-              response = Some(candidates(secureRandom.nextInt(candidates.size)).peerInfo)
-          }
+          if (forgerCandidates.nonEmpty)
+            Some(forgerCandidates(secureRandom.nextInt(forgerCandidates.size)).peerInfo)
+          else if (knownPeersCandidates.nonEmpty)
+            Some(knownPeersCandidates(secureRandom.nextInt(knownPeersCandidates.size)).peerInfo)
+          else if (candidates.nonEmpty)
+            Some(candidates.values.flatten.toSeq(secureRandom.nextInt(candidates.size)).peerInfo)
+          else None
         }
-
-        response
       }
     }
 

src/test/scala/sparkz/core/api/http/PeersApiRouteSpec.scala

@@ -188,7 +188,25 @@ class PeersApiRouteSpec extends AnyFlatSpec
       status shouldBe StatusCodes.OK
     }
 
-    Delete(prefix + "/blacklist", body).addCredentials(credentials) ~> routesWithProbes ~> check {
+    val bodyDeleteIpV4WithoutPort = HttpEntity("""{"address": "127.0.0.1"}""")
+      .withContentType(ContentTypes.`application/json`)
+    Delete(prefix + "/blacklist", bodyDeleteIpV4WithoutPort).addCredentials(credentials) ~> routesWithProbes ~> check {
+      peerManagerProbe.expectMsgClass(classOf[RemoveFromBlacklist])
+
+      status shouldBe StatusCodes.OK
+    }
+
+    val bodyDeleteIpV6 = HttpEntity("""{"address": "0:0:0:0:0:0:0:1"}""")
+      .withContentType(ContentTypes.`application/json`)
+    Delete(prefix + "/blacklist", bodyDeleteIpV6).addCredentials(credentials) ~> routesWithProbes ~> check {
+      peerManagerProbe.expectMsgClass(classOf[RemoveFromBlacklist])
+
+      status shouldBe StatusCodes.OK
+    }
+
+    val bodyDeleteHostname = HttpEntity("""{"address": "localhost"}""")
+      .withContentType(ContentTypes.`application/json`)
+    Delete(prefix + "/blacklist", bodyDeleteHostname).addCredentials(credentials) ~> routesWithProbes ~> check {
       peerManagerProbe.expectMsgClass(classOf[RemoveFromBlacklist])
 
       status shouldBe StatusCodes.OK
@@ -216,8 +234,9 @@ class PeersApiRouteSpec extends AnyFlatSpec
       }
     })
 
-
-    Delete(prefix + "/blacklist", badBody).addCredentials(credentials) ~> routesWithApiKey ~> check {
+    val bodyDeleteIpV4WithPort = HttpEntity("""{"address": "127.0.0.1:8080"}""")
+      .withContentType(ContentTypes.`application/json`)
+    Delete(prefix + "/blacklist", bodyDeleteIpV4WithPort).addCredentials(credentials) ~> routesWithApiKey ~> check {
       status shouldBe StatusCodes.BadRequest
     }
   }

src/test/scala/sparkz/core/network/peer/InMemoryPeerDatabaseSpec.scala

@@ -284,43 +284,16 @@ class InMemoryPeerDatabaseSpec extends NetworkTests with ObjectGenerators with B
     }
   }
 
-  it should "only return knownPeers if the flag is set to true" in {
-    val firstAddress = new InetSocketAddress(10)
-    val secondAddress = new InetSocketAddress(11)
-    val thirdAddress = new InetSocketAddress(12)
-    val forthAddress = new InetSocketAddress(13)
-    val fifthAddress = new InetSocketAddress(14)
-    val sixthAddress = new InetSocketAddress(15)
-    val knownPeers = Seq(firstAddress, secondAddress, thirdAddress)
-
-    def withDbHavingKnownPeers(test: InMemoryPeerDatabase => Assertion): Assertion =
-      test(new InMemoryPeerDatabase(
-        settings.copy(network = settings.network.copy(penaltySafeInterval = 1.seconds, knownPeers = knownPeers, onlyConnectToKnownPeers = true)),
-        sparkzContext
-      ))
-
-    withDbHavingKnownPeers { db =>
-      val extraPeerOne = PeerDatabaseValue(forthAddress, getPeerInfo(forthAddress), PeerConfidence.Unknown)
-      val extraPeerTwo = PeerDatabaseValue(fifthAddress, getPeerInfo(fifthAddress), PeerConfidence.Unknown)
-      val extraPeerThree = PeerDatabaseValue(sixthAddress, getPeerInfo(sixthAddress), PeerConfidence.Unknown)
-
-      db.addOrUpdateKnownPeer(extraPeerOne)
-      db.addOrUpdateKnownPeer(extraPeerTwo)
-      db.addOrUpdateKnownPeer(extraPeerThree)
-
-      val allPeers = db.allPeers
-      allPeers.size shouldBe 3
-      allPeers.foreach(p => p._2.confidence shouldBe PeerConfidence.High)
-      allPeers.contains(firstAddress) shouldBe true
-      allPeers.contains(secondAddress) shouldBe true
-      allPeers.contains(thirdAddress) shouldBe true
-
-      val randomPeersSubset = db.randomPeersSubset
-      randomPeersSubset.size shouldBe 3
-      randomPeersSubset.foreach(p => p._2.confidence shouldBe PeerConfidence.High)
-      randomPeersSubset.contains(firstAddress) shouldBe true
-      randomPeersSubset.contains(secondAddress) shouldBe true
-      randomPeersSubset.contains(thirdAddress) shouldBe true
+  it should "check blacklisted peers expiration and return only not expired" in {
+    withDb { db =>
+      db.blacklistedPeers shouldBe empty
+      db.addToBlacklist(peerAddress1, PenaltyType.CustomPenaltyDuration(0))
+      db.addToBlacklist(peerAddress2, PenaltyType.CustomPenaltyDuration(1))
+
+      // first call should filter peer1 as it penalty duration expired
+      db.blacklistedPeers shouldBe Seq(peerAddress2.getAddress)
+      //second call to check behaviour when only one peer is blacklisted and it stays in blacklist
+      db.blacklistedPeers shouldBe Seq(peerAddress2.getAddress)
     }
   }
 }

src/test/scala/sparkz/core/network/peer/PeerManagerSpec.scala

@@ -343,7 +343,7 @@ class PeerManagerSpec extends NetworkTests with BeforeAndAfter {
 
     peerManagerRef ! AddOrUpdatePeer(peerInfo)
     peerManagerRef ! AddToBlacklist(peerAddress)
-    peerManagerRef ! RemoveFromBlacklist(peerAddress)
+    peerManagerRef ! RemoveFromBlacklist(peerAddress.getAddress)
 
     // Check all peers
     peerManagerRef ! GetAllPeers