Faster base58 implementation (#2065)

* Faster Base58 encode implementation * Faster Base58 encode implementation * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation make default * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation Scala.js fixes * Faster Base58 encode implementation refactoring * Faster Base58 encode implementation refactoring
wavesplatform · Mar 21, 2019 · add13f4 · add13f4
1 parent 54268f4
commit add13f4
Show file tree

Hide file tree

Showing 52 changed files with 453 additions and 188 deletions.
diff --git a/benchmark/src/test/scala/com/wavesplatform/common/Base58Benchmark.scala b/benchmark/src/test/scala/com/wavesplatform/common/Base58Benchmark.scala
@@ -0,0 +1,46 @@
+package com.wavesplatform.common
+
+import java.util.concurrent.{ThreadLocalRandom, TimeUnit}
+
+import com.wavesplatform.common.Base58Benchmark.{Base58St, BytesSt}
+import com.wavesplatform.common.utils.{Base58, FastBase58, StdBase58}
+import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra.Blackhole
+
+@OutputTimeUnit(TimeUnit.SECONDS)
+@BenchmarkMode(Array(Mode.Throughput))
+@Threads(4)
+@Fork(1)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+class Base58Benchmark {
+  @Benchmark
+  def base58_fastEncode_test(st: BytesSt, bh: Blackhole): Unit = bh.consume(FastBase58.encode(st.bytes))
+
+  @Benchmark
+  def base58_encode_test(st: BytesSt, bh: Blackhole): Unit = bh.consume(StdBase58.encode(st.bytes))
+
+  @Benchmark
+  def base58_decode_test(st: Base58St, bh: Blackhole): Unit = bh.consume(StdBase58.decode(st.base58))
+
+  @Benchmark
+  def base58_fastDecode_test(st: Base58St, bh: Blackhole): Unit = bh.consume(FastBase58.decode(st.base58))
+}
+
+object Base58Benchmark {
+  def randomBytes(length: Int): Array[Byte] = {
+    val bytes = new Array[Byte](length)
+    ThreadLocalRandom.current().nextBytes(bytes)
+    bytes
+  }
+
+  @State(Scope.Benchmark)
+  class BytesSt {
+    val bytes = randomBytes(10000)
+  }
+
+  @State(Scope.Benchmark)
+  class Base58St extends BytesSt {
+    val base58 = Base58.encode(bytes)
+  }
+}
diff --git a/benchmark/src/test/scala/com/wavesplatform/serialization/protobuf/ProtoBufBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/serialization/protobuf/ProtoBufBenchmark.scala
@@ -37,7 +37,7 @@ class ProtoBufBenchmark {
           transfers,
           1518091313964L,
           200000,
-          Base58.decode("59QuUcqP6p").get,
+          Base58.tryDecodeWithLimit("59QuUcqP6p").get,
           Proofs(Seq(ByteStr.decodeBase58("FXMNu3ecy5zBjn9b69VtpuYRwxjCbxdkZ3xZpLzB8ZeFDvcgTkmEDrD29wtGYRPtyLS3LPYrL2d5UM6TpFBMUGQ").get))
         )
         .right
@@ -64,7 +64,7 @@ class ProtoBufBenchmark {
           transfers,
           1518091313964L,
           200000,
-          Base58.decode("59QuUcqP6p").get,
+          Base58.tryDecodeWithLimit("59QuUcqP6p").get,
           Proofs(Seq(ByteStr.decodeBase58("FXMNu3ecy5zBjn9b69VtpuYRwxjCbxdkZ3xZpLzB8ZeFDvcgTkmEDrD29wtGYRPtyLS3LPYrL2d5UM6TpFBMUGQ").get))
         )
         .right

diff --git a/...hmark/src/test/scala/com/wavesplatform/serialization/protobuf/SmartNoSmartBenchmark.scala b/...hmark/src/test/scala/com/wavesplatform/serialization/protobuf/SmartNoSmartBenchmark.scala
@@ -82,7 +82,7 @@ object SmartNoSmartBenchmark {
       1526992336241L,
       1529584336241L,
       2,
-      Base58.decode("2R6JfmNjEnbXAA6nt8YuCzSf1effDS4Wkz8owpCD9BdCNn864SnambTuwgLRYzzeP5CAsKHEviYKAJ2157vdr5Zq").get
+      Base58.tryDecodeWithLimit("2R6JfmNjEnbXAA6nt8YuCzSf1effDS4Wkz8owpCD9BdCNn864SnambTuwgLRYzzeP5CAsKHEviYKAJ2157vdr5Zq").get
     )
 
     val proofs = Proofs(Seq(ByteStr.decodeBase58("5NxNhjMrrH5EWjSFnVnPbanpThic6fnNL48APVAkwq19y2FpQp4tNSqoAZgboC2ykUfqQs9suwBQj6wERmsWWNqa").get))

diff --git a/benchmark/src/test/scala/com/wavesplatform/state/LevelDBWriterBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/LevelDBWriterBenchmark.scala
@@ -61,7 +61,7 @@ object LevelDBWriterBenchmark {
 
   @State(Scope.Benchmark)
   class TransactionByIdSt extends BaseSt {
-    val allTxs: Vector[ByteStr] = load("transactionById", benchSettings.restTxsFile)(x => ByteStr(Base58.decode(x).get))
+    val allTxs: Vector[ByteStr] = load("transactionById", benchSettings.restTxsFile)(x => ByteStr(Base58.tryDecodeWithLimit(x).get))
   }
 
   @State(Scope.Benchmark)
@@ -71,7 +71,7 @@ object LevelDBWriterBenchmark {
 
   @State(Scope.Benchmark)
   class BlocksByIdSt extends BaseSt {
-    val allBlocks: Vector[ByteStr] = load("blocksById", benchSettings.blocksFile)(x => ByteStr(Base58.decode(x).get))
+    val allBlocks: Vector[ByteStr] = load("blocksById", benchSettings.blocksFile)(x => ByteStr(Base58.tryDecodeWithLimit(x).get))
   }
 
   @State(Scope.Benchmark)

diff --git a/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala
@@ -83,7 +83,7 @@ object WavesEnvironmentBenchmark {
 
   @State(Scope.Benchmark)
   class TransactionByIdSt extends BaseSt {
-    val allTxs: Vector[Array[Byte]] = load("transactionById", benchSettings.restTxsFile)(x => Base58.decode(x).get)
+    val allTxs: Vector[Array[Byte]] = load("transactionById", benchSettings.restTxsFile)(x => Base58.tryDecodeWithLimit(x).get)
   }
 
   @State(Scope.Benchmark)
@@ -96,7 +96,7 @@ object WavesEnvironmentBenchmark {
 
   @State(Scope.Benchmark)
   class AccountBalanceOfAssetSt extends AccountBalanceOfWavesSt {
-    val assets: Vector[Array[Byte]] = load("assets", benchSettings.assetsFile)(x => Base58.decode(x).get)
+    val assets: Vector[Array[Byte]] = load("assets", benchSettings.assetsFile)(x => Base58.tryDecodeWithLimit(x).get)
   }
 
   @State(Scope.Benchmark)

diff --git a/common/shared/src/main/scala/com/wavesplatform/common/state/ByteStr.scala b/common/shared/src/main/scala/com/wavesplatform/common/state/ByteStr.scala
@@ -82,9 +82,9 @@ object ByteStr {
 
   def fill(size: Int)(b: Int): ByteStr = ByteStr(Array.fill(size)(b.toByte))
 
-  def decodeBase58(s: String): Try[ByteStr] = Base58.decode(s).map(ByteStr(_))
+  def decodeBase58(s: String): Try[ByteStr] = Base58.tryDecodeWithLimit(s).map(ByteStr(_))
 
-  def decodeBase64(s: String): Try[ByteStr] = Base64.decode(s).map(ByteStr(_))
+  def decodeBase64(s: String): Try[ByteStr] = Base64.tryDecode(s).map(ByteStr(_))
 
   implicit val byteStrOrdering: Ordering[ByteStr] = (x, y) => compare(x.arr, y.arr)
 

diff --git a/common/shared/src/main/scala/com/wavesplatform/common/utils/Base58.scala b/common/shared/src/main/scala/com/wavesplatform/common/utils/Base58.scala
@@ -1,76 +1,33 @@
 package com.wavesplatform.common.utils
-
-import java.util.Arrays
-
-import scala.util.Try
-
-object Base58 {
-  import java.nio.charset.StandardCharsets.US_ASCII
-
-  private val Alphabet: Array[Byte] = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".getBytes(US_ASCII)
-
-  private val DecodeTable: Array[Byte] = Array(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1,
-    9, 10, 11, 12, 13, 14, 15, 16, -1, 17, 18, 19, 20, 21, -1, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, -1, -1, -1, -1, -1, -1, 33, 34, 35, 36, 37,
-    38, 39, 40, 41, 42, 43, -1, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57)
-
-  private def toBase58(c: Char): Byte = if (c < DecodeTable.length) DecodeTable(c) else -1
-
-  def encode(bytes: Array[Byte]): String = {
-    val input     = Arrays.copyOf(bytes, bytes.length)
-    val zeroCount = input.takeWhile(_ == 0).length
-
-    var in                  = zeroCount
-    var out                 = input.length * 2
-    val output: Array[Byte] = new Array[Byte](out)
-    while (in < input.length) {
-      val mod = convert(input, in, 256, 58)
-      if (input(in) == 0) in += 1
-      out -= 1
-      output(out) = Alphabet(mod)
+import scala.util.control.NonFatal
+
+object Base58 extends BaseXXEncDec {
+  private[this] val useSlowBase58: Boolean = sys.props.get("waves.use-slow-base58").exists(s => s.toLowerCase == "true" || s == "1")
+  override val defaultDecodeLimit: Int     = 192
+
+  override def encode(array: Array[Byte]): String = {
+    if (useSlowBase58) {
+      StdBase58.encode(array)
+    } else {
+      try {
+        FastBase58.encode(array)
+      } catch {
+        case NonFatal(_) =>
+          StdBase58.encode(array)
+      }
     }
-
-    while (out < output.length && output(out) == Alphabet(0)) out += 1
-    for (i <- 0 until zeroCount) {
-      out -= 1
-      output(out) = Alphabet(0)
-    }
-
-    new String(output, out, output.length - out, US_ASCII)
-  }
-
-  def decode(string: String, limit: Int = 192 /* 140*log(256)/log(58) */ ): Try[Array[Byte]] = Try {
-    val input: Array[Byte] = new Array[Byte](string.length)
-    string.length.ensuring(_ <= limit, s"base58Decode input exceeds $limit")
-    for (i <- 0 until string.length)
-      input(i) = toBase58(string(i)).ensuring(_ != -1, s"Wrong char '${string(i)}' in Base58 string '$string'")
-
-    val zeroCount = input.takeWhile(_ == 0).length
-
-    var in     = zeroCount
-    var out    = input.length
-    val output = new Array[Byte](out)
-    while (in < input.length) {
-      val mod = convert(input, in, 58, 256)
-      if (input(in) == 0) in += 1
-      out -= 1
-      output(out) = mod
-    }
-
-    while (out < output.length && output(out) == 0) out += 1
-    Arrays.copyOfRange(output, out - zeroCount, output.length)
   }
 
-  private def convert(number: Array[Byte], offset: Int, from: Int, to: Int): Byte = {
-    var rem = 0
-    var i   = offset
-    while (i < number.length) {
-      val digit = number(i) & 0xff
-      val tmp   = rem * from + digit
-      number(i) = (tmp / to).toByte
-      rem = tmp % to
-      i += 1
+  override def decode(str: String): Array[Byte] = {
+    if (useSlowBase58) {
+      StdBase58.decode(str)
+    } else {
+      try {
+        FastBase58.decode(str)
+      } catch {
+        case NonFatal(_) =>
+          StdBase58.decode(str)
+      }
     }
-    rem.toByte
   }
 }
diff --git a/common/shared/src/main/scala/com/wavesplatform/common/utils/Base64.scala b/common/shared/src/main/scala/com/wavesplatform/common/utils/Base64.scala
@@ -1,12 +1,18 @@
 package com.wavesplatform.common.utils
 
-import scala.util.Try
+object Base64 extends BaseXXEncDec {
+  val Prefix                           = "base64:"
+  override val defaultDecodeLimit: Int = 1024 * 1024 * 1024 // 1 MB
 
-object Base64 {
-  def encode(input: Array[Byte]): String = new String(java.util.Base64.getEncoder.encode(input))
+  override def encode(input: Array[Byte]): String = {
+    val encoder      = java.util.Base64.getEncoder
+    val encodedBytes = encoder.encode(input)
+    new String(encodedBytes)
+  }
 
-  def decode(input: String): Try[Array[Byte]] = Try {
-    val str = if (input.startsWith("base64:")) input.substring(7) else input
-    java.util.Base64.getDecoder.decode(str)
+  override def decode(input: String): Array[Byte] = {
+    val decoder    = java.util.Base64.getDecoder
+    val encodedStr = if (input.startsWith(Prefix)) input.substring(Prefix.length) else input
+    decoder.decode(encodedStr)
   }
 }
diff --git a/common/shared/src/main/scala/com/wavesplatform/common/utils/BaseXXEncDec.scala b/common/shared/src/main/scala/com/wavesplatform/common/utils/BaseXXEncDec.scala
@@ -0,0 +1,20 @@
+package com.wavesplatform.common.utils
+
+import scala.util.Try
+
+trait BaseXXEncDec {
+  def defaultDecodeLimit: Int
+
+  def encode(array: Array[Byte]): String
+  def decode(str: String): Array[Byte]
+
+  def tryDecode(str: String): Try[Array[Byte]] = Try {
+    this.decode(str)
+  }
+
+  def tryDecodeWithLimit(str: String, limit: Int = defaultDecodeLimit): Try[Array[Byte]] =
+    Try {
+      require(str.length <= limit, s"base58Decode input exceeds $limit")
+      this.tryDecode(str)
+    }.flatten
+}
diff --git a/common/shared/src/main/scala/com/wavesplatform/common/utils/FastBase58.scala b/common/shared/src/main/scala/com/wavesplatform/common/utils/FastBase58.scala
@@ -0,0 +1,121 @@
+package com.wavesplatform.common.utils
+import java.nio.charset.StandardCharsets.US_ASCII
+
+import scala.annotation.tailrec
+
+object FastBase58 extends BaseXXEncDec {
+  private[this] val Alphabet: Array[Byte] = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".getBytes(US_ASCII)
+  private[this] val DecodeTable: Array[Byte] = Array(
+    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, 9, 10, 11, 12, 13, 14, 15, 16, -1, 17,
+    18, 19, 20, 21, -1, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, -1, -1, -1, -1, -1, -1, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, -1, 44, 45,
+    46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57
+  )
+
+  override def defaultDecodeLimit: Int = 192
+
+  override def encode(bin: Array[Byte]): String = {
+    if (bin.isEmpty) return ""
+
+    val zeroCount = bin
+      .takeWhile(_ == 0)
+      .length
+
+    val bufferSize = (bin.length - zeroCount) * 138 / 100 + 1
+    val buffer     = new Array[Byte](bufferSize)
+
+    var high = bufferSize - 1
+    for (index <- zeroCount until bin.length) {
+      var endIndex = bufferSize - 1
+      var carry    = ByteOps.toUnsignedInt(bin(index))
+
+      while (endIndex > high || carry != 0) {
+        carry = carry + (256 * ByteOps.toUnsignedInt(buffer(endIndex)))
+        buffer(endIndex) = (carry % 58).toByte
+        carry /= 58
+        endIndex -= 1
+      }
+      high = endIndex
+    }
+
+    val startIndex = buffer
+      .takeWhile(_ == 0)
+      .length
+
+    val base58Output = new Array[Byte](bufferSize - startIndex + zeroCount)
+    for (i <- 0 until zeroCount) base58Output(i) = Alphabet(0)
+
+    val bufferZeroCount = buffer.takeWhile(_ == 0).length
+    for (bufferIndex <- bufferZeroCount until bufferSize)
+      base58Output(zeroCount + bufferIndex - bufferZeroCount) = Alphabet(ByteOps.toUnsignedInt(buffer(bufferIndex)))
+
+    new String(base58Output, US_ASCII)
+  }
+
+  override def decode(str: String): Array[Byte] = {
+    if (str.isEmpty) return Array.emptyByteArray
+
+    val b58Chars = str.toCharArray
+
+    var bytesLeft = b58Chars.length % 4
+    var zeroMask  = 0
+    if (bytesLeft > 0) zeroMask = 0xffffffff << (bytesLeft * 8)
+    else bytesLeft = 4
+
+    val outArrayLength = (b58Chars.length + 3) / 4
+    val outArray       = new Array[Long](outArrayLength)
+    for (b58Char <- b58Chars) {
+      if (b58Char >= DecodeTable.length || DecodeTable(b58Char) == -1) throw new IllegalArgumentException(s"Invalid base58 digit $b58Char")
+      var base58EncMask = ByteOps.toUnsignedLong(DecodeTable(b58Char))
+      for (outIndex <- (outArrayLength - 1) until 0 by -1) {
+        val longValue = outArray(outIndex) * 58 + base58EncMask
+        base58EncMask = (longValue >>> 32) & 0x3fL
+        outArray(outIndex) = longValue & 0xffffffffL
+      }
+
+      if (base58EncMask > 0) throw new IllegalArgumentException("Output number too big (carry to the next int32)")
+      if ((outArray(0) & zeroMask) != 0) throw new IllegalArgumentException("Output number too big (last int32 filled too far)")
+    }
+
+    val outBytes      = new Array[Byte]((b58Chars.length + 3) * 3)
+    var outBytesCount = 0
+    for (outArrayIndex <- 0 until outArrayLength) {
+      var mask = (((bytesLeft - 1) & 0xff) * 8).toByte
+      while (ByteOps.toUnsignedInt(mask) <= 0x18) {
+        outBytes(outBytesCount) = (outArray(outArrayIndex) >>> mask).toByte
+        mask = (mask - 8).toByte
+        outBytesCount += 1
+      }
+      if (outArrayIndex == 0) bytesLeft = 4
+    }
+
+    val outBytesStart: Int = {
+      val zeroCount = b58Chars
+        .takeWhile(_ == '1')
+        .length
+
+      @tailrec
+      def findStart(start: Int = 0): Int = {
+        if (start >= outBytes.length) return 0
+        val element = outBytes(start)
+        if (element != 0) (start - zeroCount) max 0
+        else findStart(start + 1)
+      }
+
+      findStart()
+    }
+
+    java.util.Arrays.copyOfRange(outBytes, outBytesStart, outBytesCount)
+  }
+
+  /**
+    * Scala.js linking errors fix (from java.lang.Byte)
+    */
+  private[this] object ByteOps {
+    @inline
+    def toUnsignedInt(x: Byte): Int = x.toInt & 0xff
+
+    @inline
+    def toUnsignedLong(x: Byte): Long = x.toLong & 0xffL
+  }
+}