From 327448c65b3dfc45c3ee623972dc6e230a6130f1 Mon Sep 17 00:00:00 2001 From: ferranbt Date: Mon, 13 May 2019 18:25:05 +0200 Subject: [PATCH] Refactor ethash --- consensus/ethash/algorithm.go | 479 ++++++++--------------------- consensus/ethash/algorithm_test.go | 56 ++++ consensus/ethash/cache.go | 128 ++++++++ consensus/ethash/cache_test.go | 49 +++ consensus/ethash/dataset.go | 421 ------------------------- consensus/ethash/difficulty.go | 240 +++++---------- consensus/ethash/ethash.go | 278 ++++++++--------- consensus/ethash/sealer.go | 193 ------------ tests/blockchain_test.go | 12 +- tests/difficulty_test.go | 8 +- 10 files changed, 577 insertions(+), 1287 deletions(-) create mode 100644 consensus/ethash/algorithm_test.go create mode 100644 consensus/ethash/cache.go create mode 100644 consensus/ethash/cache_test.go delete mode 100644 consensus/ethash/dataset.go delete mode 100644 consensus/ethash/sealer.go diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index 0bbc707..25367ee 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -1,413 +1,176 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package ethash import ( "encoding/binary" - "hash" "math/big" - "reflect" - "runtime" - "sync" - "sync/atomic" - "time" - "unsafe" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) -const ( - datasetInitBytes = 1 << 30 // Bytes in dataset at genesis - datasetGrowthBytes = 1 << 23 // Dataset growth per epoch - cacheInitBytes = 1 << 24 // Bytes in cache at genesis - cacheGrowthBytes = 1 << 17 // Cache growth per epoch - epochLength = 30000 // Blocks per epoch - mixBytes = 128 // Width of mix - hashBytes = 64 // Hash length in bytes - hashWords = 16 // Number of 32 bit ints in a hash - datasetParents = 256 // Number of parents of each dataset element - cacheRounds = 3 // Number of rounds in cache production - loopAccesses = 64 // Number of accesses in hashimoto loop +// REVISION is the spec revision number of Ethash +const REVISION = 23 + +var ( + // bytes in word + wordBytes = 4 + // bytes in dataset at genesis (2 ** 30) + datasetBytesInit = 1 << 30 + // dataset growth per epoch (2 ** 23) + datasetBytesGrowth = 1 << 23 + // bytes in cache at genesis (2 ** 24) + cacheBytesInit = 1 << 24 + // cache growth per epoch (2 ** 17) + cacheBytesGrowth = 1 << 17 + // Size of the DAG relative to the cache + cacheMultiplier = 1024 + // blocks per epoch + epochLength = 30000 + // width of mix + mixBytes = 128 + // hash length in bytes + hashBytes = 64 + // number of parents of each dataset element + datasetParents = 256 + // number of rounds in cache production + cacheRounds = 3 + // number of accesses in hashimoto loop + accesses = 64 ) -// cacheSize returns the size of the ethash verification cache that belongs to a certain -// block number. -func cacheSize(block uint64) uint64 { - epoch := int(block / epochLength) - if epoch < maxEpoch { - return cacheSizes[epoch] - } - return calcCacheSize(epoch) -} - -// calcCacheSize calculates the cache size for epoch. The cache size grows linearly, -// however, we always take the highest prime below the linearly growing threshold in order -// to reduce the risk of accidental regularities leading to cyclic behavior. -func calcCacheSize(epoch int) uint64 { - size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes - for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 - size -= 2 * hashBytes - } - return size -} - -// datasetSize returns the size of the ethash mining dataset that belongs to a certain -// block number. -func datasetSize(block uint64) uint64 { - epoch := int(block / epochLength) - if epoch < maxEpoch { - return datasetSizes[epoch] - } - return calcDatasetSize(epoch) +func fnvOp(v1, v2 uint32) uint32 { + return (v1 * 0x01000193) ^ v2 } -// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly, -// however, we always take the highest prime below the linearly growing threshold in order -// to reduce the risk of accidental regularities leading to cyclic behavior. -func calcDatasetSize(epoch int) uint64 { - size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes - for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 - size -= 2 * mixBytes - } - return size -} - -// hasher is a repetitive hasher allowing the same hash data structures to be -// reused between hash runs instead of requiring new ones to be created. -type hasher func(dest []byte, data []byte) - -// makeHasher creates a repetitive hasher, allowing the same hash data structures to -// be reused between hash runs instead of requiring new ones to be created. The returned -// function is not thread safe! -func makeHasher(h hash.Hash) hasher { - // sha3.state supports Read to get the sum, use it to avoid the overhead of Sum. - // Read alters the state but we reset the hash before every operation. - type readerHash interface { - hash.Hash - Read([]byte) (int, error) - } - rh, ok := h.(readerHash) - if !ok { - panic("can't find Read method on hash") - } - outputLen := rh.Size() - return func(dest []byte, data []byte) { - rh.Reset() - rh.Write(data) - rh.Read(dest[:outputLen]) - } -} - -// seedHash is the seed to use for generating a verification cache and the mining -// dataset. -func seedHash(block uint64) []byte { +func getSeedHashByEpoch(epoch int) []byte { seed := make([]byte, 32) - if block < epochLength { + if epoch == 0 { return seed } - keccak256 := makeHasher(sha3.NewLegacyKeccak256()) - for i := 0; i < int(block/epochLength); i++ { - keccak256(seed, seed) + h := sha3.NewLegacyKeccak256() + for i := 0; i < epoch; i++ { + h.Write(seed) + seed = h.Sum(nil) + h.Reset() } return seed } -// generateCache creates a verification cache of a given size for an input seed. -// The cache production process involves first sequentially filling up 32 MB of -// memory, then performing two passes of Sergio Demian Lerner's RandMemoHash -// algorithm from Strict Memory Hard Hashing Functions (2014). The output is a -// set of 524288 64-byte values. -// This method places the result into dest in machine byte order. -func generateCache(dest []uint32, epoch uint64, seed []byte) { - // Print some debug logs to allow analysis on low end devices - // logger := log.New("epoch", epoch) - - /* - start := time.Now() - defer func() { - elapsed := time.Since(start) - - logFn := logger.Debug - if elapsed > 3*time.Second { - logFn = logger.Info - } - logFn("Generated ethash verification cache", "elapsed", common.PrettyDuration(elapsed)) - }() - */ - - // Convert our destination slice to a byte buffer - header := *(*reflect.SliceHeader)(unsafe.Pointer(&dest)) - header.Len *= 4 - header.Cap *= 4 - cache := *(*[]byte)(unsafe.Pointer(&header)) - - // Calculate the number of theoretical rows (we'll store in one buffer nonetheless) - size := uint64(len(cache)) - rows := int(size) / hashBytes - - // Start a monitoring goroutine to report progress on low end devices - var progress uint32 - - done := make(chan struct{}) - defer close(done) - - go func() { - for { - select { - case <-done: - return - case <-time.After(3 * time.Second): - // logger.Info("Generating ethash verification cache", "percentage", atomic.LoadUint32(&progress)*100/uint32(rows)/4, "elapsed", common.PrettyDuration(time.Since(start))) - } - } - }() - // Create a hasher to reuse between invocations - keccak512 := makeHasher(sha3.NewLegacyKeccak512()) - - // Sequentially produce the initial dataset - keccak512(cache, seed) - for offset := uint64(hashBytes); offset < size; offset += hashBytes { - keccak512(cache[offset:], cache[offset-hashBytes:offset]) - atomic.AddUint32(&progress, 1) +func getCacheSizeByEpoch(epoch int) uint64 { + if epoch < maxEpoch { + return cacheSizes[epoch] } - // Use a low-round version of randmemohash - temp := make([]byte, hashBytes) - - for i := 0; i < cacheRounds; i++ { - for j := 0; j < rows; j++ { - var ( - srcOff = ((j - 1 + rows) % rows) * hashBytes - dstOff = j * hashBytes - xorOff = (binary.LittleEndian.Uint32(cache[dstOff:]) % uint32(rows)) * hashBytes - ) - bitutil.XORBytes(temp, cache[srcOff:srcOff+hashBytes], cache[xorOff:xorOff+hashBytes]) - keccak512(cache[dstOff:], temp) + return calcCacheSizeByEpoch(epoch) +} - atomic.AddUint32(&progress, 1) - } - } - // Swap the byte order on big endian systems and return - if !isLittleEndian() { - swap(cache) +func getDatasetSizeByEpoch(epoch int) uint64 { + if epoch < maxEpoch { + return datasetSizes[epoch] } + return calcDatasetSizeByEpoch(epoch) } -// isLittleEndian returns whether the local system is running in little or big -// endian byte order. -func isLittleEndian() bool { - n := uint32(0x01020304) - return *(*byte)(unsafe.Pointer(&n)) == 0x04 -} +func calcSizeByEpoch(epoch, init, growth, mix int) uint64 { + sz := init + growth*(epoch) + sz -= mix + aux := big.NewInt(0) -// swap changes the byte order of the buffer assuming a uint32 representation. -func swap(buffer []byte) { - for i := 0; i < len(buffer); i += 4 { - binary.BigEndian.PutUint32(buffer[i:], binary.LittleEndian.Uint32(buffer[i:])) +BACK: + aux.SetInt64(int64(sz / mix)) + if !aux.ProbablyPrime(1) { + sz -= 2 * mix + goto BACK } + return uint64(sz) } -// fnv is an algorithm inspired by the FNV hash, which in some cases is used as -// a non-associative substitute for XOR. Note that we multiply the prime with -// the full 32-bit input, in contrast with the FNV-1 spec which multiplies the -// prime with one byte (octet) in turn. -func fnv(a, b uint32) uint32 { - return a*0x01000193 ^ b +func calcCacheSizeByEpoch(epoch int) uint64 { + return calcSizeByEpoch(epoch, cacheBytesInit, cacheBytesGrowth, hashBytes) } -// fnvHash mixes in data into mix using the ethash fnv method. -func fnvHash(mix []uint32, data []uint32) { - for i := 0; i < len(mix); i++ { - mix[i] = mix[i]*0x01000193 ^ data[i] - } +func calcDatasetSizeByEpoch(epoch int) uint64 { + return calcSizeByEpoch(epoch, datasetBytesInit, datasetBytesGrowth, mixBytes) } -// generateDatasetItem combines data from 256 pseudorandomly selected cache nodes, -// and hashes that to compute a single dataset node. -func generateDatasetItem(cache []uint32, index uint32, keccak512 hasher) []byte { - // Calculate the number of theoretical rows (we use one buffer nonetheless) - rows := uint32(len(cache) / hashWords) - - // Initialize the mix - mix := make([]byte, hashBytes) - - binary.LittleEndian.PutUint32(mix, cache[(index%rows)*hashWords]^index) - for i := 1; i < hashWords; i++ { - binary.LittleEndian.PutUint32(mix[i*4:], cache[(index%rows)*hashWords+uint32(i)]) - } - keccak512(mix, mix) - - // Convert the mix to uint32s to avoid constant bit shifting - intMix := make([]uint32, hashWords) - for i := 0; i < len(intMix); i++ { - intMix[i] = binary.LittleEndian.Uint32(mix[i*4:]) - } - // fnv it with a lot of random cache nodes based on index - for i := uint32(0); i < datasetParents; i++ { - parent := fnv(index^i, intMix[i%16]) % rows - fnvHash(intMix, cache[parent*hashWords:]) - } - // Flatten the uint32 mix into a binary one and return - for i, val := range intMix { - binary.LittleEndian.PutUint32(mix[i*4:], val) - } - keccak512(mix, mix) - return mix +func getSeedHash(num int) []byte { + return getSeedHashByEpoch(num / epochLength) } -// generateDataset generates the entire ethash dataset for mining. -// This method places the result into dest in machine byte order. -func generateDataset(dest []uint32, epoch uint64, cache []uint32) { - // Print some debug logs to allow analysis on low end devices - // logger := log.New("epoch", epoch) - - /* - start := time.Now() - defer func() { - elapsed := time.Since(start) - - logFn := logger.Debug - if elapsed > 3*time.Second { - logFn = logger.Info - } - logFn("Generated ethash verification cache", "elapsed", common.PrettyDuration(elapsed)) - - }() - */ - - // Figure out whether the bytes need to be swapped for the machine - swapped := !isLittleEndian() +type lookupFn func(index uint32) []uint32 - // Convert our destination slice to a byte buffer - header := *(*reflect.SliceHeader)(unsafe.Pointer(&dest)) - header.Len *= 4 - header.Cap *= 4 - dataset := *(*[]byte)(unsafe.Pointer(&header)) +type hashFn func(p []byte) []byte - // Generate the dataset on many goroutines since it takes a while - threads := runtime.NumCPU() - size := uint64(len(dataset)) +func hashimoto(header []byte, nonce uint64, fullSize int, sha512, sha256 hashFn, lookup lookupFn) ([]byte, []byte) { + w := mixBytes / wordBytes + n := uint32(fullSize / mixBytes) - var pend sync.WaitGroup - pend.Add(threads) + // combine header+nonce into a 64 byte seed + s := make([]byte, 40) + copy(s, header) + binary.LittleEndian.PutUint64(s[32:], nonce) - var progress uint32 - for i := 0; i < threads; i++ { - go func(id int) { - defer pend.Done() + s = sha512(s) + sHead := binary.LittleEndian.Uint32(s) - // Create a hasher to reuse between invocations - keccak512 := makeHasher(sha3.NewLegacyKeccak512()) - - // Calculate the data segment this thread should generate - batch := uint32((size + hashBytes*uint64(threads) - 1) / (hashBytes * uint64(threads))) - first := uint32(id) * batch - limit := first + batch - if limit > uint32(size/hashBytes) { - limit = uint32(size / hashBytes) - } - // Calculate the dataset segment - percent := uint32(size / hashBytes / 100) - for index := first; index < limit; index++ { - item := generateDatasetItem(cache, index, keccak512) - if swapped { - swap(item) - } - copy(dataset[index*hashBytes:], item) - - if status := atomic.AddUint32(&progress, 1); status%percent == 0 { - // logger.Info("Generating DAG in progress", "percentage", uint64(status*100)/(size/hashBytes), "elapsed", common.PrettyDuration(time.Since(start))) - } - } - }(i) + // start the mix with replicated s + mix := make([]uint32, w) + for i := 0; i < len(mix); i++ { + mix[i] = binary.LittleEndian.Uint32(s[i%16*4:]) } - // Wait for all the generators to finish and return - pend.Wait() -} - -// hashimoto aggregates data from the full dataset in order to produce our final -// value for a particular header hash and nonce. -func hashimoto(hash []byte, nonce uint64, size uint64, lookup func(index uint32) []uint32) ([]byte, []byte) { - // Calculate the number of theoretical rows (we use one buffer nonetheless) - rows := uint32(size / mixBytes) - // Combine header+nonce into a 64 byte seed - seed := make([]byte, 40) - copy(seed, hash) - binary.LittleEndian.PutUint64(seed[32:], nonce) + tmp := make([]uint32, w) - seed = crypto.Keccak512(seed) - seedHead := binary.LittleEndian.Uint32(seed) + // mix in random dataset nodes + for i := 0; i < accesses; i++ { + p := fnvOp(uint32(i)^sHead, mix[i%w]) % n - // Start the mix with replicated seed - mix := make([]uint32, mixBytes/4) - for i := 0; i < len(mix); i++ { - mix[i] = binary.LittleEndian.Uint32(seed[i%16*4:]) - } - // Mix in random dataset nodes - temp := make([]uint32, len(mix)) + // NOTE: mixBytes/hashBytes = 2. Would it be faster to unroll this loop? + for j := 0; j < mixBytes/hashBytes; j++ { + copy(tmp[j*16:], lookup(2*p+uint32(j))) + } - for i := 0; i < loopAccesses; i++ { - parent := fnv(uint32(i)^seedHead, mix[i%len(mix)]) % rows - for j := uint32(0); j < mixBytes/hashBytes; j++ { - copy(temp[j*hashWords:], lookup(2*parent+j)) + // fnv map + for o := 0; o < 32; o++ { + mix[o] = fnvOp(mix[o], tmp[o]) } - fnvHash(mix, temp) } - // Compress mix + + // compress mix + cmix := []uint32{} for i := 0; i < len(mix); i += 4 { - mix[i/4] = fnv(fnv(fnv(mix[i], mix[i+1]), mix[i+2]), mix[i+3]) + cmix = append(cmix, fnvOp(fnvOp(fnvOp(mix[i], mix[i+1]), mix[i+2]), mix[i+3])) } - mix = mix[:len(mix)/4] - digest := make([]byte, common.HashLength) - for i, val := range mix { + digest := make([]byte, 32) + for i, val := range cmix { binary.LittleEndian.PutUint32(digest[i*4:], val) } - return digest, crypto.Keccak256(append(seed, digest...)) -} - -// hashimotoLight aggregates data from the full dataset (using only a small -// in-memory cache) in order to produce our final value for a particular header -// hash and nonce. -func hashimotoLight(size uint64, cache []uint32, hash []byte, nonce uint64) ([]byte, []byte) { - keccak512 := makeHasher(sha3.NewLegacyKeccak512()) - - lookup := func(index uint32) []uint32 { - rawData := generateDatasetItem(cache, index, keccak512) - - data := make([]uint32, len(rawData)/4) - for i := 0; i < len(data); i++ { - data[i] = binary.LittleEndian.Uint32(rawData[i*4:]) - } - return data - } - return hashimoto(hash, nonce, size, lookup) -} -// hashimotoFull aggregates data from the full dataset (using the full in-memory -// dataset) in order to produce our final value for a particular header hash and -// nonce. -func hashimotoFull(dataset []uint32, hash []byte, nonce uint64) ([]byte, []byte) { - lookup := func(index uint32) []uint32 { - offset := index * hashWords - return dataset[offset : offset+hashWords] - } - return hashimoto(hash, nonce, uint64(len(dataset))*4, lookup) + result := sha256(append(s, digest...)) + return digest, result +} + +// TODO +func sealHash(header *types.Header) (hash common.Hash) { + hasher := sha3.NewLegacyKeccak256() + + rlp.Encode(hasher, []interface{}{ + header.ParentHash, + header.UncleHash, + header.Coinbase, + header.Root, + header.TxHash, + header.ReceiptHash, + header.Bloom, + header.Difficulty, + header.Number, + header.GasLimit, + header.GasUsed, + header.Time, + header.Extra, + }) + hasher.Sum(hash[:0]) + return hash } diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go new file mode 100644 index 0000000..21d8243 --- /dev/null +++ b/consensus/ethash/algorithm_test.go @@ -0,0 +1,56 @@ +package ethash + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/stretchr/testify/assert" +) + +func TestSizes(t *testing.T) { + t.Run("cache", func(t *testing.T) { + for epoch, val := range cacheSizes { + assert.Equal(t, getCacheSizeByEpoch(epoch), val) + } + }) + t.Run("dataset", func(t *testing.T) { + for epoch, val := range datasetSizes { + assert.Equal(t, getDatasetSizeByEpoch(epoch), val) + } + }) +} + +func TestBlockSeed(t *testing.T) { + cases := []struct { + block int + seed string + }{ + { + block: 0, + seed: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + { + block: 30000, + seed: "0x290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563", + }, + { + block: 60000, + seed: "0x510e4e770828ddbf7f7b00ab00a9f6adaf81c0dc9cc85f1f8249c256942d61d9", + }, + { + block: 80000, + seed: "0x510e4e770828ddbf7f7b00ab00a9f6adaf81c0dc9cc85f1f8249c256942d61d9", + }, + { + block: 90000, + seed: "0x356e5a2cc1eba076e650ac7473fccc37952b46bc2e419a200cec0c451dce2336", + }, + } + + for _, c := range cases { + t.Run("", func(t *testing.T) { + assert.Equal(t, c.seed, hexutil.Encode(getSeedHash(c.block))) + assert.Equal(t, c.seed, hexutil.Encode(getSeedHashByEpoch(c.block/epochLength))) + }) + } +} diff --git a/consensus/ethash/cache.go b/consensus/ethash/cache.go new file mode 100644 index 0000000..4324a82 --- /dev/null +++ b/consensus/ethash/cache.go @@ -0,0 +1,128 @@ +package ethash + +import ( + "encoding/binary" + "fmt" + "hash" + + "golang.org/x/crypto/sha3" +) + +// Cache is a 16 MB pseudorandom cache. +type Cache struct { + cacheSize uint32 + datasetSize int + cache [][]uint32 + sha512 hash.Hash + sha256 hash.Hash +} + +func newCache(epoch int) *Cache { + cacheSize := getCacheSizeByEpoch(epoch) + datasetSize := getDatasetSizeByEpoch(epoch) + seed := getSeedHashByEpoch(epoch) + + c := &Cache{ + sha512: sha3.NewLegacyKeccak512(), + sha256: sha3.NewLegacyKeccak256(), + datasetSize: int(datasetSize), + } + + c.mkcache(int(cacheSize), seed) + c.cacheSize = uint32(len(c.cache)) + return c +} + +func (c *Cache) calcDatasetItem(i uint32) []uint32 { + n := c.cacheSize + r := hashBytes / wordBytes + + mix := make([]uint32, len(c.cache[0])) + copy(mix[:], c.cache[i%n]) + mix[0] ^= i + c.sha512Int(mix) + + for j := 0; j < datasetParents; j++ { + cacheIndex := fnvOp(i^uint32(j), mix[j%r]) + + // fnv map + for o := 0; o < 16; o++ { + mix[o] = fnvOp(mix[o], c.cache[cacheIndex%n][o]) + } + } + + c.sha512Int(mix) + return mix +} + +func (c *Cache) sha512Aux(p []byte) []byte { + c.sha512.Reset() + c.sha512.Write(p) + return c.sha512.Sum(nil) +} + +func (c *Cache) sha256Aux(p []byte) []byte { + c.sha256.Reset() + c.sha256.Write(p) + return c.sha256.Sum(nil) +} + +func (c *Cache) sha512Int(p []uint32) { + aux := make([]byte, 4) + + c.sha512.Reset() + for _, i := range p { + binary.LittleEndian.PutUint32(aux, i) + c.sha512.Write(aux) + } + res := c.sha512.Sum(nil) + for i := 0; i < len(p); i++ { + p[i] = binary.LittleEndian.Uint32(res[i*4:]) + } +} + +func (c *Cache) mkcache(cacheSize int, seed []byte) { + n := cacheSize / hashBytes + + res := [][]byte{} + res = append(res, c.sha512Aux(seed)) + for i := 1; i < n; i++ { + aux := c.sha512Aux(res[i-1]) + res = append(res, aux) + } + + for j := 0; j < cacheRounds; j++ { + for i := 0; i < n; i++ { + v := binary.LittleEndian.Uint32(res[i]) % uint32(n) + temp := xorBytes(res[(i-1+n)%n], res[v]) + res[i] = c.sha512Aux(temp) + } + } + + // Convert bytes to words + resInt := [][]uint32{} + for _, i := range res { + entry := make([]uint32, 16) + for indx := range entry { + entry[indx] = binary.LittleEndian.Uint32(i[indx*4:]) + } + resInt = append(resInt, entry) + } + + c.cache = resInt +} + +func (c *Cache) hashimoto(header []byte, nonce uint64) ([]byte, []byte) { + return hashimoto(header, nonce, c.datasetSize, c.sha512Aux, c.sha256Aux, c.calcDatasetItem) +} + +func xorBytes(a, b []byte) []byte { + if len(a) != len(b) { + panic(fmt.Sprintf("length of byte slices is not equivalent: %d != %d", len(a), len(b))) + } + buf := make([]byte, len(a)) + for i := range a { + buf[i] = a[i] ^ b[i] + } + return buf +} diff --git a/consensus/ethash/cache_test.go b/consensus/ethash/cache_test.go new file mode 100644 index 0000000..f546e8a --- /dev/null +++ b/consensus/ethash/cache_test.go @@ -0,0 +1,49 @@ +package ethash + +import ( + "encoding/binary" + "hash/fnv" + "testing" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/stretchr/testify/assert" +) + +func TestCache(t *testing.T) { + cases := []struct { + epoch int + hash string + }{ + { + epoch: 0, + hash: "0xbdae398aa93d3e0593f55180d4d9e14a", + }, + { + epoch: 1, + hash: "0xfd9ef335c5dc2f3831abe21fa248747b", + }, + { + epoch: 100, + hash: "0x19bea946d49edfcfc01e34a58495921b", + }, + { + epoch: 1000, + hash: "0xe42941588426211fb7d56eaba630a687", + }, + } + + for _, c := range cases { + t.Run("", func(t *testing.T) { + cache := newCache(c.epoch) + hash := fnv.New128() + b := make([]byte, 4) + for _, i := range cache.cache { + for _, j := range i { + binary.BigEndian.PutUint32(b, j) + hash.Write(b) + } + } + assert.Equal(t, c.hash, hexutil.Encode(hash.Sum(nil))) + }) + } +} diff --git a/consensus/ethash/dataset.go b/consensus/ethash/dataset.go deleted file mode 100644 index d11c8a0..0000000 --- a/consensus/ethash/dataset.go +++ /dev/null @@ -1,421 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package ethash - -import ( - "errors" - "fmt" - "math/big" - "math/rand" - "os" - "path/filepath" - "reflect" - "runtime" - "strconv" - "sync" - "sync/atomic" - "unsafe" - - mmap "github.com/edsrzf/mmap-go" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/log" - "github.com/hashicorp/golang-lru/simplelru" -) - -var ErrInvalidDumpMagic = errors.New("invalid dump magic") - -var ( - // two256 is a big integer representing 2^256 - two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) - - // algorithmRevision is the data structure version used for file naming. - algorithmRevision = 23 - - // dumpMagic is a dataset dump header to sanity check a data dump. - dumpMagic = []uint32{0xbaddcafe, 0xfee1dead} -) - -// memoryMap tries to memory map a file of uint32s for read only access. -func memoryMap(path string) (*os.File, mmap.MMap, []uint32, error) { - file, err := os.OpenFile(path, os.O_RDONLY, 0644) - if err != nil { - return nil, nil, nil, err - } - mem, buffer, err := memoryMapFile(file, false) - if err != nil { - file.Close() - return nil, nil, nil, err - } - for i, magic := range dumpMagic { - if buffer[i] != magic { - mem.Unmap() - file.Close() - return nil, nil, nil, ErrInvalidDumpMagic - } - } - return file, mem, buffer[len(dumpMagic):], err -} - -// memoryMapFile tries to memory map an already opened file descriptor. -func memoryMapFile(file *os.File, write bool) (mmap.MMap, []uint32, error) { - // Try to memory map the file - flag := mmap.RDONLY - if write { - flag = mmap.RDWR - } - mem, err := mmap.Map(file, flag, 0) - if err != nil { - return nil, nil, err - } - // Yay, we managed to memory map the file, here be dragons - header := *(*reflect.SliceHeader)(unsafe.Pointer(&mem)) - header.Len /= 4 - header.Cap /= 4 - - return mem, *(*[]uint32)(unsafe.Pointer(&header)), nil -} - -// memoryMapAndGenerate tries to memory map a temporary file of uint32s for write -// access, fill it with the data from a generator and then move it into the final -// path requested. -func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint32)) (*os.File, mmap.MMap, []uint32, error) { - // Ensure the data folder exists - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return nil, nil, nil, err - } - // Create a huge temporary empty file to fill with data - temp := path + "." + strconv.Itoa(rand.Int()) - - dump, err := os.Create(temp) - if err != nil { - return nil, nil, nil, err - } - if err = dump.Truncate(int64(len(dumpMagic))*4 + int64(size)); err != nil { - return nil, nil, nil, err - } - // Memory map the file for writing and fill it with the generator - mem, buffer, err := memoryMapFile(dump, true) - if err != nil { - dump.Close() - return nil, nil, nil, err - } - copy(buffer, dumpMagic) - - data := buffer[len(dumpMagic):] - generator(data) - - if err := mem.Unmap(); err != nil { - return nil, nil, nil, err - } - if err := dump.Close(); err != nil { - return nil, nil, nil, err - } - if err := os.Rename(temp, path); err != nil { - return nil, nil, nil, err - } - return memoryMap(path) -} - -// lru tracks caches or datasets by their last use time, keeping at most N of them. -type lru struct { - what string - new func(epoch uint64) interface{} - mu sync.Mutex - // Items are kept in a LRU cache, but there is a special case: - // We always keep an item for (highest seen epoch) + 1 as the 'future item'. - cache *simplelru.LRU - future uint64 - futureItem interface{} -} - -// newlru create a new least-recently-used cache for either the verification caches -// or the mining datasets. -func newlru(what string, maxItems int, new func(epoch uint64) interface{}) *lru { - if maxItems <= 0 { - maxItems = 1 - } - cache, _ := simplelru.NewLRU(maxItems, func(key, value interface{}) { - log.Trace("Evicted ethash "+what, "epoch", key) - }) - return &lru{what: what, new: new, cache: cache} -} - -// get retrieves or creates an item for the given epoch. The first return value is always -// non-nil. The second return value is non-nil if lru thinks that an item will be useful in -// the near future. -func (lru *lru) get(epoch uint64) (item, future interface{}) { - lru.mu.Lock() - defer lru.mu.Unlock() - - // Get or create the item for the requested epoch. - item, ok := lru.cache.Get(epoch) - if !ok { - if lru.future > 0 && lru.future == epoch { - item = lru.futureItem - } else { - log.Trace("Requiring new ethash "+lru.what, "epoch", epoch) - item = lru.new(epoch) - } - lru.cache.Add(epoch, item) - } - // Update the 'future item' if epoch is larger than previously seen. - if epoch < maxEpoch-1 && lru.future < epoch+1 { - log.Trace("Requiring new future ethash "+lru.what, "epoch", epoch+1) - future = lru.new(epoch + 1) - lru.future = epoch + 1 - lru.futureItem = future - } - return item, future -} - -// cache wraps an ethash cache with some metadata to allow easier concurrent use. -type cache struct { - epoch uint64 // Epoch for which this cache is relevant - dump *os.File // File descriptor of the memory mapped cache - mmap mmap.MMap // Memory map itself to unmap before releasing - cache []uint32 // The actual cache data content (may be memory mapped) - once sync.Once // Ensures the cache is generated only once -} - -// newCache creates a new ethash verification cache and returns it as a plain Go -// interface to be usable in an LRU cache. -func newCache(epoch uint64) interface{} { - return &cache{epoch: epoch} -} - -// generate ensures that the cache content is generated before use. -func (c *cache) generate(dir string, limit int, test bool) { - c.once.Do(func() { - size := cacheSize(c.epoch*epochLength + 1) - seed := seedHash(c.epoch*epochLength + 1) - if test { - size = 1024 - } - // If we don't store anything on disk, generate and return. - if dir == "" { - c.cache = make([]uint32, size/4) - generateCache(c.cache, c.epoch, seed) - return - } - // Disk storage is needed, this will get fancy - var endian string - if !isLittleEndian() { - endian = ".be" - } - path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) - logger := log.New("epoch", c.epoch) - - // We're about to mmap the file, ensure that the mapping is cleaned up when the - // cache becomes unused. - runtime.SetFinalizer(c, (*cache).finalizer) - - // Try to load the file from disk and memory map it - var err error - c.dump, c.mmap, c.cache, err = memoryMap(path) - if err == nil { - logger.Debug("Loaded old ethash cache from disk") - return - } - logger.Debug("Failed to load old ethash cache", "err", err) - - // No previous cache available, create a new cache file to fill - c.dump, c.mmap, c.cache, err = memoryMapAndGenerate(path, size, func(buffer []uint32) { generateCache(buffer, c.epoch, seed) }) - if err != nil { - logger.Error("Failed to generate mapped ethash cache", "err", err) - - c.cache = make([]uint32, size/4) - generateCache(c.cache, c.epoch, seed) - } - // Iterate over all previous instances and delete old ones - for ep := int(c.epoch) - limit; ep >= 0; ep-- { - seed := seedHash(uint64(ep)*epochLength + 1) - path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) - os.Remove(path) - } - }) -} - -// finalizer unmaps the memory and closes the file. -func (c *cache) finalizer() { - if c.mmap != nil { - c.mmap.Unmap() - c.dump.Close() - c.mmap, c.dump = nil, nil - } -} - -// dataset wraps an ethash dataset with some metadata to allow easier concurrent use. -type dataset struct { - epoch uint64 // Epoch for which this cache is relevant - dump *os.File // File descriptor of the memory mapped cache - mmap mmap.MMap // Memory map itself to unmap before releasing - dataset []uint32 // The actual cache data content - once sync.Once // Ensures the cache is generated only once - done uint32 // Atomic flag to determine generation status -} - -// newDataset creates a new ethash mining dataset and returns it as a plain Go -// interface to be usable in an LRU cache. -func newDataset(epoch uint64) interface{} { - return &dataset{epoch: epoch} -} - -// generate ensures that the dataset content is generated before use. -func (d *dataset) generate(dir string, limit int, test bool) { - fmt.Println("-- generate --") - - d.once.Do(func() { - // Mark the dataset generated after we're done. This is needed for remote - defer atomic.StoreUint32(&d.done, 1) - - csize := cacheSize(d.epoch*epochLength + 1) - dsize := datasetSize(d.epoch*epochLength + 1) - seed := seedHash(d.epoch*epochLength + 1) - if test { - csize = 1024 - dsize = 32 * 1024 - } - // If we don't store anything on disk, generate and return - if dir == "" { - cache := make([]uint32, csize/4) - generateCache(cache, d.epoch, seed) - - d.dataset = make([]uint32, dsize/4) - generateDataset(d.dataset, d.epoch, cache) - - return - } - // Disk storage is needed, this will get fancy - var endian string - if !isLittleEndian() { - endian = ".be" - } - path := filepath.Join(dir, fmt.Sprintf("full-R%d-%x%s", algorithmRevision, seed[:8], endian)) - logger := log.New("epoch", d.epoch) - - fmt.Println("-- path --") - fmt.Println(path) - - // We're about to mmap the file, ensure that the mapping is cleaned up when the - // cache becomes unused. - runtime.SetFinalizer(d, (*dataset).finalizer) - - // Try to load the file from disk and memory map it - var err error - d.dump, d.mmap, d.dataset, err = memoryMap(path) - if err == nil { - logger.Debug("Loaded old ethash dataset from disk") - return - } - logger.Debug("Failed to load old ethash dataset", "err", err) - - // No previous dataset available, create a new dataset file to fill - cache := make([]uint32, csize/4) - generateCache(cache, d.epoch, seed) - - d.dump, d.mmap, d.dataset, err = memoryMapAndGenerate(path, dsize, func(buffer []uint32) { generateDataset(buffer, d.epoch, cache) }) - if err != nil { - logger.Error("Failed to generate mapped ethash dataset", "err", err) - - d.dataset = make([]uint32, dsize/2) - generateDataset(d.dataset, d.epoch, cache) - } - // Iterate over all previous instances and delete old ones - for ep := int(d.epoch) - limit; ep >= 0; ep-- { - seed := seedHash(uint64(ep)*epochLength + 1) - path := filepath.Join(dir, fmt.Sprintf("full-R%d-%x%s", algorithmRevision, seed[:8], endian)) - os.Remove(path) - } - - fmt.Println("- done -") - }) -} - -// generated returns whether this particular dataset finished generating already -// or not (it may not have been started at all). This is useful for remote miners -// to default to verification caches instead of blocking on DAG generations. -func (d *dataset) generated() bool { - return atomic.LoadUint32(&d.done) == 1 -} - -// finalizer closes any file handlers and memory maps open. -func (d *dataset) finalizer() { - if d.mmap != nil { - d.mmap.Unmap() - d.dump.Close() - d.mmap, d.dump = nil, nil - } -} - -// MakeCache generates a new ethash cache and optionally stores it to disk. -func MakeCache(block uint64, dir string) { - c := cache{epoch: block / epochLength} - c.generate(dir, math.MaxInt32, false) -} - -// MakeDataset generates a new ethash dataset and optionally stores it to disk. -func MakeDataset(block uint64, dir string) { - d := dataset{epoch: block / epochLength} - d.generate(dir, math.MaxInt32, false) -} - -func (e *Ethash) dataset(block uint64, async bool) *dataset { - // Retrieve the requested ethash dataset - epoch := block / epochLength - currentI, futureI := e.datasets.get(epoch) - current := currentI.(*dataset) - - // If async is specified, generate everything in a background thread - if async && !current.generated() { - go func() { - current.generate(e.DatasetDir, e.DatasetsOnDisk, e.Test) - - if futureI != nil { - future := futureI.(*dataset) - future.generate(e.DatasetDir, e.DatasetsOnDisk, e.Test) - } - }() - } else { - // Either blocking generation was requested, or already done - current.generate(e.DatasetDir, e.DatasetsOnDisk, e.Test) - - if futureI != nil { - future := futureI.(*dataset) - go future.generate(e.DatasetDir, e.DatasetsOnDisk, e.Test) - } - } - return current -} - -// cache tries to retrieve a verification cache for the specified block number -// by first checking against a list of in-memory caches, then against caches -// stored on disk, and finally generating one if none can be found. -func (e *Ethash) cache(block uint64) *cache { - epoch := block / epochLength - currentI, futureI := e.caches.get(epoch) - current := currentI.(*cache) - - // Wait for generation finish. - current.generate(e.CacheDir, e.CachesOnDisk, e.Test) - - // If we need a new future cache, now's a good time to regenerate it. - if futureI != nil { - future := futureI.(*cache) - go future.generate(e.CacheDir, e.CachesOnDisk, e.Test) - } - return current -} diff --git a/consensus/ethash/difficulty.go b/consensus/ethash/difficulty.go index 08658cd..3cfe108 100644 --- a/consensus/ethash/difficulty.go +++ b/consensus/ethash/difficulty.go @@ -1,189 +1,109 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package ethash import ( "math/big" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" ) -// Based on geth - var ( - // difficultyBoundDivisor is the bound divisor of the difficulty, used in the update calculations. - difficultyBoundDivisor = big.NewInt(2048) - // minimumDifficulty is the minimum that the difficulty may ever be. - minimumDifficulty = big.NewInt(131072) - // durationLimit is the decision boundary on the blocktime duration used to determine whether difficulty should go up or not. - durationLimit = big.NewInt(13) + big2 = big.NewInt(2) + big2048 = big.NewInt(2048) + minDiff = big.NewInt(131072) + bigNeg99 = big.NewInt(-99) ) -var ( - expDiffPeriod = big.NewInt(100000) - big1 = big.NewInt(1) - big2 = big.NewInt(2) - big9 = big.NewInt(9) - big10 = big.NewInt(10) - bigMinus99 = big.NewInt(-99) -) +// ConstantinopleBombDelay is the bomb delay for the Constantinople fork +const ConstantinopleBombDelay = 5000000 -// calcDifficultyConstantinople is the Constantinople difficulty adjustment algorithm -var calcDifficultyConstantinople = makeDifficultyCalculator(big.NewInt(5000000)) - -// calcDifficultyByzantium is the Byzantium difficulty adjustment algorithm -var calcDifficultyByzantium = makeDifficultyCalculator(big.NewInt(3000000)) - -// makeDifficultyCalculator creates a difficultyCalculator with the given bomb-delay with Byzantium rules -func makeDifficultyCalculator(bombDelay *big.Int) func(time uint64, parent *types.Header) *big.Int { - // Note, the calculations below looks at the parent number, which is 1 below - // the block number. Thus we remove one from the delay given - bombDelayFromParent := new(big.Int).Sub(bombDelay, big1) - return func(time uint64, parent *types.Header) *big.Int { - // https://github.com/ethereum/EIPs/issues/100. - // algorithm: - // diff = (parent_diff + - // (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) - // ) + 2^(periodCount - 2) - - bigTime := new(big.Int).SetUint64(time) - bigParentTime := new(big.Int).Set(parent.Time) - - // holds intermediate values to make the algo easier to read & audit - x := new(big.Int) - y := new(big.Int) - - // (2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9 - x.Sub(bigTime, bigParentTime) - x.Div(x, big9) - if parent.UncleHash == types.EmptyUncleHash { - x.Sub(big1, x) - } else { - x.Sub(big2, x) - } - // max((2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9, -99) - if x.Cmp(bigMinus99) < 0 { - x.Set(bigMinus99) - } - // parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) - y.Div(parent.Difficulty, difficultyBoundDivisor) - x.Mul(y, x) - x.Add(parent.Difficulty, x) - - // minimum difficulty can ever be (before exponential factor) - if x.Cmp(minimumDifficulty) < 0 { - x.Set(minimumDifficulty) - } - // calculate a fake block number for the ice-age delay - // Specification: https://eips.ethereum.org/EIPS/eip-1234 - fakeBlockNumber := new(big.Int) - if parent.Number.Cmp(bombDelayFromParent) >= 0 { - fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, bombDelayFromParent) - } - // for the exponential factor - periodCount := fakeBlockNumber - periodCount.Div(periodCount, expDiffPeriod) - - // the exponential factor, commonly referred to as "the bomb" - // diff = diff + 2^(periodCount - 2) - if periodCount.Cmp(big1) > 0 { - y.Sub(periodCount, big2) - y.Exp(big2, y, nil) - x.Add(x, y) - } - return x +// ByzantiumBombDelay is the bomb delay for the Byzantium fork +const ByzantiumBombDelay = 3000000 + +// MetropolisDifficulty is the difficulty calculation for the metropolis forks +func MetropolisDifficulty(time uint64, parent *types.Header, bombDelay uint64) *big.Int { + diff := new(big.Int) + aux := new(big.Int) + + uncles := int64(1) + if parent.UncleHash != types.EmptyUncleHash { + uncles = 2 + } + if val := uncles - int64(time-parent.Time.Uint64())/9; val < -99 { + aux.Set(bigNeg99) + } else { + aux.SetInt64(val) } -} -// calcDifficultyHomestead is the Homestead difficulty adjustment algorithm -func calcDifficultyHomestead(time uint64, parent *types.Header) *big.Int { - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.md - // algorithm: - // diff = (parent_diff + - // (parent_diff / 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99)) - // ) + 2^(periodCount - 2) - - bigTime := new(big.Int).SetUint64(time) - bigParentTime := new(big.Int).Set(parent.Time) - - // holds intermediate values to make the algo easier to read & audit - x := new(big.Int) - y := new(big.Int) - - // 1 - (block_timestamp - parent_timestamp) // 10 - x.Sub(bigTime, bigParentTime) - x.Div(x, big10) - x.Sub(big1, x) - - // max(1 - (block_timestamp - parent_timestamp) // 10, -99) - if x.Cmp(bigMinus99) < 0 { - x.Set(bigMinus99) + diff.Div(parent.Difficulty, big2048) + diff.Mul(diff, aux) + diff.Add(diff, parent.Difficulty) + + if diff.Cmp(minDiff) < 0 { + diff.Set(minDiff) } - // (parent_diff + parent_diff // 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99)) - y.Div(parent.Difficulty, difficultyBoundDivisor) - x.Mul(y, x) - x.Add(parent.Difficulty, x) - - // minimum difficulty can ever be (before exponential factor) - if x.Cmp(minimumDifficulty) < 0 { - x.Set(minimumDifficulty) + + // bomb-delay + period := uint64(0) + if num := (parent.Number.Uint64() + 1); num >= bombDelay { + period = num - bombDelay } - // for the exponential factor - periodCount := new(big.Int).Add(parent.Number, big1) - periodCount.Div(periodCount, expDiffPeriod) - - // the exponential factor, commonly referred to as "the bomb" - // diff = diff + 2^(periodCount - 2) - if periodCount.Cmp(big1) > 0 { - y.Sub(periodCount, big2) - y.Exp(big2, y, nil) - x.Add(x, y) + if period := (period / 100000); period > 1 { + aux.SetUint64(period - 2) + aux.Exp(big2, aux, nil) + diff.Add(diff, aux) } - return x + return diff } -// calcDifficultyFrontier is the Frontier difficulty adjustment algorithm -func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int { +// HomesteadDifficulty is the difficulty calculation for the homestead fork +func HomesteadDifficulty(time uint64, parent *types.Header) *big.Int { diff := new(big.Int) - adjust := new(big.Int).Div(parent.Difficulty, difficultyBoundDivisor) - bigTime := new(big.Int) - bigParentTime := new(big.Int) + aux := new(big.Int) + + if val := (1 - int64((time-parent.Time.Uint64())/10)); val > -99 { + aux.SetInt64(val) + } else { + aux.Set(bigNeg99) + } - bigTime.SetUint64(time) - bigParentTime.Set(parent.Time) + diff.Div(parent.Difficulty, big2048) + diff.Mul(diff, aux) + diff.Add(diff, parent.Difficulty) + + if diff.Cmp(minDiff) < 0 { + diff.Set(minDiff) + } - if bigTime.Sub(bigTime, bigParentTime).Cmp(durationLimit) < 0 { - diff.Add(parent.Difficulty, adjust) + if period := ((parent.Number.Uint64() + 1) / 100000); period > 1 { + aux.SetUint64(period - 2) + aux.Exp(big2, aux, nil) + diff.Add(diff, aux) + } + return diff +} + +// FrontierDifficulty is the difficulty calculation for the frontier fork +func FrontierDifficulty(time uint64, parent *types.Header) *big.Int { + diff := new(big.Int).SetBytes(parent.Difficulty.Bytes()) + + aux := big.NewInt(1).Div(diff, big2048) + if time-parent.Time.Uint64() < 13 { + diff.Add(diff, aux) } else { - diff.Sub(parent.Difficulty, adjust) + diff.Sub(diff, aux) } - if diff.Cmp(minimumDifficulty) < 0 { - diff.Set(minimumDifficulty) + if diff.Cmp(minDiff) < 0 { + diff.Set(minDiff) } - periodCount := new(big.Int).Add(parent.Number, big1) - periodCount.Div(periodCount, expDiffPeriod) - if periodCount.Cmp(big1) > 0 { - // diff = diff + 2^(periodCount - 2) - expDiff := periodCount.Sub(periodCount, big2) - expDiff.Exp(big2, expDiff, nil) - diff.Add(diff, expDiff) - diff = math.BigMax(diff, minimumDifficulty) + if period := ((parent.Number.Uint64() + 1) / 100000); period > 1 { + aux.SetUint64(period - 2) + aux.Exp(big2, aux, nil) + diff.Add(diff, aux) + + if diff.Cmp(minDiff) < 0 { + diff.Set(minDiff) + } } return diff } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index bfebc24..2512571 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -3,140 +3,161 @@ package ethash import ( "bytes" "context" - "errors" "fmt" "math/big" - "math/rand" - "runtime" - "sync" "time" - "github.com/umbracle/minimal/chain" - "github.com/umbracle/minimal/consensus" - "github.com/umbracle/minimal/state" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" + lru "github.com/hashicorp/golang-lru" + "github.com/umbracle/minimal/chain" + "github.com/umbracle/minimal/consensus" + "github.com/umbracle/minimal/state" ) var ( - errLargeBlockTime = errors.New("timestamp too big") - errTooManyUncles = errors.New("too many uncles") - errInvalidDifficulty = errors.New("non-positive difficulty") - errInvalidMixDigest = errors.New("invalid mix digest") - errInvalidPoW = errors.New("invalid proof-of-work") - errFutureBlock = errors.New("future block") -) - -var ( - // FrontierBlockReward is the block reward in wei for successfully mining a block - FrontierBlockReward = big.NewInt(5e+18) - // ByzantiumBlockReward is the block reward in wei for successfully mining a block upward from Byzantium - ByzantiumBlockReward = big.NewInt(3e+18) - // ConstantinopleBlockReward is the block reward in wei for successfully mining a block upward from Constantinople - ConstantinopleBlockReward = big.NewInt(2e+18) - // maxUncles is the maximum number of uncles allowed in a single block - maxUncles = 2 - // allowedFutureBlockTime is the max time from current time allowed for blocks, before they're considered future blocks - allowedFutureBlockTime = 15 * time.Second + two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) ) -// EthHash consensus algorithm +// Ethash is the ethash consensus algorithm type Ethash struct { config *chain.Params - lock sync.Mutex - rand *rand.Rand - threads int - - DatasetDir string - DatasetsOnDisk int - - CacheDir string - CachesOnDisk int - - Test bool - - FakePow bool - caches *lru // In memory caches to avoid regenerating too often - datasets *lru // In memory datasets to avoid regenerating too often + cache *lru.Cache + fakePow bool } +// Factory is the factory method to create an Ethash consensus func Factory(ctx context.Context, config *consensus.Config) (consensus.Consensus, error) { + cache, _ := lru.New(2) e := &Ethash{ - config: config.Params, - caches: newlru("cache", 1, newCache), - datasets: newlru("dataset", 1, newDataset), + config: config.Params, + cache: cache, } return e, nil } -// NewEthHash creates a new ethash consensus -func NewEthHash(config *chain.Params, FakePow bool) *Ethash { - e := &Ethash{ - config: config, - caches: newlru("cache", 1, newCache), - datasets: newlru("dataset", 1, newDataset), - FakePow: FakePow, - } - return e -} - // VerifyHeader verifies the header is correct func (e *Ethash) VerifyHeader(parent *types.Header, header *types.Header, uncle, seal bool) error { - // Ensure that the header's extra-data section is of a reasonable size + headerNum := header.Number.Uint64() + parentNum := parent.Number.Uint64() + + if headerNum != parentNum+1 { + return fmt.Errorf("header and parent are non sequential") + } + if header.Time.Cmp(parent.Time) <= 0 { + return fmt.Errorf("incorrect timestamp") + } + + if header.Difficulty.Sign() <= 0 { + return fmt.Errorf("difficulty cannot be negative") + } + if uint64(len(header.Extra)) > chain.MaximumExtraDataSize { - return fmt.Errorf("Extra data too long") + return fmt.Errorf("extradata is too long") } if uncle { if header.Time.Cmp(math.MaxBig256) > 0 { - return errLargeBlockTime + return fmt.Errorf("incorrect uncle timestamp") } } else { - if header.Time.Cmp(big.NewInt(time.Now().Add(allowedFutureBlockTime).Unix())) > 0 { - return errFutureBlock + if header.Time.Cmp(big.NewInt(time.Now().Add(15*time.Second).Unix())) > 0 { + return fmt.Errorf("future block") } } - if header.Time.Cmp(parent.Time) <= 0 { - return fmt.Errorf("timestamp lower or equal than parent") + diff := e.CalcDifficulty(header.Time.Uint64(), parent) + if diff.Cmp(header.Difficulty) != 0 { + return fmt.Errorf("incorrect difficulty") } - // Verify the block's difficulty based in it's timestamp and parent's difficulty - expected := e.CalcDifficulty(header.Time.Uint64(), parent) - if expected.Cmp(header.Difficulty) != 0 { - return fmt.Errorf("difficulty not correct: expected %d but found %d", expected.Uint64(), header.Difficulty.Uint64()) - } - // Verify that the gas limit is <= 2^63-1 + cap := uint64(0x7fffffffffffffff) if header.GasLimit > cap { - return fmt.Errorf("gas limit not correct: have %v, want %v", header.GasLimit, cap) + return fmt.Errorf("incorrect gas limit") } - // Verify that the gasUsed is <= gasLimit + if header.GasUsed > header.GasLimit { - return fmt.Errorf("gas used not correct: have %v, want %v", header.GasUsed, header.GasLimit) + return fmt.Errorf("incorrect gas used") } - // Verify that the gas limit remains within allowed bounds - diff := int64(parent.GasLimit) - int64(header.GasLimit) - if diff < 0 { - diff *= -1 + + gas := int64(parent.GasLimit) - int64(header.GasLimit) + if gas < 0 { + gas *= -1 } + limit := parent.GasLimit / chain.GasLimitBoundDivisor + if uint64(gas) >= limit || header.GasLimit < chain.MinGasLimit { + return fmt.Errorf("incorrect gas limit") + } + + if !e.fakePow { + // Verify the seal + number := header.Number.Uint64() + cache := e.getCache(number) - if uint64(diff) >= limit || header.GasLimit < chain.MinGasLimit { - return fmt.Errorf("gas limit not correct") + digest, result := cache.hashimoto(sealHash(header).Bytes(), header.Nonce.Uint64()) + + if !bytes.Equal(header.MixDigest[:], digest) { + return fmt.Errorf("incorrect digest") + } + + target := new(big.Int).Div(two256, header.Difficulty) + if new(big.Int).SetBytes(result).Cmp(target) > 0 { + return fmt.Errorf("incorrect pow") + } } - // Verify that the block number is parent's +1 - if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { - return fmt.Errorf("invalid sequence") + + return nil +} + +func (e *Ethash) getCache(blockNumber uint64) *Cache { + epoch := blockNumber / uint64(epochLength) + cache, ok := e.cache.Get(epoch) + if ok { + return cache.(*Cache) } - // Verify the engine specific seal securing the block - if seal { - if err := e.verifySeal(header); err != nil { - return err - } + cc := newCache(int(epoch)) + e.cache.Add(epoch, cc) + return cc +} + +// SetFakePow sets the fakePow flag to true, only used on tests. +func (e *Ethash) SetFakePow() { + e.fakePow = true +} + +// CalcDifficulty calculates the difficulty at a given time. +func (e *Ethash) CalcDifficulty(time uint64, parent *types.Header) *big.Int { + next := parent.Number.Uint64() + 1 + switch { + case e.config.Forks.IsConstantinople(next): + return MetropolisDifficulty(time, parent, ConstantinopleBombDelay) + + case e.config.Forks.IsByzantium(next): + return MetropolisDifficulty(time, parent, ByzantiumBombDelay) + + case e.config.Forks.IsHomestead(next): + return HomesteadDifficulty(time, parent) + + default: + return FrontierDifficulty(time, parent) } +} + +// Author checks the author of the header +func (e *Ethash) Author(header *types.Header) (common.Address, error) { + return common.Address{}, nil +} + +// Seal seals the block +func (e *Ethash) Seal(ctx context.Context, block *types.Block) (*types.Block, error) { + return nil, nil +} + +// Prepare runs before processing the head during mining. +func (e *Ethash) Prepare(parent *types.Header, header *types.Header) error { return nil } @@ -145,19 +166,32 @@ var ( big32 = big.NewInt(32) ) +// Block rewards at different forks +var ( + // FrontierBlockReward is the block reward for the Frontier fork + FrontierBlockReward = big.NewInt(5e+18) + + // ByzantiumBlockReward is the block reward for the Byzantium fork + ByzantiumBlockReward = big.NewInt(3e+18) + + // ConstantinopleBlockReward is the block reward for the Constantinople fork + ConstantinopleBlockReward = big.NewInt(2e+18) +) + +// Finalize runs after the block has been processed func (e *Ethash) Finalize(txn *state.Txn, block *types.Block) error { number := block.Number() - // Select the correct block reward based on chain progression - blockReward := FrontierBlockReward - if e.config.Forks.IsByzantium(number.Uint64()) { - blockReward = ByzantiumBlockReward - } - if e.config.Forks.IsConstantinople(number.Uint64()) { + var blockReward *big.Int + switch num := number.Uint64(); { + case e.config.Forks.IsConstantinople(num): blockReward = ConstantinopleBlockReward + case e.config.Forks.IsByzantium(num): + blockReward = ByzantiumBlockReward + default: + blockReward = FrontierBlockReward } - // Accumulate the rewards for the miner and any included uncles reward := new(big.Int).Set(blockReward) r := new(big.Int) @@ -177,62 +211,6 @@ func (e *Ethash) Finalize(txn *state.Txn, block *types.Block) error { return nil } -func (e *Ethash) verifySeal(header *types.Header) error { - if e.FakePow { - return nil - } - - // Ensure that we have a valid difficulty for the block - if header.Difficulty.Sign() <= 0 { - return errInvalidDifficulty - } - // Recompute the digest and PoW values - number := header.Number.Uint64() - - var ( - digest []byte - result []byte - ) - - cache := e.cache(number) - - size := datasetSize(number) - digest, result = hashimotoLight(size, cache.cache, e.sealHash(header).Bytes(), header.Nonce.Uint64()) - - // Caches are unmapped in a finalizer. Ensure that the cache stays alive - // until after the call to hashimotoLight so it's not unmapped while being used. - runtime.KeepAlive(cache) - - // Verify the calculated values against the ones provided in the header - if !bytes.Equal(header.MixDigest[:], digest) { - return errInvalidMixDigest - } - target := new(big.Int).Div(two256, header.Difficulty) - if new(big.Int).SetBytes(result).Cmp(target) > 0 { - return errInvalidPoW - } - return nil -} - -// Author checks the author of the header -func (e *Ethash) Author(header *types.Header) (common.Address, error) { - return common.Address{}, nil -} - -func (e *Ethash) CalcDifficulty(time uint64, parent *types.Header) *big.Int { - next := parent.Number.Uint64() + 1 - switch { - case e.config.Forks.IsConstantinople(next): - return calcDifficultyConstantinople(time, parent) - case e.config.Forks.IsByzantium(next): - return calcDifficultyByzantium(time, parent) - case e.config.Forks.IsHomestead(next): - return calcDifficultyHomestead(time, parent) - default: - return calcDifficultyFrontier(time, parent) - } -} - // Close closes the connection func (e *Ethash) Close() error { return nil diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go deleted file mode 100644 index bd6805e..0000000 --- a/consensus/ethash/sealer.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package ethash - -import ( - "context" - crand "crypto/rand" - "errors" - "fmt" - "math" - "math/big" - "math/rand" - "runtime" - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" -) - -const ( - // staleThreshold is the maximum depth of the acceptable stale but valid ethash solution. - staleThreshold = 7 -) - -var ( - errNoMiningWork = errors.New("no mining work available yet") - errInvalidSealResult = errors.New("invalid or stale proof-of-work solution") -) - -func (e *Ethash) Prepare(parent *types.Header, header *types.Header) error { - header.Difficulty = e.CalcDifficulty(header.Time.Uint64(), parent) - return nil -} - -func (e *Ethash) Seal(ctx context.Context, block *types.Block) (*types.Block, error) { - fmt.Println("- seal -") - - // Create a runner and the multiple search threads it directs - abort := make(chan struct{}) - - e.lock.Lock() - threads := e.threads - if e.rand == nil { - seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) - if err != nil { - e.lock.Unlock() - return nil, err - } - e.rand = rand.New(rand.NewSource(seed.Int64())) - } - e.lock.Unlock() - if threads == 0 { - threads = runtime.NumCPU() - } - if threads < 0 { - threads = 0 // Allows disabling local mining without extra logic around local/remote - } - - var ( - pend sync.WaitGroup - locals = make(chan *types.Block) - ) - - threads = 1 - - for i := 0; i < threads; i++ { - pend.Add(1) - go func(id int, nonce uint64) { - defer pend.Done() - e.mine(block, id, nonce, abort, locals) - }(i, uint64(e.rand.Int63())) - } - - resultCh := make(chan *types.Block, 1) - - // Wait until sealing is terminated or a nonce is found - go func() { - var result *types.Block - select { - case <-ctx.Done(): - // Outside abort, stop all miner threads - close(abort) - case result = <-locals: - // One of the threads found a block, abort all others - close(abort) - } - // Wait for all miners to terminate and return the block - pend.Wait() - resultCh <- result - }() - - res := <-resultCh - return res, nil -} - -// SealHash returns the hash of a block prior to it being sealed. -func (e *Ethash) sealHash(header *types.Header) (hash common.Hash) { - hasher := sha3.NewLegacyKeccak256() - - rlp.Encode(hasher, []interface{}{ - header.ParentHash, - header.UncleHash, - header.Coinbase, - header.Root, - header.TxHash, - header.ReceiptHash, - header.Bloom, - header.Difficulty, - header.Number, - header.GasLimit, - header.GasUsed, - header.Time, - header.Extra, - }) - hasher.Sum(hash[:0]) - return hash -} - -// mine is the actual proof-of-work miner that searches for a nonce starting from -// seed that results in correct final block difficulty. -func (e *Ethash) mine(block *types.Block, id int, seed uint64, abort chan struct{}, found chan *types.Block) { - // Extract some data from the header - fmt.Println("- mine -") - - header := block.Header() - hash := e.sealHash(header).Bytes() - target := new(big.Int).Div(two256, header.Difficulty) - number := header.Number.Uint64() - dataset := e.dataset(number, false) - - // Start generating random nonces until we abort or find a good one - var ( - attempts = int64(0) - nonce = seed - ) - logger := log.New("miner", id) - logger.Trace("Started ethash search for new nonces", "seed", seed) - -search: - for { - select { - case <-abort: - // Mining terminated, update stats and abort - logger.Trace("Ethash nonce search aborted", "attempts", nonce-seed) - break search - - default: - // We don't have to update hash rate on every nonce, so update after after 2^X nonces - attempts++ - if (attempts % (1 << 15)) == 0 { - attempts = 0 - } - // Compute the PoW value of this nonce - digest, result := hashimotoFull(dataset.dataset, hash, nonce) - if new(big.Int).SetBytes(result).Cmp(target) <= 0 { - // Correct nonce found, create a new header with it - header = types.CopyHeader(header) - header.Nonce = types.EncodeNonce(nonce) - header.MixDigest = common.BytesToHash(digest) - - // Seal and return a block (if still needed) - select { - case found <- block.WithSeal(header): - logger.Trace("Ethash nonce found and reported", "attempts", nonce-seed, "nonce", nonce) - case <-abort: - logger.Trace("Ethash nonce found but discarded", "attempts", nonce-seed, "nonce", nonce) - } - break search - } - nonce++ - } - } - // Datasets are unmapped in a finalizer. Ensure that the dataset stays live - // during sealing so it's not unmapped while being read. - runtime.KeepAlive(dataset) -} diff --git a/tests/blockchain_test.go b/tests/blockchain_test.go index fcca83d..4aea456 100644 --- a/tests/blockchain_test.go +++ b/tests/blockchain_test.go @@ -2,6 +2,7 @@ package tests import ( "bytes" + "context" "encoding/json" "io/ioutil" "path/filepath" @@ -15,11 +16,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/umbracle/minimal/blockchain/storage/memory" "github.com/umbracle/minimal/chain" + "github.com/umbracle/minimal/consensus" "github.com/umbracle/minimal/consensus/ethash" - "github.com/umbracle/minimal/state" - trie "github.com/umbracle/minimal/state/immutable-trie" "github.com/umbracle/minimal/blockchain" + "github.com/umbracle/minimal/state" + trie "github.com/umbracle/minimal/state/immutable-trie" ) const blockchainTests = "BlockchainTests" @@ -95,7 +97,11 @@ func testBlockChainCase(t *testing.T, c *BlockchainTest) { fakePow = false } - engine := ethash.NewEthHash(params, fakePow) + engine, _ := ethash.Factory(context.Background(), &consensus.Config{Params: params}) + if fakePow { + engine.(*ethash.Ethash).SetFakePow() + } + genesis := c.buildGenesis() st := trie.NewState(trie.NewMemoryStorage()) diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index 5e2d25b..6dada3e 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -1,6 +1,7 @@ package tests import ( + "context" "encoding/json" "io/ioutil" "math/big" @@ -8,6 +9,7 @@ import ( "testing" "github.com/umbracle/minimal/chain" + "github.com/umbracle/minimal/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" @@ -121,7 +123,9 @@ func testDifficultyCase(t *testing.T, file string, config *chain.Forks) { t.Fatal(err) } - engine := ethash.NewEthHash(&chain.Params{Forks: config}, false) + engine, _ := ethash.Factory(context.Background(), &consensus.Config{Params: &chain.Params{Forks: config}}) + engineEthash := engine.(*ethash.Ethash) + for name, i := range cases { t.Run(name, func(t *testing.T) { if i.ParentDifficulty.Cmp(params.MinimumDifficulty) < 0 { @@ -138,7 +142,7 @@ func testDifficultyCase(t *testing.T, file string, config *chain.Forks) { UncleHash: i.UncleHash, } - difficulty := engine.CalcDifficulty(i.CurrentTimestamp.Uint64(), parent) + difficulty := engineEthash.CalcDifficulty(i.CurrentTimestamp.Uint64(), parent) if difficulty.Cmp(i.CurrentDifficulty) != 0 { t.Fatal() }