From 42f5960325ad03280911c3819e0f2cce01133c76 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 11 Jan 2024 12:24:13 -0500 Subject: [PATCH] initial commit --- LICENSE | 21 ++ README.md | 5 + db.go | 79 ++++ go.mod | 15 + go.sum | 23 ++ seed.go | 154 ++++++++ syncer/syncer.go | 918 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 1215 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 db.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 seed.go create mode 100644 syncer/syncer.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6de4e7f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 The Sia Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..971a3e6 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# [![Sia Core](https://sia.tech/assets/banners/sia-banner-core.png)](http://sia.tech) + +[![GoDoc](https://godoc.org/go.sia.tech/coreutils?status.svg)](https://godoc.org/go.sia.tech/coreutils) + +This repo contains implementations of core Sia components, such as a P2P syncing/relay node, database-backed blockchain state manager, single-address wallet, and more. It is intended for use by Sia node implementations like `walletd`, `renterd`, and `hostd`. diff --git a/db.go b/db.go new file mode 100644 index 0000000..ee2995a --- /dev/null +++ b/db.go @@ -0,0 +1,79 @@ +package coreutils + +import ( + "go.etcd.io/bbolt" + "go.sia.tech/core/chain" +) + +// BoltChainDB implements chain.DB with a BoltDB database. +type BoltChainDB struct { + tx *bbolt.Tx + db *bbolt.DB +} + +func (db *BoltChainDB) newTx() (err error) { + if db.tx == nil { + db.tx, err = db.db.Begin(true) + } + return +} + +// Bucket implements chain.DB. +func (db *BoltChainDB) Bucket(name []byte) chain.DBBucket { + if err := db.newTx(); err != nil { + panic(err) + } + // NOTE: can't simply return db.tx.Bucket here, since it returns a concrete + // type and we need a nil interface if the bucket does not exist + b := db.tx.Bucket(name) + if b == nil { + return nil + } + return b +} + +// CreateBucket implements chain.DB. +func (db *BoltChainDB) CreateBucket(name []byte) (chain.DBBucket, error) { + if err := db.newTx(); err != nil { + return nil, err + } + // NOTE: unlike Bucket, ok to return directly here, because caller should + // always check err first + return db.tx.CreateBucket(name) +} + +// Flush implements chain.DB. +func (db *BoltChainDB) Flush() error { + if db.tx == nil { + return nil + } + err := db.tx.Commit() + db.tx = nil + return err +} + +// Cancel implements chain.DB. +func (db *BoltChainDB) Cancel() { + if db.tx == nil { + return + } + db.tx.Rollback() + db.tx = nil +} + +// Close closes the BoltDB database. +func (db *BoltChainDB) Close() error { + db.Flush() + return db.db.Close() +} + +// NewBoltChainDB creates a new BoltChainDB. +func NewBoltChainDB(db *bbolt.DB) *BoltChainDB { + return &BoltChainDB{db: db} +} + +// OpenBoltChainDB opens a BoltDB database. +func OpenBoltChainDB(path string) (*BoltChainDB, error) { + db, err := bbolt.Open(path, 0600, nil) + return NewBoltChainDB(db), err +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ad39246 --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module go.sia.tech/coreutils + +go 1.21.5 + +require ( + go.etcd.io/bbolt v1.3.8 + go.sia.tech/core v0.1.12 +) + +require ( + github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect + golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect + golang.org/x/sys v0.5.0 // indirect + lukechampine.com/frand v1.4.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..46f5c60 --- /dev/null +++ b/go.sum @@ -0,0 +1,23 @@ +github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= +github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.sia.tech/core v0.1.11 h1:fRI8Mp1G280pOy0WYXSu6nI5dDDGPjHX1fbDwpwZLvc= +go.sia.tech/core v0.1.11/go.mod h1:D17UWSn99SEfQnEaR9G9n6Kz9+BwqMoUgZ6Cl424LsQ= +go.sia.tech/core v0.1.12 h1:nrq/BvYbTGVLtZu0MHBTExUAP5nfNbcGhaJbuK839gc= +go.sia.tech/core v0.1.12/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= +golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= +lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= diff --git a/seed.go b/seed.go new file mode 100644 index 0000000..138f1fc --- /dev/null +++ b/seed.go @@ -0,0 +1,154 @@ +package coreutils + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "strings" + + "go.sia.tech/core/types" + "golang.org/x/crypto/blake2b" + "lukechampine.com/frand" +) + +// NOTE: This is not a full implementation of BIP39; only 12-word phrases (128 +// bits of entropy) are supported. + +func memclr(p []byte) { + for i := range p { + p[i] = 0 + } +} + +// NewSeedPhrase generates a random seed phrase. +func NewSeedPhrase() string { + var entropy [16]byte + frand.Read(entropy[:]) + p := encodeBIP39Phrase(&entropy) + memclr(entropy[:]) + return p +} + +// SeedFromPhrase derives a 32-byte seed from the supplied phrase. +func SeedFromPhrase(seed *[32]byte, phrase string) error { + var entropy [16]byte + if err := decodeBIP39Phrase(&entropy, phrase); err != nil { + return err + } + h := blake2b.Sum256(entropy[:]) + memclr(entropy[:]) + copy(seed[:], h[:]) + memclr(h[:]) + return nil +} + +// KeyFromSeed returns the Ed25519 key derived from the supplied seed and index. +func KeyFromSeed(seed *[32]byte, index uint64) types.PrivateKey { + buf := make([]byte, 32+8) + copy(buf[:32], seed[:]) + binary.LittleEndian.PutUint64(buf[32:], index) + h := blake2b.Sum256(buf) + key := types.NewPrivateKeyFromSeed(h[:]) + memclr(h[:]) + return key +} + +func bip39checksum(entropy *[16]byte) uint64 { + hash := sha256.Sum256(entropy[:]) + return uint64((hash[0] & 0xF0) >> 4) +} + +func encodeBIP39Phrase(entropy *[16]byte) string { + // convert entropy to a 128-bit integer + hi := binary.BigEndian.Uint64(entropy[:8]) + lo := binary.BigEndian.Uint64(entropy[8:]) + + // convert each group of 11 bits into a word + words := make([]string, 12) + // last word is special: 4 bits are checksum + w := ((lo & 0x7F) << 4) | bip39checksum(entropy) + words[len(words)-1] = bip39EnglishWordList[w] + lo = lo>>7 | hi<<(64-7) + hi >>= 7 + for i := len(words) - 2; i >= 0; i-- { + words[i] = bip39EnglishWordList[lo&0x7FF] + lo = lo>>11 | hi<<(64-11) + hi >>= 11 + } + + return strings.Join(words, " ") +} + +func decodeBIP39Phrase(entropy *[16]byte, phrase string) error { + // validate that the phrase is well formed and only contains words that + // are present in the word list + words := strings.Fields(phrase) + if n := len(words); n != 12 { + return errors.New("wrong number of words in seed phrase") + } + for _, word := range words { + if _, ok := wordMap[word]; !ok { + return fmt.Errorf("unrecognized word %q in seed phrase", word) + } + } + + // convert words to 128 bits, 11 bits at a time + var lo, hi uint64 + for _, v := range words[:len(words)-1] { + hi = hi<<11 | lo>>(64-11) + lo = lo<<11 | wordMap[v] + } + // last word is special: least-significant 4 bits are checksum, so shift + // them off and only add the remaining 7 bits + w := wordMap[words[len(words)-1]] + checksum := w & 0xF + hi = hi<<7 | lo>>(64-7) + lo = lo<<7 | w>>4 + + // convert to big-endian byte slice + binary.BigEndian.PutUint64(entropy[:8], hi) + binary.BigEndian.PutUint64(entropy[8:], lo) + + // validate checksum + if bip39checksum(entropy) != checksum { + return errors.New("invalid checksum") + } + return nil +} + +var wordMap = func() map[string]uint64 { + m := make(map[string]uint64, len(bip39EnglishWordList)) + for i, v := range bip39EnglishWordList { + m[v] = uint64(i) + } + return m +}() + +var bip39EnglishWordList = []string{ + "abandon", "ability", "able", "about", "above", "absent", "absorb", "abstract", "absurd", "abuse", "access", "accident", "account", "accuse", "achieve", "acid", "acoustic", "acquire", "across", "act", "action", "actor", "actress", "actual", "adapt", "add", "addict", "address", "adjust", "admit", "adult", "advance", "advice", "aerobic", "affair", "afford", "afraid", "again", "age", "agent", "agree", "ahead", "aim", "air", "airport", "aisle", "alarm", "album", "alcohol", "alert", "alien", "all", "alley", "allow", "almost", "alone", "alpha", "already", "also", "alter", "always", "amateur", "amazing", "among", "amount", "amused", "analyst", "anchor", "ancient", "anger", "angle", "angry", "animal", "ankle", "announce", "annual", "another", "answer", "antenna", "antique", "anxiety", "any", "apart", "apology", "appear", "apple", "approve", "april", "arch", "arctic", "area", "arena", "argue", "arm", "armed", "armor", "army", "around", "arrange", "arrest", "arrive", "arrow", "art", "artefact", "artist", "artwork", "ask", "aspect", "assault", "asset", "assist", "assume", "asthma", "athlete", "atom", "attack", "attend", "attitude", "attract", "auction", "audit", "august", "aunt", "author", "auto", "autumn", "average", "avocado", "avoid", "awake", "aware", "away", "awesome", "awful", "awkward", "axis", + "baby", "bachelor", "bacon", "badge", "bag", "balance", "balcony", "ball", "bamboo", "banana", "banner", "bar", "barely", "bargain", "barrel", "base", "basic", "basket", "battle", "beach", "bean", "beauty", "because", "become", "beef", "before", "begin", "behave", "behind", "believe", "below", "belt", "bench", "benefit", "best", "betray", "better", "between", "beyond", "bicycle", "bid", "bike", "bind", "biology", "bird", "birth", "bitter", "black", "blade", "blame", "blanket", "blast", "bleak", "bless", "blind", "blood", "blossom", "blouse", "blue", "blur", "blush", "board", "boat", "body", "boil", "bomb", "bone", "bonus", "book", "boost", "border", "boring", "borrow", "boss", "bottom", "bounce", "box", "boy", "bracket", "brain", "brand", "brass", "brave", "bread", "breeze", "brick", "bridge", "brief", "bright", "bring", "brisk", "broccoli", "broken", "bronze", "broom", "brother", "brown", "brush", "bubble", "buddy", "budget", "buffalo", "build", "bulb", "bulk", "bullet", "bundle", "bunker", "burden", "burger", "burst", "bus", "business", "busy", "butter", "buyer", "buzz", + "cabbage", "cabin", "cable", "cactus", "cage", "cake", "call", "calm", "camera", "camp", "can", "canal", "cancel", "candy", "cannon", "canoe", "canvas", "canyon", "capable", "capital", "captain", "car", "carbon", "card", "cargo", "carpet", "carry", "cart", "case", "cash", "casino", "castle", "casual", "cat", "catalog", "catch", "category", "cattle", "caught", "cause", "caution", "cave", "ceiling", "celery", "cement", "census", "century", "cereal", "certain", "chair", "chalk", "champion", "change", "chaos", "chapter", "charge", "chase", "chat", "cheap", "check", "cheese", "chef", "cherry", "chest", "chicken", "chief", "child", "chimney", "choice", "choose", "chronic", "chuckle", "chunk", "churn", "cigar", "cinnamon", "circle", "citizen", "city", "civil", "claim", "clap", "clarify", "claw", "clay", "clean", "clerk", "clever", "click", "client", "cliff", "climb", "clinic", "clip", "clock", "clog", "close", "cloth", "cloud", "clown", "club", "clump", "cluster", "clutch", "coach", "coast", "coconut", "code", "coffee", "coil", "coin", "collect", "color", "column", "combine", "come", "comfort", "comic", "common", "company", "concert", "conduct", "confirm", "congress", "connect", "consider", "control", "convince", "cook", "cool", "copper", "copy", "coral", "core", "corn", "correct", "cost", "cotton", "couch", "country", "couple", "course", "cousin", "cover", "coyote", "crack", "cradle", "craft", "cram", "crane", "crash", "crater", "crawl", "crazy", "cream", "credit", "creek", "crew", "cricket", "crime", "crisp", "critic", "crop", "cross", "crouch", "crowd", "crucial", "cruel", "cruise", "crumble", "crunch", "crush", "cry", "crystal", "cube", "culture", "cup", "cupboard", "curious", "current", "curtain", "curve", "cushion", "custom", "cute", "cycle", + "dad", "damage", "damp", "dance", "danger", "daring", "dash", "daughter", "dawn", "day", "deal", "debate", "debris", "decade", "december", "decide", "decline", "decorate", "decrease", "deer", "defense", "define", "defy", "degree", "delay", "deliver", "demand", "demise", "denial", "dentist", "deny", "depart", "depend", "deposit", "depth", "deputy", "derive", "describe", "desert", "design", "desk", "despair", "destroy", "detail", "detect", "develop", "device", "devote", "diagram", "dial", "diamond", "diary", "dice", "diesel", "diet", "differ", "digital", "dignity", "dilemma", "dinner", "dinosaur", "direct", "dirt", "disagree", "discover", "disease", "dish", "dismiss", "disorder", "display", "distance", "divert", "divide", "divorce", "dizzy", "doctor", "document", "dog", "doll", "dolphin", "domain", "donate", "donkey", "donor", "door", "dose", "double", "dove", "draft", "dragon", "drama", "drastic", "draw", "dream", "dress", "drift", "drill", "drink", "drip", "drive", "drop", "drum", "dry", "duck", "dumb", "dune", "during", "dust", "dutch", "duty", "dwarf", "dynamic", + "eager", "eagle", "early", "earn", "earth", "easily", "east", "easy", "echo", "ecology", "economy", "edge", "edit", "educate", "effort", "egg", "eight", "either", "elbow", "elder", "electric", "elegant", "element", "elephant", "elevator", "elite", "else", "embark", "embody", "embrace", "emerge", "emotion", "employ", "empower", "empty", "enable", "enact", "end", "endless", "endorse", "enemy", "energy", "enforce", "engage", "engine", "enhance", "enjoy", "enlist", "enough", "enrich", "enroll", "ensure", "enter", "entire", "entry", "envelope", "episode", "equal", "equip", "era", "erase", "erode", "erosion", "error", "erupt", "escape", "essay", "essence", "estate", "eternal", "ethics", "evidence", "evil", "evoke", "evolve", "exact", "example", "excess", "exchange", "excite", "exclude", "excuse", "execute", "exercise", "exhaust", "exhibit", "exile", "exist", "exit", "exotic", "expand", "expect", "expire", "explain", "expose", "express", "extend", "extra", "eye", "eyebrow", + "fabric", "face", "faculty", "fade", "faint", "faith", "fall", "false", "fame", "family", "famous", "fan", "fancy", "fantasy", "farm", "fashion", "fat", "fatal", "father", "fatigue", "fault", "favorite", "feature", "february", "federal", "fee", "feed", "feel", "female", "fence", "festival", "fetch", "fever", "few", "fiber", "fiction", "field", "figure", "file", "film", "filter", "final", "find", "fine", "finger", "finish", "fire", "firm", "first", "fiscal", "fish", "fit", "fitness", "fix", "flag", "flame", "flash", "flat", "flavor", "flee", "flight", "flip", "float", "flock", "floor", "flower", "fluid", "flush", "fly", "foam", "focus", "fog", "foil", "fold", "follow", "food", "foot", "force", "forest", "forget", "fork", "fortune", "forum", "forward", "fossil", "foster", "found", "fox", "fragile", "frame", "frequent", "fresh", "friend", "fringe", "frog", "front", "frost", "frown", "frozen", "fruit", "fuel", "fun", "funny", "furnace", "fury", "future", + "gadget", "gain", "galaxy", "gallery", "game", "gap", "garage", "garbage", "garden", "garlic", "garment", "gas", "gasp", "gate", "gather", "gauge", "gaze", "general", "genius", "genre", "gentle", "genuine", "gesture", "ghost", "giant", "gift", "giggle", "ginger", "giraffe", "girl", "give", "glad", "glance", "glare", "glass", "glide", "glimpse", "globe", "gloom", "glory", "glove", "glow", "glue", "goat", "goddess", "gold", "good", "goose", "gorilla", "gospel", "gossip", "govern", "gown", "grab", "grace", "grain", "grant", "grape", "grass", "gravity", "great", "green", "grid", "grief", "grit", "grocery", "group", "grow", "grunt", "guard", "guess", "guide", "guilt", "guitar", "gun", "gym", "habit", + "hair", "half", "hammer", "hamster", "hand", "happy", "harbor", "hard", "harsh", "harvest", "hat", "have", "hawk", "hazard", "head", "health", "heart", "heavy", "hedgehog", "height", "hello", "helmet", "help", "hen", "hero", "hidden", "high", "hill", "hint", "hip", "hire", "history", "hobby", "hockey", "hold", "hole", "holiday", "hollow", "home", "honey", "hood", "hope", "horn", "horror", "horse", "hospital", "host", "hotel", "hour", "hover", "hub", "huge", "human", "humble", "humor", "hundred", "hungry", "hunt", "hurdle", "hurry", "hurt", "husband", "hybrid", + "ice", "icon", "idea", "identify", "idle", "ignore", "ill", "illegal", "illness", "image", "imitate", "immense", "immune", "impact", "impose", "improve", "impulse", "inch", "include", "income", "increase", "index", "indicate", "indoor", "industry", "infant", "inflict", "inform", "inhale", "inherit", "initial", "inject", "injury", "inmate", "inner", "innocent", "input", "inquiry", "insane", "insect", "inside", "inspire", "install", "intact", "interest", "into", "invest", "invite", "involve", "iron", "island", "isolate", "issue", "item", "ivory", + "jacket", "jaguar", "jar", "jazz", "jealous", "jeans", "jelly", "jewel", "job", "join", "joke", "journey", "joy", "judge", "juice", "jump", "jungle", "junior", "junk", "just", + "kangaroo", "keen", "keep", "ketchup", "key", "kick", "kid", "kidney", "kind", "kingdom", "kiss", "kit", "kitchen", "kite", "kitten", "kiwi", "knee", "knife", "knock", "know", + "lab", "label", "labor", "ladder", "lady", "lake", "lamp", "language", "laptop", "large", "later", "latin", "laugh", "laundry", "lava", "law", "lawn", "lawsuit", "layer", "lazy", "leader", "leaf", "learn", "leave", "lecture", "left", "leg", "legal", "legend", "leisure", "lemon", "lend", "length", "lens", "leopard", "lesson", "letter", "level", "liar", "liberty", "library", "license", "life", "lift", "light", "like", "limb", "limit", "link", "lion", "liquid", "list", "little", "live", "lizard", "load", "loan", "lobster", "local", "lock", "logic", "lonely", "long", "loop", "lottery", "loud", "lounge", "love", "loyal", "lucky", "luggage", "lumber", "lunar", "lunch", "luxury", "lyrics", + "machine", "mad", "magic", "magnet", "maid", "mail", "main", "major", "make", "mammal", "man", "manage", "mandate", "mango", "mansion", "manual", "maple", "marble", "march", "margin", "marine", "market", "marriage", "mask", "mass", "master", "match", "material", "math", "matrix", "matter", "maximum", "maze", "meadow", "mean", "measure", "meat", "mechanic", "medal", "media", "melody", "melt", "member", "memory", "mention", "menu", "mercy", "merge", "merit", "merry", "mesh", "message", "metal", "method", "middle", "midnight", "milk", "million", "mimic", "mind", "minimum", "minor", "minute", "miracle", "mirror", "misery", "miss", "mistake", "mix", "mixed", "mixture", "mobile", "model", "modify", "mom", "moment", "monitor", "monkey", "monster", "month", "moon", "moral", "more", "morning", "mosquito", "mother", "motion", "motor", "mountain", "mouse", "move", "movie", "much", "muffin", "mule", "multiply", "muscle", "museum", "mushroom", "music", "must", "mutual", "myself", "mystery", "myth", + "naive", "name", "napkin", "narrow", "nasty", "nation", "nature", "near", "neck", "need", "negative", "neglect", "neither", "nephew", "nerve", "nest", "net", "network", "neutral", "never", "news", "next", "nice", "night", "noble", "noise", "nominee", "noodle", "normal", "north", "nose", "notable", "note", "nothing", "notice", "novel", "now", "nuclear", "number", "nurse", "nut", + "oak", "obey", "object", "oblige", "obscure", "observe", "obtain", "obvious", "occur", "ocean", "october", "odor", "off", "offer", "office", "often", "oil", "okay", "old", "olive", "olympic", "omit", "once", "one", "onion", "online", "only", "open", "opera", "opinion", "oppose", "option", "orange", "orbit", "orchard", "order", "ordinary", "organ", "orient", "original", "orphan", "ostrich", "other", "outdoor", "outer", "output", "outside", "oval", "oven", "over", "own", "owner", "oxygen", "oyster", "ozone", + "pact", "paddle", "page", "pair", "palace", "palm", "panda", "panel", "panic", "panther", "paper", "parade", "parent", "park", "parrot", "party", "pass", "patch", "path", "patient", "patrol", "pattern", "pause", "pave", "payment", "peace", "peanut", "pear", "peasant", "pelican", "pen", "penalty", "pencil", "people", "pepper", "perfect", "permit", "person", "pet", "phone", "photo", "phrase", "physical", "piano", "picnic", "picture", "piece", "pig", "pigeon", "pill", "pilot", "pink", "pioneer", "pipe", "pistol", "pitch", "pizza", "place", "planet", "plastic", "plate", "play", "please", "pledge", "pluck", "plug", "plunge", "poem", "poet", "point", "polar", "pole", "police", "pond", "pony", "pool", "popular", "portion", "position", "possible", "post", "potato", "pottery", "poverty", "powder", "power", "practice", "praise", "predict", "prefer", "prepare", "present", "pretty", "prevent", "price", "pride", "primary", "print", "priority", "prison", "private", "prize", "problem", "process", "produce", "profit", "program", "project", "promote", "proof", "property", "prosper", "protect", "proud", "provide", "public", "pudding", "pull", "pulp", "pulse", "pumpkin", "punch", "pupil", "puppy", "purchase", "purity", "purpose", "purse", "push", "put", "puzzle", "pyramid", + "quality", "quantum", "quarter", "question", "quick", "quit", "quiz", "quote", + "rabbit", "raccoon", "race", "rack", "radar", "radio", "rail", "rain", "raise", "rally", "ramp", "ranch", "random", "range", "rapid", "rare", "rate", "rather", "raven", "raw", "razor", "ready", "real", "reason", "rebel", "rebuild", "recall", "receive", "recipe", "record", "recycle", "reduce", "reflect", "reform", "refuse", "region", "regret", "regular", "reject", "relax", "release", "relief", "rely", "remain", "remember", "remind", "remove", "render", "renew", "rent", "reopen", "repair", "repeat", "replace", "report", "require", "rescue", "resemble", "resist", "resource", "response", "result", "retire", "retreat", "return", "reunion", "reveal", "review", "reward", "rhythm", "rib", "ribbon", "rice", "rich", "ride", "ridge", "rifle", "right", "rigid", "ring", "riot", "ripple", "risk", "ritual", "rival", "river", "road", "roast", "robot", "robust", "rocket", "romance", "roof", "rookie", "room", "rose", "rotate", "rough", "round", "route", "royal", "rubber", "rude", "rug", "rule", "run", "runway", "rural", + "sad", "saddle", "sadness", "safe", "sail", "salad", "salmon", "salon", "salt", "salute", "same", "sample", "sand", "satisfy", "satoshi", "sauce", "sausage", "save", "say", "scale", "scan", "scare", "scatter", "scene", "scheme", "school", "science", "scissors", "scorpion", "scout", "scrap", "screen", "script", "scrub", "sea", "search", "season", "seat", "second", "secret", "section", "security", "seed", "seek", "segment", "select", "sell", "seminar", "senior", "sense", "sentence", "series", "service", "session", "settle", "setup", "seven", "shadow", "shaft", "shallow", "share", "shed", "shell", "sheriff", "shield", "shift", "shine", "ship", "shiver", "shock", "shoe", "shoot", "shop", "short", "shoulder", "shove", "shrimp", "shrug", "shuffle", "shy", "sibling", "sick", "side", "siege", "sight", "sign", "silent", "silk", "silly", "silver", "similar", "simple", "since", "sing", "siren", "sister", "situate", "six", "size", "skate", "sketch", "ski", "skill", "skin", "skirt", "skull", "slab", "slam", "sleep", "slender", "slice", "slide", "slight", "slim", "slogan", "slot", "slow", "slush", "small", "smart", "smile", "smoke", "smooth", "snack", "snake", "snap", "sniff", "snow", "soap", "soccer", "social", "sock", "soda", "soft", "solar", "soldier", "solid", "solution", "solve", "someone", "song", "soon", "sorry", "sort", "soul", "sound", "soup", "source", "south", "space", "spare", "spatial", "spawn", "speak", "special", "speed", "spell", "spend", "sphere", "spice", "spider", "spike", "spin", "spirit", "split", "spoil", "sponsor", "spoon", "sport", "spot", "spray", "spread", "spring", "spy", "square", "squeeze", "squirrel", "stable", "stadium", "staff", "stage", "stairs", "stamp", "stand", "start", "state", "stay", "steak", "steel", "stem", "step", "stereo", "stick", "still", "sting", "stock", "stomach", "stone", "stool", "story", "stove", "strategy", "street", "strike", "strong", "struggle", "student", "stuff", "stumble", "style", "subject", "submit", "subway", "success", "such", "sudden", "suffer", "sugar", "suggest", "suit", "summer", "sun", "sunny", "sunset", "super", "supply", "supreme", "sure", "surface", "surge", "surprise", "surround", "survey", "suspect", "sustain", "swallow", "swamp", "swap", "swarm", "swear", "sweet", "swift", "swim", "swing", "switch", "sword", "symbol", "symptom", "syrup", "system", + "table", "tackle", "tag", "tail", "talent", "talk", "tank", "tape", "target", "task", "taste", "tattoo", "taxi", "teach", "team", "tell", "ten", "tenant", "tennis", "tent", "term", "test", "text", "thank", "that", "theme", "then", "theory", "there", "they", "thing", "this", "thought", "three", "thrive", "throw", "thumb", "thunder", "ticket", "tide", "tiger", "tilt", "timber", "time", "tiny", "tip", "tired", "tissue", "title", "toast", "tobacco", "today", "toddler", "toe", "together", "toilet", "token", "tomato", "tomorrow", "tone", "tongue", "tonight", "tool", "tooth", "top", "topic", "topple", "torch", "tornado", "tortoise", "toss", "total", "tourist", "toward", "tower", "town", "toy", "track", "trade", "traffic", "tragic", "train", "transfer", "trap", "trash", "travel", "tray", "treat", "tree", "trend", "trial", "tribe", "trick", "trigger", "trim", "trip", "trophy", "trouble", "truck", "true", "truly", "trumpet", "trust", "truth", "try", "tube", "tuition", "tumble", "tuna", "tunnel", "turkey", "turn", "turtle", "twelve", "twenty", "twice", "twin", "twist", "two", "type", "typical", + "ugly", "umbrella", "unable", "unaware", "uncle", "uncover", "under", "undo", "unfair", "unfold", "unhappy", "uniform", "unique", "unit", "universe", "unknown", "unlock", "until", "unusual", "unveil", "update", "upgrade", "uphold", "upon", "upper", "upset", "urban", "urge", "usage", "use", "used", "useful", "useless", "usual", "utility", + "vacant", "vacuum", "vague", "valid", "valley", "valve", "van", "vanish", "vapor", "various", "vast", "vault", "vehicle", "velvet", "vendor", "venture", "venue", "verb", "verify", "version", "very", "vessel", "veteran", "viable", "vibrant", "vicious", "victory", "video", "view", "village", "vintage", "violin", "virtual", "virus", "visa", "visit", "visual", "vital", "vivid", "vocal", "voice", "void", "volcano", "volume", "vote", "voyage", + "wage", "wagon", "wait", "walk", "wall", "walnut", "want", "warfare", "warm", "warrior", "wash", "wasp", "waste", "water", "wave", "way", "wealth", "weapon", "wear", "weasel", "weather", "web", "wedding", "weekend", "weird", "welcome", "west", "wet", "whale", "what", "wheat", "wheel", "when", "where", "whip", "whisper", "wide", "width", "wife", "wild", "will", "win", "window", "wine", "wing", "wink", "winner", "winter", "wire", "wisdom", "wise", "wish", "witness", "wolf", "woman", "wonder", "wood", "wool", "word", "work", "world", "worry", "worth", "wrap", "wreck", "wrestle", "wrist", "write", "wrong", + "yard", "year", "yellow", "you", "young", "youth", + "zebra", "zero", "zone", "zoo", +} diff --git a/syncer/syncer.go b/syncer/syncer.go new file mode 100644 index 0000000..cacf757 --- /dev/null +++ b/syncer/syncer.go @@ -0,0 +1,918 @@ +package coreutils + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net" + "reflect" + "sync" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" + "go.sia.tech/core/types" + "lukechampine.com/frand" +) + +// A ChainManager manages blockchain state. +type ChainManager interface { + History() ([32]types.BlockID, error) + BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) + Block(id types.BlockID) (types.Block, bool) + State(id types.BlockID) (consensus.State, bool) + AddBlocks(blocks []types.Block) error + Tip() types.ChainIndex + TipState() consensus.State + + PoolTransaction(txid types.TransactionID) (types.Transaction, bool) + AddPoolTransactions(txns []types.Transaction) (bool, error) + V2PoolTransaction(txid types.TransactionID) (types.V2Transaction, bool) + AddV2PoolTransactions(basis types.ChainIndex, txns []types.V2Transaction) (bool, error) + TransactionsForPartialBlock(missing []types.Hash256) ([]types.Transaction, []types.V2Transaction) +} + +// PeerInfo contains metadata about a peer. +type PeerInfo struct { + FirstSeen time.Time `json:"firstSeen"` + LastConnect time.Time `json:"lastConnect,omitempty"` + SyncedBlocks uint64 `json:"syncedBlocks,omitempty"` + SyncDuration time.Duration `json:"syncDuration,omitempty"` +} + +// A PeerStore stores peers and bans. +type PeerStore interface { + AddPeer(peer string) + Peers() []string + UpdatePeerInfo(peer string, fn func(*PeerInfo)) + PeerInfo(peer string) (PeerInfo, bool) + + // Ban temporarily bans one or more IPs. The addr should either be a single + // IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. 1.2.3.4/16). + Ban(addr string, duration time.Duration, reason string) + Banned(peer string) bool +} + +// Subnet normalizes the provided CIDR subnet string. +func Subnet(addr, mask string) string { + ip, ipnet, err := net.ParseCIDR(addr + mask) + if err != nil { + return "" // shouldn't happen + } + return ip.Mask(ipnet.Mask).String() + mask +} + +type config struct { + MaxInboundPeers int + MaxOutboundPeers int + MaxInflightRPCs int + ConnectTimeout time.Duration + ShareNodesTimeout time.Duration + SendBlockTimeout time.Duration + SendTransactionsTimeout time.Duration + RelayHeaderTimeout time.Duration + RelayBlockOutlineTimeout time.Duration + RelayTransactionSetTimeout time.Duration + SendBlocksTimeout time.Duration + MaxSendBlocks uint64 + PeerDiscoveryInterval time.Duration + SyncInterval time.Duration + Logger *log.Logger +} + +// An Option modifies a Syncer's configuration. +type Option func(*config) + +// WithMaxInboundPeers sets the maximum number of inbound connections. The +// default is 8. +func WithMaxInboundPeers(n int) Option { + return func(c *config) { c.MaxInboundPeers = n } +} + +// WithMaxOutboundPeers sets the maximum number of outbound connections. The +// default is 8. +func WithMaxOutboundPeers(n int) Option { + return func(c *config) { c.MaxOutboundPeers = n } +} + +// WithMaxInflightRPCs sets the maximum number of concurrent RPCs per peer. The +// default is 3. +func WithMaxInflightRPCs(n int) Option { + return func(c *config) { c.MaxInflightRPCs = n } +} + +// WithConnectTimeout sets the timeout when connecting to a peer. The default is +// 5 seconds. +func WithConnectTimeout(d time.Duration) Option { + return func(c *config) { c.ConnectTimeout = d } +} + +// WithShareNodesTimeout sets the timeout for the ShareNodes RPC. The default is +// 5 seconds. +func WithShareNodesTimeout(d time.Duration) Option { + return func(c *config) { c.ShareNodesTimeout = d } +} + +// WithSendBlockTimeout sets the timeout for the SendBlock RPC. The default is +// 60 seconds. +func WithSendBlockTimeout(d time.Duration) Option { + return func(c *config) { c.SendBlockTimeout = d } +} + +// WithSendBlocksTimeout sets the timeout for the SendBlocks RPC. The default is +// 120 seconds. +func WithSendBlocksTimeout(d time.Duration) Option { + return func(c *config) { c.SendBlocksTimeout = d } +} + +// WithMaxSendBlocks sets the maximum number of blocks requested per SendBlocks +// RPC. The default is 10. +func WithMaxSendBlocks(n uint64) Option { + return func(c *config) { c.MaxSendBlocks = n } +} + +// WithSendTransactionsTimeout sets the timeout for the SendTransactions RPC. +// The default is 60 seconds. +func WithSendTransactionsTimeout(d time.Duration) Option { + return func(c *config) { c.SendTransactionsTimeout = d } +} + +// WithRelayHeaderTimeout sets the timeout for the RelayHeader and RelayV2Header +// RPCs. The default is 5 seconds. +func WithRelayHeaderTimeout(d time.Duration) Option { + return func(c *config) { c.RelayHeaderTimeout = d } +} + +// WithRelayBlockOutlineTimeout sets the timeout for the RelayV2BlockOutline +// RPC. The default is 60 seconds. +func WithRelayBlockOutlineTimeout(d time.Duration) Option { + return func(c *config) { c.RelayBlockOutlineTimeout = d } +} + +// WithRelayTransactionSetTimeout sets the timeout for the RelayTransactionSet +// RPC. The default is 60 seconds. +func WithRelayTransactionSetTimeout(d time.Duration) Option { + return func(c *config) { c.RelayTransactionSetTimeout = d } +} + +// WithPeerDiscoveryInterval sets the frequency at which the syncer attempts to +// discover and connect to new peers. The default is 5 seconds. +func WithPeerDiscoveryInterval(d time.Duration) Option { + return func(c *config) { c.PeerDiscoveryInterval = d } +} + +// WithSyncInterval sets the frequency at which the syncer attempts to sync with +// peers. The default is 5 seconds. +func WithSyncInterval(d time.Duration) Option { + return func(c *config) { c.SyncInterval = d } +} + +// WithLogger sets the logger used by a Syncer. The default is a logger that +// outputs to io.Discard. +func WithLogger(l *log.Logger) Option { + return func(c *config) { c.Logger = l } +} + +// A Syncer synchronizes blockchain data with peers. +type Syncer struct { + l net.Listener + cm ChainManager + pm PeerStore + header gateway.Header + config config + log *log.Logger // redundant, but convenient + + mu sync.Mutex + peers map[string]*gateway.Peer + synced map[string]bool + strikes map[string]int +} + +type rpcHandler struct { + s *Syncer +} + +func (h *rpcHandler) resync(p *gateway.Peer, reason string) { + h.s.mu.Lock() + alreadyResyncing := !h.s.synced[p.Addr] + h.s.synced[p.Addr] = false + h.s.mu.Unlock() + if !alreadyResyncing { + h.s.log.Printf("triggering resync with %v: %v", p, reason) + } +} + +func (h *rpcHandler) PeersForShare() (peers []string) { + peers = h.s.pm.Peers() + if len(peers) > 10 { + frand.Shuffle(len(peers), reflect.Swapper(peers)) + peers = peers[:10] + } + return peers +} + +func (h *rpcHandler) Block(id types.BlockID) (types.Block, error) { + b, ok := h.s.cm.Block(id) + if !ok { + return types.Block{}, errors.New("block not found") + } + return b, nil +} + +func (h *rpcHandler) BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) { + return h.s.cm.BlocksForHistory(history, max) +} + +func (h *rpcHandler) Transactions(index types.ChainIndex, txnHashes []types.Hash256) (txns []types.Transaction, v2txns []types.V2Transaction, _ error) { + if b, ok := h.s.cm.Block(index.ID); ok { + // get txns from block + want := make(map[types.Hash256]bool) + for _, h := range txnHashes { + want[h] = true + } + for _, txn := range b.Transactions { + if want[txn.FullHash()] { + txns = append(txns, txn) + } + } + for _, txn := range b.V2Transactions() { + if want[txn.FullHash()] { + v2txns = append(v2txns, txn) + } + } + return + } + txns, v2txns = h.s.cm.TransactionsForPartialBlock(txnHashes) + return +} + +func (h *rpcHandler) Checkpoint(index types.ChainIndex) (types.Block, consensus.State, error) { + b, ok1 := h.s.cm.Block(index.ID) + cs, ok2 := h.s.cm.State(b.ParentID) + if !ok1 || !ok2 { + return types.Block{}, consensus.State{}, errors.New("checkpoint not found") + } + return b, cs, nil +} + +func (h *rpcHandler) RelayHeader(bh gateway.BlockHeader, origin *gateway.Peer) { + cs, ok := h.s.cm.State(bh.ParentID) + if !ok { + h.resync(origin, fmt.Sprintf("peer relayed a header with unknown parent (%v)", bh.ParentID)) + return + } + bid := bh.ID() + if _, ok := h.s.cm.State(bid); ok { + return // already seen + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent header with insufficient work")) + return + } else if bh.ParentID != h.s.cm.Tip().ID { + // block extends a sidechain, which peer (if honest) believes to be the + // heaviest chain + h.resync(origin, "peer relayed a header that does not attach to our tip") + return + } + // request + validate full block + if b, err := origin.SendBlock(bh.ID(), h.s.config.SendBlockTimeout); err != nil { + // log-worthy, but not ban-worthy + h.s.log.Printf("couldn't retrieve new block %v after header relay from %v: %v", bh.ID(), origin, err) + return + } else if err := h.s.cm.AddBlocks([]types.Block{b}); err != nil { + h.s.ban(origin, err) + return + } + + h.s.relayHeader(bh, origin) // non-blocking +} + +func (h *rpcHandler) RelayTransactionSet(txns []types.Transaction, origin *gateway.Peer) { + if len(txns) == 0 { + h.s.ban(origin, errors.New("peer sent an empty transaction set")) + } else if known, err := h.s.cm.AddPoolTransactions(txns); !known { + if err != nil { + // too risky to ban here (txns are probably just outdated), but at least + // log it if we think we're synced + if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { + h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) + } + } else { + h.s.relayTransactionSet(txns, origin) // non-blocking + } + } +} + +func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { + cs, ok := h.s.cm.State(bh.Parent.ID) + if !ok { + h.resync(origin, fmt.Sprintf("peer relayed a v2 header with unknown parent (%v)", bh.Parent.ID)) + return + } + bid := bh.ID(cs) + if _, ok := h.s.cm.State(bid); ok { + return // already seen + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent v2 header with insufficient work")) + return + } else if bh.Parent != h.s.cm.Tip() { + // block extends a sidechain, which peer (if honest) believes to be the + // heaviest chain + h.resync(origin, "peer relayed a v2 header that does not attach to our tip") + return + } + + // header is sufficiently valid; relay it + // + // NOTE: The purpose of header announcements is to inform the network as + // quickly as possible that a new block has been found. A proper + // BlockOutline should follow soon after, allowing peers to obtain the + // actual block. As such, we take no action here other than relaying. + h.s.relayV2Header(bh, origin) // non-blocking +} + +func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gateway.Peer) { + cs, ok := h.s.cm.State(bo.ParentID) + if !ok { + h.resync(origin, fmt.Sprintf("peer relayed a v2 outline with unknown parent (%v)", bo.ParentID)) + return + } + bid := bo.ID(cs) + if _, ok := h.s.cm.State(bid); ok { + return // already seen + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent v2 outline with insufficient work")) + return + } else if bo.ParentID != h.s.cm.Tip().ID { + // block extends a sidechain, which peer (if honest) believes to be the + // heaviest chain + h.resync(origin, "peer relayed a v2 outline that does not attach to our tip") + return + } + + // block has sufficient work and attaches to our tip, but may be missing + // transactions; first, check for them in our txpool; then, if block is + // still incomplete, request remaining transactions from the peer + txns, v2txns := h.s.cm.TransactionsForPartialBlock(bo.Missing()) + b, missing := bo.Complete(cs, txns, v2txns) + if len(missing) > 0 { + index := types.ChainIndex{Height: bo.Height, ID: bid} + txns, v2txns, err := origin.SendTransactions(index, missing, h.s.config.SendTransactionsTimeout) + if err != nil { + // log-worthy, but not ban-worthy + h.s.log.Printf("couldn't retrieve missing transactions of %v after relay from %v: %v", bid, origin, err) + return + } + b, missing = bo.Complete(cs, txns, v2txns) + if len(missing) > 0 { + // inexcusable + h.s.ban(origin, errors.New("peer sent wrong missing transactions for a block it relayed")) + return + } + } + if err := h.s.cm.AddBlocks([]types.Block{b}); err != nil { + h.s.ban(origin, err) + return + } + + // when we forward the block, exclude any txns that were in our txpool, + // since they're probably present in our peers' txpools as well + // + // NOTE: crucially, we do NOT exclude any txns we had to request from the + // sending peer, since other peers probably don't have them either + bo.RemoveTransactions(txns, v2txns) + + h.s.relayV2BlockOutline(bo, origin) // non-blocking +} + +func (h *rpcHandler) RelayV2TransactionSet(basis types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { + if _, ok := h.s.cm.Block(basis.ID); !ok { + h.resync(origin, fmt.Sprintf("peer %v relayed a v2 transaction set with unknown basis (%v)", origin, basis)) + } else if len(txns) == 0 { + h.s.ban(origin, errors.New("peer sent an empty transaction set")) + } else if known, err := h.s.cm.AddV2PoolTransactions(basis, txns); !known { + if err != nil { + h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) + } else { + h.s.relayV2TransactionSet(basis, txns, origin) // non-blocking + } + } +} + +func (s *Syncer) ban(p *gateway.Peer, err error) { + s.log.Printf("banning %v: %v", p, err) + p.SetErr(errors.New("banned")) + s.pm.Ban(p.ConnAddr, 24*time.Hour, err.Error()) + + host, _, err := net.SplitHostPort(p.ConnAddr) + if err != nil { + return // shouldn't happen + } + // add a strike to each subnet + for subnet, maxStrikes := range map[string]int{ + Subnet(host, "/32"): 2, // 1.2.3.4:* + Subnet(host, "/24"): 8, // 1.2.3.* + Subnet(host, "/16"): 64, // 1.2.* + Subnet(host, "/8"): 512, // 1.* + } { + s.mu.Lock() + ban := (s.strikes[subnet] + 1) >= maxStrikes + if ban { + delete(s.strikes, subnet) + } else { + s.strikes[subnet]++ + } + s.mu.Unlock() + if ban { + s.pm.Ban(subnet, 24*time.Hour, "too many strikes") + } + } +} + +func (s *Syncer) runPeer(p *gateway.Peer) { + s.pm.AddPeer(p.Addr) + s.pm.UpdatePeerInfo(p.Addr, func(info *PeerInfo) { + info.LastConnect = time.Now() + }) + s.mu.Lock() + s.peers[p.Addr] = p + s.mu.Unlock() + defer func() { + s.mu.Lock() + delete(s.peers, p.Addr) + s.mu.Unlock() + }() + + h := &rpcHandler{s: s} + inflight := make(chan struct{}, s.config.MaxInflightRPCs) + for { + if p.Err() != nil { + return + } + id, stream, err := p.AcceptRPC() + if err != nil { + p.SetErr(err) + return + } + inflight <- struct{}{} + go func() { + defer stream.Close() + // NOTE: we do not set any deadlines on the stream. If a peer is + // slow, fine; we don't need to worry about resource exhaustion + // unless we have tons of peers. + if err := p.HandleRPC(id, stream, h); err != nil { + s.log.Printf("incoming RPC %v from peer %v failed: %v", id, p, err) + } + <-inflight + }() + } +} + +func (s *Syncer) relayHeader(h gateway.BlockHeader, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin { + continue + } + go p.RelayHeader(h, s.config.RelayHeaderTimeout) + } +} + +func (s *Syncer) relayTransactionSet(txns []types.Transaction, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin { + continue + } + go p.RelayTransactionSet(txns, s.config.RelayTransactionSetTimeout) + } +} + +func (s *Syncer) relayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin || !p.SupportsV2() { + continue + } + go p.RelayV2Header(bh, s.config.RelayHeaderTimeout) + } +} + +func (s *Syncer) relayV2BlockOutline(pb gateway.V2BlockOutline, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin || !p.SupportsV2() { + continue + } + go p.RelayV2BlockOutline(pb, s.config.RelayBlockOutlineTimeout) + } +} + +func (s *Syncer) relayV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin || !p.SupportsV2() { + continue + } + go p.RelayV2TransactionSet(index, txns, s.config.RelayTransactionSetTimeout) + } +} + +func (s *Syncer) allowConnect(peer string, inbound bool) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.l == nil { + return errors.New("syncer is shutting down") + } + if s.pm.Banned(peer) { + return errors.New("banned") + } + var in, out int + for _, p := range s.peers { + if p.Inbound { + in++ + } else { + out++ + } + } + // TODO: subnet-based limits + if inbound && in >= s.config.MaxInboundPeers { + return errors.New("too many inbound peers") + } else if !inbound && out >= s.config.MaxOutboundPeers { + return errors.New("too many outbound peers") + } + return nil +} + +func (s *Syncer) alreadyConnected(peer *gateway.Peer) bool { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p.UniqueID == peer.UniqueID { + return true + } + } + return false +} + +func (s *Syncer) acceptLoop() error { + for { + conn, err := s.l.Accept() + if err != nil { + return err + } + go func() { + defer conn.Close() + if err := s.allowConnect(conn.RemoteAddr().String(), true); err != nil { + s.log.Printf("rejected inbound connection from %v: %v", conn.RemoteAddr(), err) + } else if p, err := gateway.Accept(conn, s.header); err != nil { + s.log.Printf("failed to accept inbound connection from %v: %v", conn.RemoteAddr(), err) + } else if s.alreadyConnected(p) { + s.log.Printf("rejected inbound connection from %v: already connected", conn.RemoteAddr()) + } else { + s.runPeer(p) + } + }() + } +} + +func (s *Syncer) peerLoop(closeChan <-chan struct{}) error { + numOutbound := func() (n int) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if !p.Inbound { + n++ + } + } + return + } + + lastTried := make(map[string]time.Time) + peersForConnect := func() (peers []string) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.pm.Peers() { + // TODO: don't include port in comparison + if _, ok := s.peers[p]; !ok && time.Since(lastTried[p]) > 5*time.Minute { + peers = append(peers, p) + } + } + // TODO: weighted random selection? + frand.Shuffle(len(peers), reflect.Swapper(peers)) + return peers + } + discoverPeers := func() { + // try up to three randomly-chosen peers + var peers []*gateway.Peer + s.mu.Lock() + for _, p := range s.peers { + if peers = append(peers, p); len(peers) >= 3 { + break + } + } + s.mu.Unlock() + for _, p := range peers { + nodes, err := p.ShareNodes(s.config.ShareNodesTimeout) + if err != nil { + continue + } + for _, n := range nodes { + s.pm.AddPeer(n) + } + } + } + + ticker := time.NewTicker(s.config.PeerDiscoveryInterval) + defer ticker.Stop() + sleep := func() bool { + select { + case <-ticker.C: + return true + case <-closeChan: + return false + } + } + closing := func() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.l == nil + } + for fst := true; fst || sleep(); fst = false { + if numOutbound() >= s.config.MaxOutboundPeers { + continue + } + candidates := peersForConnect() + if len(candidates) == 0 { + discoverPeers() + continue + } + for _, p := range candidates { + if numOutbound() >= s.config.MaxOutboundPeers || closing() { + break + } + // NOTE: we don't bother logging failure here, since it's common and + // not particularly interesting or actionable + if _, err := s.Connect(p); err == nil { + s.log.Printf("formed outbound connection to %v", p) + } + lastTried[p] = time.Now() + } + } + return nil +} + +func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { + peersForSync := func() (peers []*gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if s.synced[p.Addr] { + continue + } + if peers = append(peers, p); len(peers) >= 3 { + break + } + } + return + } + + ticker := time.NewTicker(s.config.SyncInterval) + defer ticker.Stop() + sleep := func() bool { + select { + case <-ticker.C: + return true + case <-closeChan: + return false + } + } + for fst := true; fst || sleep(); fst = false { + for _, p := range peersForSync() { + history, err := s.cm.History() + if err != nil { + return err // generally fatal + } + s.mu.Lock() + s.synced[p.Addr] = true + s.mu.Unlock() + s.log.Printf("starting sync with %v", p) + oldTip := s.cm.Tip() + oldTime := time.Now() + lastPrint := time.Now() + startTime, startHeight := oldTime, oldTip.Height + var sentBlocks uint64 + addBlocks := func(blocks []types.Block) error { + if err := s.cm.AddBlocks(blocks); err != nil { + return err + } + sentBlocks += uint64(len(blocks)) + endTime, endHeight := time.Now(), s.cm.Tip().Height + s.pm.UpdatePeerInfo(p.Addr, func(info *PeerInfo) { + info.SyncedBlocks += endHeight - startHeight + info.SyncDuration += endTime.Sub(startTime) + }) + startTime, startHeight = endTime, endHeight + if time.Since(lastPrint) > 30*time.Second { + s.log.Printf("syncing with %v, tip now %v (avg %.2f blocks/s)", p, s.cm.Tip(), float64(s.cm.Tip().Height-oldTip.Height)/endTime.Sub(oldTime).Seconds()) + lastPrint = time.Now() + } + return nil + } + if p.SupportsV2() { + history := history[:] + err = func() error { + for { + blocks, rem, err := p.SendV2Blocks(history, s.config.MaxSendBlocks, s.config.SendBlocksTimeout) + if err != nil { + return err + } else if err := addBlocks(blocks); err != nil { + return err + } else if rem == 0 { + return nil + } + history = []types.BlockID{blocks[len(blocks)-1].ID()} + } + }() + } else { + err = p.SendBlocks(history, s.config.SendBlocksTimeout, addBlocks) + } + totalBlocks := s.cm.Tip().Height - oldTip.Height + if err != nil { + s.log.Printf("syncing with %v failed after %v blocks: %v", p, totalBlocks, err) + } else if newTip := s.cm.Tip(); newTip != oldTip { + s.log.Printf("finished syncing %v blocks with %v, tip now %v", totalBlocks, p, newTip) + } else { + s.log.Printf("finished syncing %v blocks with %v, tip unchanged", sentBlocks, p) + } + } + } + return nil +} + +// Run spawns goroutines for accepting inbound connections, forming outbound +// connections, and syncing the blockchain from active peers. It blocks until an +// error occurs, upon which all connections are closed and goroutines are +// terminated. To gracefully shutdown a Syncer, close its net.Listener. +func (s *Syncer) Run() error { + errChan := make(chan error) + closeChan := make(chan struct{}) + go func() { errChan <- s.acceptLoop() }() + go func() { errChan <- s.peerLoop(closeChan) }() + go func() { errChan <- s.syncLoop(closeChan) }() + err := <-errChan + + // when one goroutine exits, shutdown and wait for the others + close(closeChan) + s.l.Close() + s.mu.Lock() + s.l = nil + for _, p := range s.peers { + p.Close() + } + s.mu.Unlock() + <-errChan + <-errChan + + // wait for all peer goroutines to exit + // TODO: a cond would be nicer than polling here + s.mu.Lock() + for len(s.peers) != 0 { + s.mu.Unlock() + time.Sleep(100 * time.Millisecond) + s.mu.Lock() + } + s.mu.Unlock() + + if errors.Is(err, net.ErrClosed) { + return nil // graceful shutdown + } + return err +} + +// Connect forms an outbound connection to a peer. +func (s *Syncer) Connect(addr string) (*gateway.Peer, error) { + if err := s.allowConnect(addr, false); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), s.config.ConnectTimeout) + defer cancel() + // slightly gross polling hack so that we shutdown quickly + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + s.mu.Lock() + if s.l == nil { + cancel() + } + s.mu.Unlock() + } + } + }() + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(s.config.ConnectTimeout)) + defer conn.SetDeadline(time.Time{}) + p, err := gateway.Dial(conn, s.header) + if err != nil { + conn.Close() + return nil, err + } else if s.alreadyConnected(p) { + conn.Close() + return nil, errors.New("already connected") + } + go s.runPeer(p) + + // runPeer does this too, but doing it outside the goroutine prevents a race + s.mu.Lock() + s.peers[p.Addr] = p + s.mu.Unlock() + return p, nil +} + +// BroadcastHeader broadcasts a header to all peers. +func (s *Syncer) BroadcastHeader(h gateway.BlockHeader) { s.relayHeader(h, nil) } + +// BroadcastV2Header broadcasts a v2 header to all peers. +func (s *Syncer) BroadcastV2Header(h gateway.V2BlockHeader) { s.relayV2Header(h, nil) } + +// BroadcastV2BlockOutline broadcasts a v2 block outline to all peers. +func (s *Syncer) BroadcastV2BlockOutline(b gateway.V2BlockOutline) { s.relayV2BlockOutline(b, nil) } + +// BroadcastTransactionSet broadcasts a transaction set to all peers. +func (s *Syncer) BroadcastTransactionSet(txns []types.Transaction) { s.relayTransactionSet(txns, nil) } + +// BroadcastV2TransactionSet broadcasts a v2 transaction set to all peers. +func (s *Syncer) BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) { + s.relayV2TransactionSet(index, txns, nil) +} + +// Peers returns the set of currently-connected peers. +func (s *Syncer) Peers() []*gateway.Peer { + s.mu.Lock() + defer s.mu.Unlock() + var peers []*gateway.Peer + for _, p := range s.peers { + peers = append(peers, p) + } + return peers +} + +// PeerInfo returns metadata about the specified peer. +func (s *Syncer) PeerInfo(peer string) (PeerInfo, bool) { + s.mu.Lock() + defer s.mu.Unlock() + info, ok := s.pm.PeerInfo(peer) + return info, ok +} + +// Addr returns the address of the Syncer. +func (s *Syncer) Addr() string { + return s.l.Addr().String() +} + +// New returns a new Syncer. +func New(l net.Listener, cm ChainManager, pm PeerStore, header gateway.Header, opts ...Option) *Syncer { + config := config{ + MaxInboundPeers: 8, + MaxOutboundPeers: 8, + MaxInflightRPCs: 3, + ConnectTimeout: 5 * time.Second, + ShareNodesTimeout: 5 * time.Second, + SendBlockTimeout: 60 * time.Second, + SendTransactionsTimeout: 60 * time.Second, + RelayHeaderTimeout: 5 * time.Second, + RelayBlockOutlineTimeout: 60 * time.Second, + RelayTransactionSetTimeout: 60 * time.Second, + SendBlocksTimeout: 120 * time.Second, + MaxSendBlocks: 10, + PeerDiscoveryInterval: 5 * time.Second, + SyncInterval: 5 * time.Second, + Logger: log.New(io.Discard, "", 0), + } + for _, opt := range opts { + opt(&config) + } + return &Syncer{ + l: l, + cm: cm, + pm: pm, + header: header, + config: config, + log: config.Logger, + peers: make(map[string]*gateway.Peer), + synced: make(map[string]bool), + strikes: make(map[string]int), + } +}