testing/loadbot: Add moving markets.

Add several tools to the loadbot to allow for sideways, trending, and
volatile markets. Remove most mining as continuous mining is provided by
the harnesses themselves, and this allows us to run multiple bots at
once without excessive mining.

To allow for sideways and trending markets, new options for the
sidestacker can be used. These are a flat increase to the rate or an
automatic oscillating effect. Volitile markets can be acheived by
running the ts bot separately that randomly buys, or sells, large sums
moving the market violently.

dex/testing/dcrdex/harness.sh

@@ -130,6 +130,14 @@ if [ $ETH_ON -eq 0 ]; then
             "epochDuration": ${EPOCH_DURATION},
             "marketBuyBuffer": 1.2
         },
+        {
+            "base": "BTC_simnet",
+            "quote": "ETH_simnet",
+            "lotSize": 1000000,
+            "rateStep": 1000,
+            "epochDuration": ${EPOCH_DURATION},
+            "marketBuyBuffer": 1.2
+        },
         {
             "base": "DCR_simnet",
             "quote": "DEXTT_simnet",

dex/testing/loadbot/README.md

@@ -24,7 +24,8 @@ pingpong **traders** in parallel.
 The **sidestacker** program runs 2 **traders**, one seller and one buyer. Each
 epoch, the **traders** will attempt to create order book depth on their side.
 If the book is deep enough already, they will place taker orders targeting
-the other side.
+the other side. Can be further altered by setting a linear increase
+(trending market) or an automatic ascending/descending pattern (sideways market).
 
 #### compound
 
@@ -38,6 +39,12 @@ The **heavy** program is like **compound** on steroids. Four
 **sidestacker traders** with 6 orders per epoch, and a 5-order per epoch
 **sniper**.
 
+#### ts
+
+The **ts** program runs multiple **traders** that randomly push the market in
+one direction or the other. Can be run separately with other programs to create
+a volitile market.
+
 ### Logging
 
 Debug logging can be enabled with the `-debug` flag. Trace logging can be

dex/testing/loadbot/compound.go

@@ -13,17 +13,16 @@ import (
 // runCompound runs the 'compound' program, consisting of 2 (5/3) unmetered
 // sideStackers, a 1-order sniper, and a pingPonger.
 func runCompound() {
-	go blockEvery2()
-
+	var oscillator uint64
 	var wg sync.WaitGroup
 	wg.Add(4)
 	go func() {
 		defer wg.Done()
-		runTrader(newSideStacker(true, 5, 3, alpha, false, log.SubLogger("STACKER:0")), "CMPD:STACKER:0")
+		runTrader(newSideStacker(20, 3, alpha, true, false, true, &oscillator, log.SubLogger("STACKER:0")), "CMPD:STACKER:0")
 	}()
 	go func() {
 		defer wg.Done()
-		runTrader(newSideStacker(false, 5, 3, alpha, false, log.SubLogger("STACKER:1")), "CMPD:STACKER:1")
+		runTrader(newSideStacker(20, 3, alpha, false, false, false, &oscillator, log.SubLogger("STACKER:1")), "CMPD:STACKER:1")
 	}()
 	go func() {
 		defer wg.Done()
@@ -59,25 +58,24 @@ func runHeavy() {
 		}
 	}
 
-	go blockEvery2()
-
+	var oscillator uint64
 	var wg sync.WaitGroup
 	wg.Add(5)
 	go func() {
 		defer wg.Done()
-		runTrader(newSideStacker(true, 12, 6, alpha, true, log.SubLogger("STACKER:0")), "HEAVY:STACKER:0")
+		runTrader(newSideStacker(24, 6, alpha, true, true, true, &oscillator, log.SubLogger("STACKER:0")), "HEAVY:STACKER:0")
 	}()
 	go func() {
 		defer wg.Done()
-		runTrader(newSideStacker(false, 12, 6, alpha, true, log.SubLogger("STACKER:1")), "HEAVY:STACKER:1")
+		runTrader(newSideStacker(24, 6, alpha, false, true, false, &oscillator, log.SubLogger("STACKER:1")), "HEAVY:STACKER:1")
 	}()
 	go func() {
 		defer wg.Done()
-		runTrader(newSideStacker(true, 8, 4, beta, false, log.SubLogger("STACKER:2")), "HEAVY:STACKER:2")
+		runTrader(newSideStacker(16, 4, beta, true, false, false, &oscillator, log.SubLogger("STACKER:2")), "HEAVY:STACKER:2")
 	}()
 	go func() {
 		defer wg.Done()
-		runTrader(newSideStacker(false, 8, 4, beta, false, log.SubLogger("STACKER:3")), "HEAVY:STACKER:3")
+		runTrader(newSideStacker(16, 4, beta, false, false, false, &oscillator, log.SubLogger("STACKER:3")), "HEAVY:STACKER:3")
 	}()
 	go func() {
 		defer wg.Done()

dex/testing/loadbot/loadbot.go

@@ -87,7 +87,7 @@ var (
 
 	usr, _       = user.Current()
 	dextestDir   = filepath.Join(usr.HomeDir, "dextest")
-	botDir       = filepath.Join(dextestDir, "loadbot")
+	botDir       = filepath.Join(dextestDir, fmt.Sprintf("loadbot_%d", time.Now().Unix()))
 	alphaIPCFile = filepath.Join(dextestDir, "eth", "alpha", "node", "geth.ipc")
 	betaIPCFile  = filepath.Join(dextestDir, "eth", "beta", "node", "geth.ipc")
 
@@ -102,12 +102,14 @@ var (
 	epochDuration                                         uint64
 	lotSize                                               uint64
 	rateStep                                              uint64
+	rateIncrease                                          int64
 	conversionFactors                                     = make(map[string]uint64)
 
-	ethInitFee                     = (dexeth.InitGas(1, 0) + dexeth.RefundGas(0)) * ethFeeRate
-	ethRedeemFee                   = dexeth.RedeemGas(1, 0) * ethFeeRate
-	defaultMidGap, marketBuyBuffer float64
-	keepMidGap                     bool
+	ethInitFee                                     = (dexeth.InitGas(1, 0) + dexeth.RefundGas(0)) * ethFeeRate
+	ethRedeemFee                                   = dexeth.RedeemGas(1, 0) * ethFeeRate
+	defaultMidGap, marketBuyBuffer                 float64
+	keepMidGap, oscillate, randomOsc, ignoreErrors bool
+	oscInterval, oscStep, tsFrequency              uint64
 
 	processesMtx sync.Mutex
 	processes    []*process
@@ -352,6 +354,13 @@ func run() error {
 	flag.BoolVar(&trace, "trace", false, "use trace logging")
 	flag.IntVar(&m, "m", 0, "for compound and sidestacker, m is the number of makers to stack before placing takers")
 	flag.IntVar(&n, "n", 0, "for compound and sidestacker, n is the number of orders to place per epoch (default 3)")
+	flag.Int64Var(&rateIncrease, "increase", 0, "for compound and sidestacker, increase is applied to every order and increases or decreases price by the chosen flat rate, use to create a market trending in one direction. Must be a multiple of the market's rate step (default 0)")
+	flag.BoolVar(&oscillate, "oscillate", false, "for compound and sidestacker, whether the price should move up and down inside a window, use to emulate a sideways market (default false)")
+	flag.BoolVar(&randomOsc, "randomosc", false, "for compound and sidestacker, oscillate more randomly")
+	flag.Uint64Var(&oscInterval, "oscinterval", 300, "for compound and sidestacker, the number of epochs to take for a full oscillation cycle.")
+	flag.Uint64Var(&oscStep, "oscstep", 50, "for compound and sidestacker, the number of rate step to increase or decrese per epoch.")
+	flag.BoolVar(&ignoreErrors, "ignoreerrors", false, "log and ignore errors rather than the default behavior of stopping loadbot")
+	flag.Uint64Var(&tsFrequency, "tsfrequency", 4, "controls the frequency with which the ts fires after it is ready. 1 fires immediately while larger numbers fire less often")
 	flag.Parse()
 
 	if programName == "" {
@@ -435,6 +444,11 @@ func run() error {
 		marketBuyBuffer = mkt.MBBuffer
 		break
 	}
+
+	if rateIncrease%int64(rateStep) != 0 {
+		return fmt.Errorf("rate increase must be divisible by the rate step")
+	}
+
 	// Adjust to be comparable to the dcr_btc market.
 	defaultMidGap = defaultBtcPerDcr * float64(rateStep) / 100
 
@@ -464,7 +478,7 @@ func run() error {
 	if err != nil {
 		return fmt.Errorf("error creating LoggerMaker: %v", err)
 	}
-	log /* global */ = loggerMaker.NewLogger("LOADBOT", dex.LevelInfo)
+	log /* global */ = loggerMaker.NewLogger("LOADBOT")
 
 	log.Infof("Running program %s", programName)
 
@@ -534,6 +548,7 @@ func run() error {
 	if err != nil {
 		return fmt.Errorf("error creating loadbot directory: %v", err)
 	}
+	defer os.RemoveAll(botDir)
 
 	// Run any specified network conditions.
 	var toxics toxiproxy.Toxics
@@ -651,11 +666,13 @@ func run() error {
 	case "pingpong4":
 		runPingPong(4)
 	case "sidestacker":
-		runSideStacker(5, 3)
+		runSideStacker(20, 3)
 	case "compound":
 		runCompound()
 	case "heavy":
 		runHeavy()
+	case "ts":
+		runTS()
 	default:
 		log.Criticalf("program " + programName + " not known")
 	}
@@ -698,3 +715,28 @@ func loadNodeConfig(symbol, node string) map[string]string {
 	}
 	return cfg
 }
+
+func symmetricWalletConfig(numCoins int, midGap uint64) (
+	minBaseQty, maxBaseQty, minQuoteQty, maxQuoteQty uint64) {
+
+	minBaseQty = uint64(maxOrderLots) * uint64(numCoins) * lotSize
+	minQuoteQty = calc.BaseToQuote(midGap, minBaseQty)
+	// Ensure enough for registration fees.
+	if minBaseQty < 2e8 {
+		minBaseQty = 2e8
+	}
+	if minQuoteQty < 2e8 {
+		minQuoteQty = 2e8
+	}
+	// eth fee estimation calls for more reserves.
+	if quoteSymbol == eth {
+		add := (ethRedeemFee + ethInitFee) * uint64(maxOrderLots)
+		minQuoteQty += add
+	}
+	if baseSymbol == eth {
+		add := (ethRedeemFee + ethInitFee) * uint64(maxOrderLots)
+		minBaseQty += add
+	}
+	maxBaseQty, maxQuoteQty = minBaseQty*2, minQuoteQty*2
+	return
+}

dex/testing/loadbot/mantle.go

@@ -81,7 +81,7 @@ out:
 		case note := <-m.notes:
 			if note.Severity() >= db.ErrorLevel {
 				m.fatalError("Error note received: %s", mustJSON(note))
-				return
+				continue
 			}
 			switch n := note.(type) {
 			case *core.FeePaymentNote:
@@ -111,9 +111,6 @@ out:
 				}
 			case *core.EpochNotification:
 				m.log.Debugf("Epoch note received: %s", mustJSON(note))
-				if n.MarketID == market {
-					m.replenishBalances()
-				}
 			case *core.MatchNote:
 				if n.Topic() == core.TopicNewMatch {
 					atomic.AddUint32(&matchCounter, 1)
@@ -185,7 +182,9 @@ func newMantle(name string) (*Mantle, error) {
 // fatalError kills the LoadBot by cancelling the global Context.
 func (m *Mantle) fatalError(s string, a ...interface{}) {
 	m.log.Criticalf(s, a...)
-	quit()
+	if !ignoreErrors || ctx.Err() != nil {
+		quit()
+	}
 }
 
 // order places an order on the market.
@@ -407,10 +406,20 @@ func (m *Mantle) createWallet(symbol, node string, minFunds, maxFunds uint64, nu
 
 		if nCoins != 0 {
 			chunk := (maxFunds + minFunds) / 2 / uint64(nCoins)
-			for i := 0; i < nCoins; i++ {
+			for i := 0; i < nCoins; {
 				if err = send(walletSymbol, node, addr, chunk); err != nil {
+					if ignoreErrors && ctx.Err() == nil {
+						m.log.Errorf("Trouble sending %d %s to %s: %v\n Sleeping and trying again.", valString(chunk, walletSymbol), walletSymbol, addr, err)
+						// It often happens that the wallet is not able to
+						// create enough outputs. mine and try indefinitely
+						// if we are ignoring errors.
+						<-harnessCtl(ctx, walletSymbol, fmt.Sprintf("./mine-%s", node), "1")
+						time.Sleep(time.Second)
+						continue
+					}
 					return "", err
 				}
+				i++
 			}
 		}
 		<-harnessCtl(ctx, walletSymbol, fmt.Sprintf("./mine-%s", node), "1")
@@ -437,6 +446,7 @@ func (m *Mantle) createWallet(symbol, node string, minFunds, maxFunds uint64, nu
 }
 
 func send(symbol, node, addr string, val uint64) error {
+	log.Tracef("Sending %s %s from %s node to %s", valString(val, symbol), symbol, node, addr)
 	var res *harnessResult
 	switch symbol {
 	case btc, dcr, ltc, doge, firo, bch, dgb:
@@ -466,18 +476,28 @@ func send(symbol, node, addr string, val uint64) error {
 
 }
 
+type walletMinMax map[uint32]struct {
+	min, max uint64
+}
+
 // replenishBalances will run replenishBalance for all wallets.
-func (m *Mantle) replenishBalances() {
+func (m *Mantle) replenishBalances(wmm walletMinMax) {
+	if wmm != nil {
+		for k, v := range wmm {
+			m.replenishBalance(m.wallets[k], v.min, v.max)
+		}
+		return
+	}
 	for _, w := range m.wallets {
-		m.replenishBalance(w)
+		m.replenishBalance(w, w.minFunds, w.maxFunds)
 	}
 	// TODO: Check balance in parent wallets? We send them some initial funds,
 	// and maybe that's enough for our purposes, since it just covers fees.
 }
 
 // replenishBalance will bring the balance with allowable limits by requesting
 // funds from or sending funds to the wallet's node.
-func (m *Mantle) replenishBalance(w *botWallet) {
+func (m *Mantle) replenishBalance(w *botWallet, minFunds, maxFunds uint64) {
 	// Get the Balance from the user in case it changed while while this note
 	// was in the notification pipeline.
 	bal, err := m.AssetBalance(w.assetID)
@@ -487,21 +507,32 @@ func (m *Mantle) replenishBalance(w *botWallet) {
 	}
 
 	m.log.Debugf("Balance note received for %s (minFunds = %s, maxFunds = %s): %s",
-		w.symbol, valString(w.minFunds, w.symbol), valString(w.maxFunds, w.symbol), mustJSON(bal))
+		w.symbol, valString(minFunds, w.symbol), valString(maxFunds, w.symbol), mustJSON(bal))
 
 	// If over or under max, make the average of the two.
-	wantBal := (w.maxFunds + w.minFunds) / 2
+	wantBal := (maxFunds + minFunds) / 2
 
-	if bal.Available < w.minFunds {
+	if bal.Available < minFunds {
 		chunk := (wantBal - bal.Available) / uint64(w.numCoins)
-		for i := 0; i < w.numCoins; i++ {
+		for i := 0; i < w.numCoins; {
 			m.log.Debugf("Requesting %s from %s alpha node", valString(chunk, w.symbol), w.symbol)
 			if err = send(w.symbol, alpha, w.address, chunk); err != nil {
+				if ignoreErrors && ctx.Err() == nil {
+					m.log.Errorf("Trouble sending %d %s to %s: %v\n Sleeping and trying again.",
+						valString(chunk, w.symbol), w.symbol, w.address, err)
+					// It often happens that the wallet is not able to
+					// create enough outputs. mine and try indefinitely
+					// if we are ignoring errors.
+					<-harnessCtl(ctx, w.symbol, fmt.Sprintf("./mine-%s", alpha), "1")
+					time.Sleep(time.Second)
+					continue
+				}
 				m.fatalError("error refreshing balance for %s: %v", w.symbol, err)
 				return
 			}
+			i++
 		}
-	} else if bal.Available > w.maxFunds {
+	} else if bal.Available > maxFunds {
 		// Send some back to the alpha address.
 		amt := bal.Available - wantBal
 		m.log.Debugf("Sending %s back to %s alpha node", valString(amt, w.symbol), w.symbol)
@@ -563,7 +594,11 @@ func midGap(book *core.OrderBook) uint64 {
 
 // truncate rounds the provided v down to an integer-multiple of mod.
 func truncate(v, mod int64) uint64 {
-	return uint64(v - (v % mod))
+	t := uint64(v - (v % mod))
+	if t < uint64(mod) {
+		return uint64(mod)
+	}
+	return t
 }
 
 // clamp returns the closest value to v within the bounds of [min, max].