mixclient: Respect standard tx size limits

In order to meet the standard tx size requirements on mainnet, peers who
contribute too much to the coinjoin transaction size must be excluded from the
current run before the pairing is finalized.  Failing to do so will only
result in a transaction that fails to publish if all peers continue the mix to
completion.

Exclusion is performed by iterating all current PRs in the random order
determined by the set of all PRs and the current epoch.  A fake coinjoin
transaction which is only meant to measure the expected tx size is added to
only if that peer does not overcontribute to the size.  The random peer order
is sufficient to prevent this algorithm from being gamed.

Any peers that become excluded will attempt a mix again in the following
epoch.  While it could be possible, we do not currently run multiple mixes for
the same pairing type by splitting the PR set into two or more sets.

mixing/mixclient/client.go

@@ -882,6 +882,29 @@ func (c *Client) pairSession(ctx context.Context, ps *pairedSessions, prs []*wir
 		c.testHook(hookBeforeRun, currentRun, nil)
 		err := c.run(ctx, ps, &madePairing)
 
+		var sizeLimitedErr *sizeLimited
+		if errors.As(err, &sizeLimitedErr) {
+			if len(sizeLimitedErr.prs) < MinPeers {
+				sesLog.logf("Aborting session with too few remaining peers")
+				return
+			}
+
+			d.shift()
+
+			sesLog.logf("Recreating as session %x due to standard tx size limits (pairid=%x)",
+				sizeLimitedErr.sid[:], ps.pairing)
+
+			rerun = &sessionRun{
+				sid:       sizeLimitedErr.sid,
+				run:       0,
+				prs:       sizeLimitedErr.prs,
+				freshGen:  false,
+				prngRun:   0,
+				deadlines: d,
+			}
+			continue
+		}
+
 		var altses *alternateSession
 		if errors.As(err, &altses) {
 			if altses.err != nil {
@@ -1132,6 +1155,73 @@ func (c *Client) run(ctx context.Context, ps *pairedSessions, madePairing *bool)
 		}
 	}
 
+	// Before confirming the pairing in run-0, check all of the
+	// agreed-upon PRs that they will not result in a coinjoin transaction
+	// that exceeds the standard size limits.
+	//
+	// PRs are randomly ordered in each epoch based on the session ID, so
+	// they can be iterated in order to discover any PR that would
+	// increase the final coinjoin size above the limits.
+	if run == 0 {
+		var sizeExcluded []*wire.MsgMixPairReq
+
+		cj := wire.NewMsgTx()
+		prUnmixed := wire.NewMsgTx()
+
+		prevScriptStake := make([]byte, 26)
+		prevScriptReg := prevScriptStake[:25]
+		var op wire.OutPoint
+
+		for _, pr := range sesRun.prs {
+			prUnmixed.TxIn = prUnmixed.TxIn[:0]
+			prUnmixed.TxOut = prUnmixed.TxOut[:0]
+
+			for _, utxo := range pr.UTXOs {
+				switch utxo.Opcode {
+				case 0:
+					prUnmixed.AddTxIn(wire.NewTxIn(&op, 0, prevScriptReg))
+				case txscript.OP_SSGEN, txscript.OP_SSRTX, txscript.OP_TGEN:
+					prUnmixed.AddTxIn(wire.NewTxIn(&op, 0, prevScriptStake))
+				}
+			}
+			if pr.Change != nil {
+				prUnmixed.TxOut = append(prUnmixed.TxOut, pr.Change)
+			}
+
+			err := checkLimited(cj, prUnmixed, pr.MessageCount)
+			if err == nil {
+				mergeTx(cj, prUnmixed)
+			} else {
+				sizeExcluded = append(sizeExcluded, pr)
+			}
+		}
+
+		if len(sizeExcluded) > 0 {
+			// sizeExcluded and sesRun.prs are in the same order;
+			// can zip down both to create the slice of peers that
+			// can continue the mix.
+			excl := sizeExcluded
+			kept := make([]*wire.MsgMixPairReq, 0, len(sesRun.prs))
+			for _, pr := range sesRun.prs {
+				if pr != excl[0] {
+					kept = append(kept, pr)
+					continue
+				}
+				excl = excl[1:]
+				if len(excl) == 0 {
+					break
+				}
+			}
+
+			sid := mixing.SortPRsForSession(kept, unixEpoch)
+			return &sizeLimited{
+				prs:      kept,
+				sid:      sid,
+				excluded: sizeExcluded,
+			}
+		}
+	}
+
 	for _, ke := range kes {
 		if idx, ok := identityIndices[ke.Identity]; ok {
 			sesRun.peers[idx].ke = ke
@@ -1703,6 +1793,16 @@ prs:
 	return blamed
 }
 
+type sizeLimited struct {
+	prs      []*wire.MsgMixPairReq
+	sid      [32]byte
+	excluded []*wire.MsgMixPairReq
+}
+
+func (e *sizeLimited) Error() string {
+	return "mix session coinjoin exceeds standard size limits"
+}
+
 type alternateSession struct {
 	prs          []*wire.MsgMixPairReq
 	sid          [32]byte

mixing/mixclient/limits.go

@@ -6,13 +6,11 @@ import (
 	"github.com/decred/dcrd/wire"
 )
 
-// nolint: unused
 const (
 	redeemP2PKHv0SigScriptSize = 1 + 73 + 1 + 33
 	p2pkhv0PkScriptSize        = 1 + 1 + 1 + 20 + 1 + 1
 )
 
-// nolint: unused
 func estimateP2PKHv0SerializeSize(inputs, outputs int, hasChange bool) int {
 	// Sum the estimated sizes of the inputs and outputs.
 	txInsSize := inputs * estimateInputSize(redeemP2PKHv0SigScriptSize)
@@ -31,8 +29,6 @@ func estimateP2PKHv0SerializeSize(inputs, outputs int, hasChange bool) int {
 }
 
 // estimateInputSize returns the worst case serialize size estimate for a tx input
-//
-// nolint: unused
 func estimateInputSize(scriptSize int) int {
 	return 32 + // previous tx
 		4 + // output index
@@ -46,16 +42,13 @@ func estimateInputSize(scriptSize int) int {
 }
 
 // estimateOutputSize returns the worst case serialize size estimate for a tx output
-//
-// nolint: unused
 func estimateOutputSize(scriptSize int) int {
 	return 8 + // previous tx
 		2 + // version
 		wire.VarIntSerializeSize(uint64(scriptSize)) + // size of script
 		scriptSize // script itself
 }
 
-// nolint: unused
 func estimateIsStandardSize(inputs, outputs int) bool {
 	const maxSize = 100000
 
@@ -68,13 +61,20 @@ func estimateIsStandardSize(inputs, outputs int) bool {
 // exceed the maximum allowed size.  Peers must be excluded from mixes if
 // their contributions would cause the total transaction size to be too large,
 // even if they have not acted maliciously in the mixing protocol.
-//
-// nolint: unused
-func checkLimited(currentTx, unmixed *wire.MsgTx, totalMessages int) error {
+func checkLimited(currentTx, unmixed *wire.MsgTx, totalMessages uint32) error {
 	totalInputs := len(currentTx.TxIn) + len(unmixed.TxIn)
-	totalOutputs := len(currentTx.TxOut) + len(unmixed.TxOut) + totalMessages
+	totalOutputs := len(currentTx.TxOut) + len(unmixed.TxOut) + int(totalMessages)
 	if !estimateIsStandardSize(totalInputs, totalOutputs) {
 		return errors.New("tx size would exceed standardness rules")
 	}
 	return nil
 }
+
+func mergeTx(currentTx, unmixed *wire.MsgTx) {
+	for _, in := range unmixed.TxIn {
+		currentTx.AddTxIn(in)
+	}
+	for _, out := range unmixed.TxOut {
+		currentTx.AddTxOut(out)
+	}
+}