Add box_splice and box_resize opcodes

These opcodes simplify working with large boxes. There are simple
combinations of existing codes to do the equivalent on boxes smaller
than 4k. But they require moving box content into AVM values that
cannot exceed 4k. So, for big boxes it is useful to have these
operations. (box_splice may also be a nice convenience, even for small
boxes)

data/transactions/logic/assembler_test.go

@@ -430,7 +430,13 @@ pushbytess "1" "2" "1"
 const v8Nonsense = v7Nonsense + switchNonsense + frameNonsense + matchNonsense + boxNonsense
 
 const v9Nonsense = v8Nonsense
-const v10Nonsense = v9Nonsense + pairingNonsense
+
+const spliceNonsence = `
+  box_splice
+  box_resize
+`
+
+const v10Nonsense = v9Nonsense + pairingNonsense + spliceNonsence
 
 const v6Compiled = "2004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f2310231123122313231418191a1b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b400b53a03b6b7043cb8033a0c2349c42a9631007300810881088120978101c53a8101c6003a"
 
@@ -447,7 +453,10 @@ const matchCompiled = "83030102018e02fff500008203013101320131"
 const v8Compiled = v7Compiled + switchCompiled + frameCompiled + matchCompiled + boxCompiled
 
 const v9Compiled = v8Compiled
-const v10Compiled = v9Compiled + pairingCompiled
+
+const spliceCompiled = "d2d3"
+
+const v10Compiled = v9Compiled + pairingCompiled + spliceCompiled
 
 var nonsense = map[uint64]string{
 	1:  v1Nonsense,
@@ -527,7 +536,7 @@ func TestAssemble(t *testing.T) {
 	}
 }
 
-var experiments = []uint64{pairingVersion}
+var experiments = []uint64{pairingVersion, spliceVersion}
 
 // TestExperimental forces a conscious choice to promote "experimental" opcode
 // groups. This will fail when we increment vFuture's LogicSigVersion. If we had

data/transactions/logic/box.go

@@ -35,6 +35,8 @@ const (
 	BoxWriteOperation
 	// BoxDeleteOperation deletes a box
 	BoxDeleteOperation
+	// BoxResizeOperation resizes a box
+	BoxResizeOperation
 )
 
 func (cx *EvalContext) availableBox(name string, operation BoxOperation, createSize uint64) ([]byte, bool, error) {
@@ -81,6 +83,13 @@ func (cx *EvalContext) availableBox(name string, operation BoxOperation, createS
 			cx.available.dirtyBytes += writeSize
 		}
 		dirty = true
+	case BoxResizeOperation:
+		newSize := createSize
+		if dirty {
+			cx.available.dirtyBytes -= uint64(len(content))
+		}
+		cx.available.dirtyBytes += newSize
+		dirty = true
 	case BoxDeleteOperation:
 		if dirty {
 			cx.available.dirtyBytes -= uint64(len(content))
@@ -199,6 +208,34 @@ func opBoxReplace(cx *EvalContext) error {
 	return cx.Ledger.SetBox(cx.appID, name, bytes)
 }
 
+func opBoxSplice(cx *EvalContext) error {
+	last := len(cx.Stack) - 1 // replacement
+	replacement := cx.Stack[last].Bytes
+	length := cx.Stack[last-1].Uint
+	start := cx.Stack[last-2].Uint
+	name := string(cx.Stack[last-3].Bytes)
+
+	err := argCheck(cx, name, 0)
+	if err != nil {
+		return err
+	}
+
+	contents, exists, err := cx.availableBox(name, BoxWriteOperation, 0 /* size is already known */)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return fmt.Errorf("no such box %#x", name)
+	}
+
+	bytes, err := spliceCarefully(contents, replacement, start, length)
+	if err != nil {
+		return err
+	}
+	cx.Stack = cx.Stack[:last-3]
+	return cx.Ledger.SetBox(cx.appID, name, bytes)
+}
+
 func opBoxDel(cx *EvalContext) error {
 	last := len(cx.Stack) - 1 // name
 	name := string(cx.Stack[last].Bytes)
@@ -222,6 +259,48 @@ func opBoxDel(cx *EvalContext) error {
 	return nil
 }
 
+func opBoxResize(cx *EvalContext) error {
+	last := len(cx.Stack) - 1 // size
+	prev := last - 1          // name
+
+	name := string(cx.Stack[prev].Bytes)
+	size := cx.Stack[last].Uint
+
+	err := argCheck(cx, name, size)
+	if err != nil {
+		return err
+	}
+
+	contents, exists, err := cx.availableBox(name, BoxResizeOperation, size)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return fmt.Errorf("no such box %#x", name)
+	}
+	appAddr := cx.GetApplicationAddress(cx.appID)
+	_, err = cx.Ledger.DelBox(cx.appID, name, appAddr)
+	if err != nil {
+		return err
+	}
+	var resized []byte
+	if size > uint64(len(contents)) {
+		resized = make([]byte, size)
+		copy(resized, contents)
+	} else {
+		resized = contents[:size]
+	}
+	err = cx.Ledger.NewBox(cx.appID, name, resized, appAddr)
+	if err != nil {
+		return err
+	}
+
+	cx.Stack = cx.Stack[:prev]
+	return err
+
+}
+
 func opBoxLen(cx *EvalContext) error {
 	last := len(cx.Stack) - 1 // name
 	name := string(cx.Stack[last].Bytes)
@@ -292,3 +371,34 @@ func opBoxPut(cx *EvalContext) error {
 	appAddr := cx.GetApplicationAddress(cx.appID)
 	return cx.Ledger.NewBox(cx.appID, name, value, appAddr)
 }
+
+// spliceCarefully is used to make a NEW byteslice copy of original, with
+// replacement written over the bytes from start to start+length. Returned slice
+// is always the same size as original. Zero bytes are "shifted in" or high
+// bytes are "shifted out" as needed.
+func spliceCarefully(original []byte, replacement []byte, start uint64, olen uint64) ([]byte, error) {
+	if start > uint64(len(original)) {
+		return nil, fmt.Errorf("replacement start %d beyond length: %d", start, len(original))
+	}
+	oend := start + olen
+	if oend < start {
+		return nil, fmt.Errorf("splice end exceeds uint64")
+	}
+
+	if oend > uint64(len(original)) {
+		return nil, fmt.Errorf("splice end %d beyond original length: %d", oend, len(original))
+	}
+
+	// Do NOT use the append trick to make a copy here.
+	// append(nil, []byte{}...) would return a nil, which means "not a bytearray" to AVM.
+	clone := make([]byte, len(original))
+	copy(clone[:start], original)
+	copied := copy(clone[start:], replacement)
+	if copied != len(replacement) {
+		return nil, fmt.Errorf("splice inserted bytes too long")
+	}
+	// If original is "too short" we get zeros at the end. If original is "too
+	// long" we lose some bytes. Fortunately, that's what we want.
+	copy(clone[int(start)+copied:], original[oend:])
+	return clone, nil
+}

data/transactions/logic/box_test.go

@@ -41,11 +41,15 @@ func TestBoxNewDel(t *testing.T) {
 			ep, txn, ledger := MakeSampleEnv()
 
 			createSelf := fmt.Sprintf(`byte "self"; int %d; box_create;`, size)
+			growSelf := fmt.Sprintf(`byte "self"; int %d; box_resize; int 1`, size+5)
 			createOther := fmt.Sprintf(`byte "other"; int %d; box_create;`, size)
 
 			ledger.NewApp(txn.Sender, 888, basics.AppParams{})
 
+			TestApp(t, growSelf, ep, "no such box")
+
 			TestApp(t, createSelf, ep)
+			TestApp(t, growSelf, ep)
 			ledger.DelBoxes(888, "self")
 
 			TestApp(t, createSelf+`assert;`+createSelf+`!`, ep)
@@ -77,10 +81,13 @@ func TestBoxNewBad(t *testing.T) {
 	ledger.NewApp(txn.Sender, 888, basics.AppParams{})
 	TestApp(t, `byte "self"; int 999; box_create`, ep, "write budget")
 
-	// In test proto, you get 100 I/O budget per boxref
+	// In test proto, you get 100 I/O budget per boxref, and 1000 is the
+	// absolute biggest box.
 	ten := [10]transactions.BoxRef{}
 	txn.Boxes = append(txn.Boxes, ten[:]...) // write budget is now 11*100 = 1100
 	TestApp(t, `byte "self"; int 999; box_create`, ep)
+	TestApp(t, `byte "self"; int 1000; box_resize; int 1`, ep)
+	TestApp(t, `byte "self"; int 1001; box_resize; int 1`, ep, "box size too large")
 	ledger.DelBoxes(888, "self")
 	TestApp(t, `byte "self"; int 1000; box_create`, ep)
 	ledger.DelBoxes(888, "self")
@@ -141,6 +148,63 @@ func TestBoxReadWrite(t *testing.T) {
 		"invalid Box reference")
 }
 
+func TestBoxSplice(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	t.Parallel()
+
+	ep, txn, ledger := MakeSampleEnv()
+
+	ledger.NewApp(txn.Sender, 888, basics.AppParams{})
+	// extract some bytes until past the end, confirm the begin as zeros, and
+	// when it fails.
+	TestApp(t, `byte "self"; int 4; box_create;`, ep)
+
+	// replace two bytes with two bytes. would usually use box_replace
+	TestApp(t, `byte "self"; int 1; int 2; byte 0x5555; box_splice;
+                byte "self"; box_get; assert; byte 0x00555500; ==`, ep)
+
+	// replace first 55 with two 44s.
+	TestApp(t, `byte "self"; int 1; int 1; byte 0x4444; box_splice;
+                byte "self"; box_get; assert; byte 0x00444455; ==`, ep)
+
+	// replace second 44 with two 33s. (loses the 55)
+	TestApp(t, `byte "self"; int 2; int 1; byte 0x3333; box_splice;
+                byte "self"; box_get; assert; byte 0x00443333; ==`, ep)
+
+	// replace 0044 with 22. (shifts in a 0x00)
+	TestApp(t, `byte "self"; int 0; int 2; byte 0x22; box_splice;
+                byte "self"; box_get; assert; byte 0x22333300; ==`, ep)
+
+	// dumb: try to replace 00 with 1111, but growing is illegal
+	TestApp(t, `byte "self"; int 3; int 1; byte 0x1111; box_splice;
+                byte "self"; box_get; assert; byte 0x2233331111; ==`, ep,
+		"inserted bytes too long")
+
+	// dumber: try to replace 00__ with 1111, but placing outside bounds is illegal
+	TestApp(t, `byte "self"; int 3; int 2; byte 0x1111; box_splice;
+                byte "self"; box_get; assert; byte 0x2233331111; ==`, ep,
+		"splice end 5 beyond original length")
+
+	// try to replace AT end (fails because it would extend)
+	TestApp(t, `byte "self"; int 4; int 0; byte 0x1111; box_splice;
+                byte "self"; box_get; assert; byte 0x223333001111; ==`, ep,
+		"splice inserted bytes too long")
+
+	// so it's ok if you splice in nothing
+	TestApp(t, `byte "self"; int 4; int 0; byte 0x; box_splice;
+                byte "self"; box_get; assert; byte 0x22333300; ==`, ep)
+
+	// try to replace BEYOND end (fails no matter what)
+	TestApp(t, `byte "self"; int 5; int 0; byte 0x1111; box_splice;
+                byte "self"; box_get; assert; byte 0x22333300001111; ==`, ep,
+		"replacement start 5 beyond length")
+
+	// even doing nothing is illegal beyond the end
+	TestApp(t, `byte "self"; int 5; int 0; byte 0x; box_splice;
+                byte "self"; box_get; assert; byte 0x22333300; ==`, ep,
+		"replacement start 5 beyond length")
+}
+
 func TestBoxAcrossTxns(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	t.Parallel()
@@ -167,22 +231,37 @@ func TestDirtyTracking(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	t.Parallel()
 
-	ep, txn, ledger := MakeSampleEnv()
+	ep, txn, ledger := MakeSampleEnv() // has two box refs, "self", "other" = 200 budget
 
 	ledger.NewApp(txn.Sender, 888, basics.AppParams{})
 	TestApp(t, `byte "self"; int 200; box_create`, ep)
+	TestApp(t, `byte "self"; int 201; box_resize; int 1`, ep, "write budget")
 	TestApp(t, `byte "other"; int 201; box_create`, ep, "write budget")
 	// deleting "self" doesn't give extra write budget to create big "other"
-	TestApp(t, `byte "self"; box_del; !; byte "other"; int 201; box_create`, ep,
+	TestApp(t, `byte "self"; box_del; assert; byte "other"; int 201; box_create`, ep,
 		"write budget")
 
 	// though it cancels out a creation that happened here
 	TestApp(t, `byte "self"; int 200; box_create; assert
                       byte "self"; box_del; assert
-                      byte "self"; int 200; box_create;
+                      byte "other"; int 200; box_create;
+                     `, ep)
+	ledger.DelBoxes(888, "self", "other")
+
+	// create 200, but shrink it, then the write budget frees up
+	TestApp(t, `byte "self"; int 200; box_create; assert
+                      byte "self"; int 150; box_resize;
+                      byte "other"; int 50; box_create;
                      `, ep)
+	ledger.DelBoxes(888, "self", "other")
 
+	// confirm that the exactly right amount freed up
+	TestApp(t, `byte "self"; int 200; box_create; assert
+                      byte "self"; int 150; box_resize;
+                      byte "other"; int 51; box_create;
+                     `, ep, "write budget")
 	ledger.DelBoxes(888, "self", "other")
+
 	// same, but create a different box than deleted
 	TestApp(t, `byte "self"; int 200; box_create; assert
                       byte "self"; box_del; assert
@@ -217,6 +296,7 @@ func TestBoxUnavailableWithClearState(t *testing.T) {
 		"box_len":     `byte "self"; box_len`,
 		"box_put":     `byte "put"; byte "self"; box_put`,
 		"box_replace": `byte "self"; int 0; byte "new"; box_replace`,
+		"box_resize":  `byte "self"; int 10; box_resize`,
 	}
 
 	for name, program := range tests {

data/transactions/logic/doc.go

@@ -289,10 +289,12 @@ var opDescByName = map[string]OpDesc{
 	"box_create":  {"create a box named A, of length B. Fail if A is empty or B exceeds 32,768. Returns 0 if A already existed, else 1", "Newly created boxes are filled with 0 bytes. `box_create` will fail if the referenced box already exists with a different size. Otherwise, existing boxes are unchanged by `box_create`.", nil},
 	"box_extract": {"read C bytes from box A, starting at offset B. Fail if A does not exist, or the byte range is outside A's size.", "", nil},
 	"box_replace": {"write byte-array C into box A, starting at offset B. Fail if A does not exist, or the byte range is outside A's size.", "", nil},
+	"box_splice":  {"set box A to contain its previous bytes up to index B, followed by D, followed by the original bytes of A that began at index B+C.", "Boxes are of constant length. If C < len(D), then len(D)-C bytes will be removed from the end. If C > len(D), zero bytes will be appended to the end to reach the box length.", nil},
 	"box_del":     {"delete box named A if it exists. Return 1 if A existed, 0 otherwise", "", nil},
 	"box_len":     {"X is the length of box A if A exists, else 0. Y is 1 if A exists, else 0.", "", nil},
 	"box_get":     {"X is the contents of box A if A exists, else ''. Y is 1 if A exists, else 0.", "For boxes that exceed 4,096 bytes, consider `box_create`, `box_extract`, and `box_replace`", nil},
 	"box_put":     {"replaces the contents of box A with byte-array B. Fails if A exists and len(B) != len(box A). Creates A if it does not exist", "For boxes that exceed 4,096 bytes, consider `box_create`, `box_extract`, and `box_replace`", nil},
+	"box_resize":  {"change the size of box A to be of length B, adding zero bytes to end or removing bytes from the end, as needed. Fail if A is empty, is not an existing box, or B exceeds 32,768.", "", nil},
 }
 
 // OpDoc returns a description of the op
@@ -350,7 +352,7 @@ var OpGroups = map[string][]string{
 	"Loading Values":          {"intcblock", "intc", "intc_0", "intc_1", "intc_2", "intc_3", "pushint", "pushints", "bytecblock", "bytec", "bytec_0", "bytec_1", "bytec_2", "bytec_3", "pushbytes", "pushbytess", "bzero", "arg", "arg_0", "arg_1", "arg_2", "arg_3", "args", "txn", "gtxn", "txna", "txnas", "gtxna", "gtxnas", "gtxns", "gtxnsa", "gtxnsas", "global", "load", "loads", "store", "stores", "gload", "gloads", "gloadss", "gaid", "gaids"},
 	"Flow Control":            {"err", "bnz", "bz", "b", "return", "pop", "popn", "dup", "dup2", "dupn", "dig", "bury", "cover", "uncover", "frame_dig", "frame_bury", "swap", "select", "assert", "callsub", "proto", "retsub", "switch", "match"},
 	"State Access":            {"balance", "min_balance", "app_opted_in", "app_local_get", "app_local_get_ex", "app_global_get", "app_global_get_ex", "app_local_put", "app_global_put", "app_local_del", "app_global_del", "asset_holding_get", "asset_params_get", "app_params_get", "acct_params_get", "log", "block"},
-	"Box Access":              {"box_create", "box_extract", "box_replace", "box_del", "box_len", "box_get", "box_put"},
+	"Box Access":              {"box_create", "box_extract", "box_replace", "box_splice", "box_del", "box_len", "box_get", "box_put", "box_resize"},
 	"Inner Transactions":      {"itxn_begin", "itxn_next", "itxn_field", "itxn_submit", "itxn", "itxna", "itxnas", "gitxn", "gitxna", "gitxnas"},
 }
 

data/transactions/logic/eval.go

@@ -4227,7 +4227,7 @@ func replaceCarefully(original []byte, replacement []byte, start uint64) ([]byte
 		return nil, fmt.Errorf("replacement start %d beyond length: %d", start, len(original))
 	}
 	end := start + uint64(len(replacement))
-	if end < start { // impossible because it is sum of two avm value lengths
+	if end < start { // impossible because it is sum of two avm value (or box) lengths
 		return nil, fmt.Errorf("replacement end exceeds uint64")
 	}
 

data/transactions/logic/opcodeExplain.go

@@ -197,6 +197,12 @@ func opBoxReplaceStateChange(cx *EvalContext) (AppStateEnum, AppStateOpEnum, bas
 	return BoxState, AppStateWrite, cx.appID, basics.Address{}, string(cx.Stack[pprev].Bytes)
 }
 
+func opBoxSpliceStateChange(cx *EvalContext) (AppStateEnum, AppStateOpEnum, basics.AppIndex, basics.Address, string) {
+	name := len(cx.Stack) - 4 // name, start, length, replacement
+
+	return BoxState, AppStateWrite, cx.appID, basics.Address{}, string(cx.Stack[name].Bytes)
+}
+
 func opBoxDelStateChange(cx *EvalContext) (AppStateEnum, AppStateOpEnum, basics.AppIndex, basics.Address, string) {
 	last := len(cx.Stack) - 1 // name
 
@@ -210,6 +216,12 @@ func opBoxPutStateChange(cx *EvalContext) (AppStateEnum, AppStateOpEnum, basics.
 	return BoxState, AppStateWrite, cx.appID, basics.Address{}, string(cx.Stack[prev].Bytes)
 }
 
+func opBoxResizeStateChange(cx *EvalContext) (AppStateEnum, AppStateOpEnum, basics.AppIndex, basics.Address, string) {
+	name := len(cx.Stack) - 2 // name, size
+
+	return BoxState, AppStateWrite, cx.appID, basics.Address{}, string(cx.Stack[name].Bytes)
+}
+
 func opAppLocalGetStateChange(cx *EvalContext) (AppStateEnum, AppStateOpEnum, basics.AppIndex, basics.Address, string) {
 	last := len(cx.Stack) - 1 // state key
 	prev := last - 1          // account

data/transactions/logic/opcodes.go

@@ -75,6 +75,7 @@ const sharedResourcesVersion = 9 // apps can access resources from other transac
 // moved from vFuture to a new consensus version. If they remain unready, bump
 // their version, and fixup TestAssemble() in assembler_test.go.
 const pairingVersion = 10 // bn256 opcodes. will add bls12-381, and unify the available opcodes.
+const spliceVersion = 10  // box splicing/resizing
 
 // Unlimited Global Storage opcodes
 const boxVersion = 8 // box_*
@@ -718,9 +719,10 @@ var OpSpecs = []OpSpec{
 	{0xc5, "itxnas", opItxnas, proto("i:a"), 6, field("f", &TxnArrayFields).only(ModeApp)},
 	{0xc6, "gitxnas", opGitxnas, proto("i:a"), 6, immediates("t", "f").field("f", &TxnArrayFields).only(ModeApp)},
 
-	// randomness support
 	{0xd0, "vrf_verify", opVrfVerify, proto("bbb:bT"), randomnessVersion, field("s", &VrfStandards).costs(5700)},
 	{0xd1, "block", opBlock, proto("i:a"), randomnessVersion, field("f", &BlockFields)},
+	{0xd2, "box_splice", opBoxSplice, proto("Niib:").appStateExplain(opBoxSpliceStateChange), spliceVersion, only(ModeApp)},
+	{0xd3, "box_resize", opBoxResize, proto("Ni:").appStateExplain(opBoxResizeStateChange), spliceVersion, only(ModeApp)},
 
 	{0xe0, "ec_add", opEcAdd, proto("bb:b"), pairingVersion,
 		costByField("g", &EcGroups, []int{