Merge branch 'main' into rp/rename-info-byte

.github/CODEOWNERS

@@ -0,0 +1,16 @@
+# CODEOWNERS: <https://help.github.com/articles/about-codeowners/>
+
+# Everything goes through the following "global owners" by default. Unless a later
+# match takes precedence, the "global owners" will be requested for review when
+# someone opens a PR. Note that the last matching pattern takes precedence, so
+# global owners are only requested if there isn't a more specific codeowner
+# specified below. For this reason, the global owners are often repeated in
+# directory-level definitions.
+
+# global owners
+* @evan-forbes
+
+# directory owners
+docs @liamsi @adlerjohn @MSevey
+x/qgb @SweeXordious
+pkg/shares @rootulp

pkg/appconsts/appconsts.go

@@ -2,6 +2,7 @@ package appconsts
 
 import (
 	"bytes"
+	"encoding/binary"
 
 	"github.com/celestiaorg/nmt/namespace"
 	"github.com/celestiaorg/rsmt2d"
@@ -115,4 +116,29 @@ var (
 	// that the next message starts at an index that conforms to non-interactive
 	// defaults.
 	NameSpacedPaddedShareBytes = bytes.Repeat([]byte{0}, SparseShareContentSize)
+
+	// FirstCompactShareDataLengthBytes is the number of bytes reserved for the total
+	// data length that is stored in the first compact share of a reserved
+	// namespace. This value is the maximum number of bytes required to store
+	// the data length of a block that only contains compact shares of one type.
+	// For example, if a block contains only evidence then it could contain:
+	// MaxSquareSize * MaxSquareSize * ShareSize bytes of evidence.
+	//
+	// Assuming MaxSquareSize is 128 and ShareSize is 256, this is 4194304 bytes
+	// of evidence. It takes 4 bytes to store a varint of 4194304.
+	//
+	// https://go.dev/play/p/MynwcDHQ_me
+	FirstCompactShareDataLengthBytes = numberOfBytesVarint(MaxSquareSize * MaxSquareSize * ShareSize)
+
+	// FirstCompactShareContentSize is the number of bytes usable for data in
+	// the first compact share of a reserved namespace. This type of share
+	// contains less space for data than a ContinuationCompactShare because the
+	// first compact share includes a total data length varint.
+	FirstCompactShareContentSize = ContinuationCompactShareContentSize - FirstCompactShareDataLengthBytes
 )
+
+// numberOfBytesVarint calculates the number of bytes needed to write a varint of n
+func numberOfBytesVarint(n uint64) (numberOfBytes int) {
+	buf := make([]byte, binary.MaxVarintLen64)
+	return binary.PutUvarint(buf, n)
+}

pkg/shares/README.md

@@ -0,0 +1,64 @@
+# Shares
+
+Package shares provides primitives for splitting block data into shares and parsing shares back into block data.
+
+## Compact vs. Sparse
+
+There are two types of shares:
+
+1. Compact shares
+1. Sparse shares
+
+Compact shares can contain data from one or more unit (transactions, intermediate state roots, evidence). Sparse shares can contain data from zero or one message. Compact shares and sparse shares are encoded differently. The motivation behind the distinction is that transactions, intermediate state roots, and evidence are expected to have small lengths so they are encoded in compact shares to minimize the number of shares needed to store them. On the other hand, messages are expected to be larger and have the desideratum that clients should be able to create proofs of message inclusion. This desiradum is infeasible if client A's message is encoded into a share with another client B's message that is unknown to A. It follows that client A's message is encoded into a share such that the contents can be determined by client A without any additional information. See [message layout rational](https://celestiaorg.github.io/celestia-specs/latest/rationale/message_block_layout.html#message-layout-rationale) or [adr-006-non-interactive-defaults.md](https://github.com/celestiaorg/celestia-app/pull/673) for additional details.
+
+## Universal Prefix
+
+Both types of shares have a universal prefix. The first 8 bytes of a share contain the [namespace.ID](https://github.com/celestiaorg/nmt/blob/master/namespace/id.go).  The next byte is an [InfoByte](./info_byte.go) that contains the share version and a message start indicator. If the message start indicator is `1` (i.e. this is the first share of a message) then the next 1-10 bytes contain a varint of the uint64 message length.
+
+> **Note**:
+> Although compact shares don't contain message data, the message start indicator is still applicable. Conceptually, it can be helpful to think of all data in a reserved namespace as part of one message. It follows that each reserved namespace has a single "first" share that has a message start indicator of `1` and subsequent continuation shares with a message start indicator of `0`.
+
+For the first share of a message:
+
+```ascii
+| universal prefix                          | data                      |
+| namespace_id | info_byte | message_length | message data              |
+| 8 bytes      | 1 byte    | 1-10 bytes     | remaining bytes of share  |
+```
+
+For continuation share of a message:
+
+```ascii
+| universal prefix         | data                      |
+| namespace_id | info_byte | message data              |
+| 8 bytes      | 1 byte    | remaining bytes of share  |
+```
+
+The remaining bytes depend on the share type.
+
+### Compact Share Schema
+
+The first byte after the universal prefix is a reserved byte that indicates the location in the share of the first unit of data that starts in this share.
+
+For the first compact share in a reserved namespace:
+
+```ascii
+| universal prefix                          | reserved byte          | data                                                |
+| namespace_id | info_byte | message_length | location_of_first_unit | transactions, intermediate state roots, or evidence |
+| 8 bytes      | 1 byte    | 1-10 bytes     | 1 byte                 | remaining bytes of share                            |
+```
+
+For continuation compact share in a reserved namespace:
+
+```ascii
+| universal prefix         | reserved byte          | data                                                |
+| namespace_id | info_byte | location_of_first_unit | transactions, intermediate state roots, or evidence |
+| 8 bytes      | 1 byte    | 1 byte                 | remaining bytes of share                            |
+```
+
+> **Note**:
+> Each unit (transaction, intermediate state root, evidence) in data is prefixed with a varint of the length of the unit.
+
+### Sparse Share Schema
+
+The remaining bytes contain message data.

pkg/shares/doc.go

@@ -0,0 +1,3 @@
+// Package shares provides primitives for splitting block data into shares and
+// parsing shares back into block data.
+package shares

pkg/shares/parse_sparse_shares.go

@@ -92,7 +92,7 @@ func ParseDelimiter(input []byte) (inputWithoutLengthDelimiter []byte, dataLengt
 		l = len(input)
 	}
 
-	delimiter := zeroPadIfNecessary(input[:l], binary.MaxVarintLen64)
+	delimiter, _ := zeroPadIfNecessary(input[:l], binary.MaxVarintLen64)
 
 	// read the length of the data
 	r := bytes.NewBuffer(delimiter)
@@ -112,15 +112,15 @@ func ParseDelimiter(input []byte) (inputWithoutLengthDelimiter []byte, dataLengt
 // zeroPadIfNecessary pads the share with trailing zero bytes if the provided
 // share has fewer bytes than width. Returns the share unmodified if the
 // len(share) is greater than or equal to width.
-func zeroPadIfNecessary(share []byte, width int) []byte {
+func zeroPadIfNecessary(share []byte, width int) (padded []byte, bytesOfPadding int) {
 	oldLen := len(share)
 	if oldLen >= width {
-		return share
+		return share, 0
 	}
 
 	missingBytes := width - oldLen
 	padByte := []byte{0}
 	padding := bytes.Repeat(padByte, missingBytes)
 	share = append(share, padding...)
-	return share
+	return share, missingBytes
 }

pkg/shares/shares_test.go

@@ -176,19 +176,24 @@ func Test_zeroPadIfNecessary(t *testing.T) {
 		width int
 	}
 	tests := []struct {
-		name string
-		args args
-		want []byte
+		name               string
+		args               args
+		wantPadded         []byte
+		wantBytesOfPadding int
 	}{
-		{"pad", args{[]byte{1, 2, 3}, 6}, []byte{1, 2, 3, 0, 0, 0}},
-		{"not necessary (equal to shareSize)", args{[]byte{1, 2, 3}, 3}, []byte{1, 2, 3}},
-		{"not necessary (greater shareSize)", args{[]byte{1, 2, 3}, 2}, []byte{1, 2, 3}},
+		{"pad", args{[]byte{1, 2, 3}, 6}, []byte{1, 2, 3, 0, 0, 0}, 3},
+		{"not necessary (equal to shareSize)", args{[]byte{1, 2, 3}, 3}, []byte{1, 2, 3}, 0},
+		{"not necessary (greater shareSize)", args{[]byte{1, 2, 3}, 2}, []byte{1, 2, 3}, 0},
 	}
 	for _, tt := range tests {
 		tt := tt // stupid scopelint :-/
 		t.Run(tt.name, func(t *testing.T) {
-			if got := zeroPadIfNecessary(tt.args.share, tt.args.width); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("zeroPadIfNecessary() = %v, want %v", got, tt.want)
+			gotPadded, gotBytesOfPadding := zeroPadIfNecessary(tt.args.share, tt.args.width)
+			if !reflect.DeepEqual(gotPadded, tt.wantPadded) {
+				t.Errorf("zeroPadIfNecessary gotPadded %v, wantPadded %v", gotPadded, tt.wantPadded)
+			}
+			if gotBytesOfPadding != tt.wantBytesOfPadding {
+				t.Errorf("zeroPadIfNecessary gotBytesOfPadding %v, wantBytesOfPadding %v", gotBytesOfPadding, tt.wantBytesOfPadding)
 			}
 		})
 	}

pkg/shares/split_compact_shares.go

@@ -132,7 +132,7 @@ func (css *CompactShareSplitter) stackPending() {
 func (css *CompactShareSplitter) Export() NamespacedShares {
 	// add the pending share to the current shares before returning
 	if len(css.pendingShare.Share) > appconsts.NamespaceSize+appconsts.ShareInfoBytes {
-		css.pendingShare.Share = zeroPadIfNecessary(css.pendingShare.Share, appconsts.ShareSize)
+		css.pendingShare.Share, _ = zeroPadIfNecessary(css.pendingShare.Share, appconsts.ShareSize)
 		css.shares = append(css.shares, css.pendingShare)
 	}
 	// force the last share to have a reserve byte of zero

pkg/shares/split_sparse_shares.go

@@ -104,7 +104,7 @@ func AppendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte)
 			byte(infoByte)),
 			rawData...,
 		)
-		paddedShare := zeroPadIfNecessary(rawShare, appconsts.ShareSize)
+		paddedShare, _ := zeroPadIfNecessary(rawShare, appconsts.ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
 	} else { // len(rawData) > MsgShareSize
@@ -151,7 +151,7 @@ func splitMessage(rawData []byte, nid namespace.ID) NamespacedShares {
 			byte(infoByte)),
 			rawData[:shareSizeOrLen]...,
 		)
-		paddedShare := zeroPadIfNecessary(rawShare, appconsts.ShareSize)
+		paddedShare, _ := zeroPadIfNecessary(rawShare, appconsts.ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
 		rawData = rawData[shareSizeOrLen:]

x/payment/types/payfordata.go

@@ -3,7 +3,6 @@ package types
 import (
 	"crypto/sha256"
 	"fmt"
-	"math/bits"
 
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
 	"github.com/celestiaorg/nmt"
@@ -232,8 +231,3 @@ func powerOf2(v uint64) bool {
 	}
 	return false
 }
-
-// DelimLen calculates the length of the delimiter for a given message size
-func DelimLen(x uint64) int {
-	return 8 - bits.LeadingZeros64(x)%8
-}

x/payment/types/payfordata_test.go

@@ -6,6 +6,7 @@ import (
 
 	sdkerrors "cosmossdk.io/errors"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	shares "github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/nmt/namespace"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/stretchr/testify/assert"
@@ -421,7 +422,7 @@ func TestValidateBasic(t *testing.T) {
 // totalMsgSize subtracts the delimiter size from the desired total size. this
 // is useful for testing for messages that occupy exactly so many shares.
 func totalMsgSize(size int) int {
-	return size - DelimLen(uint64(size))
+	return size - shares.DelimLen(uint64(size))
 }
 
 func validWirePayForData(t *testing.T) *MsgWirePayForData {

x/payment/types/square_sizes.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	shares "github.com/celestiaorg/celestia-app/pkg/shares"
 )
 
 // https://github.com/celestiaorg/celestia-app/issues/236
@@ -45,7 +46,7 @@ func AllSquareSizes(msgSize int) []uint64 {
 // It accounts for the necessary delimiter and potential padding.
 func MsgSharesUsed(msgSize int) int {
 	// add the delimiter to the message size
-	msgSize = DelimLen(uint64(msgSize)) + msgSize
+	msgSize = shares.DelimLen(uint64(msgSize)) + msgSize
 	shareCount := msgSize / appconsts.SparseShareContentSize
 	// increment the share count if the message overflows the last counted share
 	if msgSize%appconsts.SparseShareContentSize != 0 {