[syncer/storage] Pre-store Blocks

Makefile

@@ -38,8 +38,7 @@ lint-examples:
 	golangci-lint run -v -E ${LINT_SETTINGS}
 
 lint: | lint-examples
-	golangci-lint run --timeout 2m0s -v -E ${LINT_SETTINGS},gomnd && \
-	make check-comments;
+	golangci-lint run --timeout 2m0s -v -E ${LINT_SETTINGS},gomnd
 
 format:
 	gofmt -s -w -l .

statefulsyncer/stateful_syncer.go

@@ -219,6 +219,21 @@ func (s *StatefulSyncer) Prune(ctx context.Context, helper PruneHelper) error {
 	return ctx.Err()
 }
 
+// BlockSeen is called by the syncer when a block is seen.
+func (s *StatefulSyncer) BlockSeen(ctx context.Context, block *types.Block) error {
+	err := s.blockStorage.SeeBlock(ctx, block)
+	if err != nil {
+		return fmt.Errorf(
+			"%w: unable to encounter block to storage %s:%d",
+			err,
+			block.BlockIdentifier.Hash,
+			block.BlockIdentifier.Index,
+		)
+	}
+
+	return nil
+}
+
 // BlockAdded is called by the syncer when a block is added.
 func (s *StatefulSyncer) BlockAdded(ctx context.Context, block *types.Block) error {
 	err := s.blockStorage.AddBlock(ctx, block)

storage/encoder/encoder.go

@@ -221,22 +221,6 @@ const (
 	unicodeRecordSeparator = '\u001E'
 )
 
-// Indexes of encoded AccountCoin struct
-const (
-	accountAddress = iota
-	coinIdentifier
-	amountValue
-	amountCurrencySymbol
-	amountCurrencyDecimals
-
-	// If none exist below, we stop after amount.
-	accountMetadata
-	subAccountAddress
-	subAccountMetadata
-	amountMetadata
-	currencyMetadata
-)
-
 func (e *Encoder) encodeAndWrite(output *bytes.Buffer, object interface{}) error {
 	buf := e.pool.Get()
 	err := getEncoder(buf).Encode(object)
@@ -368,6 +352,22 @@ func (e *Encoder) DecodeAccountCoin( // nolint:gocognit
 	accountCoin *types.AccountCoin,
 	reclaimInput bool,
 ) error {
+	// Indices of encoded AccountCoin struct
+	const (
+		accountAddress = iota
+		coinIdentifier
+		amountValue
+		amountCurrencySymbol
+		amountCurrencyDecimals
+
+		// If none exist below, we stop after amount.
+		accountMetadata
+		subAccountAddress
+		subAccountMetadata
+		amountMetadata
+		currencyMetadata
+	)
+
 	count := 0
 	currentBytes := b
 	for {
@@ -475,3 +475,192 @@ func (e *Encoder) DecodeAccountCoin( // nolint:gocognit
 
 	return nil
 }
+
+// EncodeAccountCurrency is used to encode an AccountCurrency using the scheme (on the happy path):
+// accountAddress|currencySymbol|currencyDecimals
+//
+// And the following scheme on the unhappy path:
+// accountAddress|currencySymbol|currencyDecimals|accountMetadata|
+// subAccountAddress|subAccountMetadata|currencyMetadata
+//
+// In both cases, the | character is represented by the unicodeRecordSeparator rune.
+func (e *Encoder) EncodeAccountCurrency( // nolint:gocognit
+	accountCurrency *types.AccountCurrency,
+) ([]byte, error) {
+	output := e.pool.Get()
+	if _, err := output.WriteString(accountCurrency.Account.Address); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+	if _, err := output.WriteRune(unicodeRecordSeparator); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+	if _, err := output.WriteString(accountCurrency.Currency.Symbol); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+	if _, err := output.WriteRune(unicodeRecordSeparator); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+	if _, err := output.WriteString(
+		strconv.FormatInt(int64(accountCurrency.Currency.Decimals), 10),
+	); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+
+	// Exit early if we don't have any complex data to record (this helps
+	// us save a lot of space on the happy path).
+	if accountCurrency.Account.Metadata == nil &&
+		accountCurrency.Account.SubAccount == nil &&
+		accountCurrency.Currency.Metadata == nil {
+		return output.Bytes(), nil
+	}
+
+	if _, err := output.WriteRune(unicodeRecordSeparator); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+	if accountCurrency.Account.Metadata != nil {
+		if err := e.encodeAndWrite(output, accountCurrency.Account.Metadata); err != nil {
+			return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+		}
+	}
+	if _, err := output.WriteRune(unicodeRecordSeparator); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+
+	if accountCurrency.Account.SubAccount != nil {
+		if _, err := output.WriteString(accountCurrency.Account.SubAccount.Address); err != nil {
+			return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+		}
+	}
+	if _, err := output.WriteRune(unicodeRecordSeparator); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+
+	if accountCurrency.Account.SubAccount != nil &&
+		accountCurrency.Account.SubAccount.Metadata != nil {
+		if err := e.encodeAndWrite(output, accountCurrency.Account.SubAccount.Metadata); err != nil {
+			return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+		}
+	}
+	if _, err := output.WriteRune(unicodeRecordSeparator); err != nil {
+		return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+	}
+
+	if accountCurrency.Currency.Metadata != nil {
+		if err := e.encodeAndWrite(output, accountCurrency.Currency.Metadata); err != nil {
+			return nil, fmt.Errorf("%w: %s", errors.ErrObjectEncodeFailed, err.Error())
+		}
+	}
+
+	return output.Bytes(), nil
+}
+
+// DecodeAccountCurrency decodes an AccountCurrency and optionally
+// reclaims the memory associated with the input.
+func (e *Encoder) DecodeAccountCurrency( // nolint:gocognit
+	b []byte,
+	accountCurrency *types.AccountCurrency,
+	reclaimInput bool,
+) error {
+	// Indices of encoded AccountCurrency struct
+	const (
+		accountAddress = iota
+		currencySymbol
+		currencyDecimals
+
+		// If none exist below, we stop after amount.
+		accountMetadata
+		subAccountAddress
+		subAccountMetadata
+		currencyMetadata
+	)
+
+	count := 0
+	currentBytes := b
+	for {
+		nextRune := bytes.IndexRune(currentBytes, unicodeRecordSeparator)
+		if nextRune == -1 {
+			if count != currencyDecimals && count != currencyMetadata {
+				return fmt.Errorf("%w: next rune is -1 at %d", errors.ErrRawDecodeFailed, count)
+			}
+
+			nextRune = len(currentBytes)
+		}
+
+		val := currentBytes[:nextRune]
+		if len(val) == 0 {
+			goto handleNext
+		}
+
+		switch count {
+		case accountAddress:
+			accountCurrency.Account = &types.AccountIdentifier{
+				Address: string(val),
+			}
+		case currencySymbol:
+			accountCurrency.Currency = &types.Currency{
+				Symbol: string(val),
+			}
+		case currencyDecimals:
+			i, err := strconv.ParseInt(string(val), 10, 32)
+			if err != nil {
+				return fmt.Errorf("%w: %s", errors.ErrRawDecodeFailed, err.Error())
+			}
+
+			accountCurrency.Currency.Decimals = int32(i)
+		case accountMetadata:
+			m, err := e.decodeMap(val)
+			if err != nil {
+				return fmt.Errorf("%w: account metadata %s", errors.ErrRawDecodeFailed, err.Error())
+			}
+
+			accountCurrency.Account.Metadata = m
+		case subAccountAddress:
+			accountCurrency.Account.SubAccount = &types.SubAccountIdentifier{
+				Address: string(val),
+			}
+		case subAccountMetadata:
+			if accountCurrency.Account.SubAccount == nil {
+				return errors.ErrRawDecodeFailed // must have address
+			}
+
+			m, err := e.decodeMap(val)
+			if err != nil {
+				return fmt.Errorf(
+					"%w: subaccount metadata %s",
+					errors.ErrRawDecodeFailed,
+					err.Error(),
+				)
+			}
+
+			accountCurrency.Account.SubAccount.Metadata = m
+		case currencyMetadata:
+			m, err := e.decodeMap(val)
+			if err != nil {
+				return fmt.Errorf(
+					"%w: currency metadata %s",
+					errors.ErrRawDecodeFailed,
+					err.Error(),
+				)
+			}
+
+			accountCurrency.Currency.Metadata = m
+		default:
+			return fmt.Errorf("%w: count %d > end", errors.ErrRawDecodeFailed, count)
+		}
+
+	handleNext:
+		if nextRune == len(currentBytes) &&
+			(count == currencyDecimals || count == currencyMetadata) {
+			break
+		}
+
+		currentBytes = currentBytes[nextRune+1:]
+		count++
+	}
+
+	if reclaimInput {
+		e.pool.PutByteSlice(b)
+	}
+
+	return nil
+}

storage/encoder/encoder_test.go

@@ -257,3 +257,75 @@ func TestEncodeDecodeAccountCoin(t *testing.T) {
 		})
 	}
 }
+
+func TestEncodeDecodeAccountCurrency(t *testing.T) {
+	tests := map[string]struct {
+		accountCurrency *types.AccountCurrency
+	}{
+		"simple": {
+			accountCurrency: &types.AccountCurrency{
+				Account: &types.AccountIdentifier{
+					Address: "hello",
+				},
+				Currency: &types.Currency{
+					Symbol:   "BTC",
+					Decimals: 8,
+				},
+			},
+		},
+		"sub account info": {
+			accountCurrency: &types.AccountCurrency{
+				Account: &types.AccountIdentifier{
+					Address: "hello",
+					SubAccount: &types.SubAccountIdentifier{
+						Address: "sub",
+						Metadata: map[string]interface{}{
+							"test": "stuff",
+						},
+					},
+				},
+				Currency: &types.Currency{
+					Symbol:   "BTC",
+					Decimals: 8,
+				},
+			},
+		},
+		"currency metadata": {
+			accountCurrency: &types.AccountCurrency{
+				Account: &types.AccountIdentifier{
+					Address: "hello",
+				},
+				Currency: &types.Currency{
+					Symbol:   "BTC",
+					Decimals: 8,
+					Metadata: map[string]interface{}{
+						"issuer": "satoshi",
+					},
+				},
+			},
+		},
+	}
+
+	for name, test := range tests {
+		e, err := NewEncoder(nil, NewBufferPool(), true)
+		assert.NoError(t, err)
+
+		t.Run(name, func(t *testing.T) {
+			standardResult, err := e.Encode("", test.accountCurrency)
+			assert.NoError(t, err)
+			optimizedResult, err := e.EncodeAccountCurrency(test.accountCurrency)
+			assert.NoError(t, err)
+			fmt.Printf(
+				"Uncompressed: %d, Standard Compressed: %d, Optimized: %d\n",
+				len(types.PrintStruct(test.accountCurrency)),
+				len(standardResult),
+				len(optimizedResult),
+			)
+
+			var decoded types.AccountCurrency
+			assert.NoError(t, e.DecodeAccountCurrency(optimizedResult, &decoded, true))
+
+			assert.Equal(t, test.accountCurrency, &decoded)
+		})
+	}
+}