Skip to content

Commit

Permalink
satellite/metabase: use custom spanner encoding/decoding
Browse files Browse the repository at this point in the history 
Use it with types like storj.NodeID, storj.StreamID, storj.PieceID or
metabase.SegmentPosition.

Additionally some DB scans were simplified.

Change-Id: If6644a43ce292868bc36e638c33dc220ee8e087a
  • Loading branch information
@mniewrzal
mniewrzal authored and Storj Robot committed Apr 23, 2024
1 parent e9fed23 commit 4e7a3c2
Show file tree
Hide file tree
Showing 9 changed files with 44 additions and 52 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Expand Up @@ -62,7 +62,7 @@ require (
google.golang.org/grpc v1.62.0
gopkg.in/segmentio/analytics-go.v3 v3.1.0
gopkg.in/yaml.v3 v3.0.1
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5
storj.io/common v0.0.0-20240423093537-3f87f2a59098
storj.io/drpc v0.0.34
storj.io/eventkit v0.0.0-20240306141230-6cb545e5f892
storj.io/monkit-jaeger v0.0.0-20240221095020-52b0792fa6cd
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Expand Up @@ -901,8 +901,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
storj.io/common v0.0.0-20220719163320-cd2ef8e1b9b0/go.mod h1:mCYV6Ud5+cdbuaxdPD5Zht/HYaIn0sffnnws9ErkrMQ=
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5 h1:WyM77KVNO9+Q6Ox82iyEzYeGMAaAfiDedTIQLwOirTU=
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5/go.mod h1:MFl009RHY4tIqySVNy/6EmgRw2q60d26h9N/nb7JxGU=
storj.io/common v0.0.0-20240423093537-3f87f2a59098 h1:Xh0fk7anHRsrb9aYXro4SNas/FD0/+kEc3g9w9nkr3s=
storj.io/common v0.0.0-20240423093537-3f87f2a59098/go.mod h1:MFl009RHY4tIqySVNy/6EmgRw2q60d26h9N/nb7JxGU=
storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
storj.io/drpc v0.0.34 h1:q9zlQKfJ5A7x8NQNFk8x7eKUF78FMhmAbZLnFK+og7I=
storj.io/drpc v0.0.34/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
Expand Down
8 changes: 2 additions & 6 deletions satellite/metabase/alias.go
View file
Expand Up @@ -177,18 +177,14 @@ func (s *SpannerAdapter) ListNodeAliases(ctx context.Context) (aliases []NodeAli
return nil, Error.Wrap(err)
}

var nodeID []byte
var nodeID storj.NodeID
var nodeAlias int64
if err := row.Columns(&nodeID, &nodeAlias); err != nil {
return nil, Error.New("ListNodeAliases scan failed: %w", err)
}

id, err := storj.NodeIDFromBytes(nodeID)
if err != nil {
return nil, Error.Wrap(err)
}
aliases = append(aliases, NodeAliasEntry{
ID: id,
ID: nodeID,
Alias: NodeAlias(nodeAlias),
})
}
Expand Down
22 changes: 22 additions & 0 deletions satellite/metabase/common.go
View file
Expand Up @@ -373,6 +373,28 @@ func (pos SegmentPosition) Encode() uint64 { return uint64(pos.Part)<<32 | uint6
// Less returns whether pos should before b.
func (pos SegmentPosition) Less(b SegmentPosition) bool { return pos.Encode() < b.Encode() }

// DecodeSpanner implements spanner.Decoder.
func (pos *SegmentPosition) DecodeSpanner(val any) (err error) {
switch value := val.(type) {
case int64:
*pos = SegmentPositionFromEncoded(uint64(value))
case string:
parsedValue, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return Error.New("unable to scan %T into SegmentPosition: %v", val, err)
}
*pos = SegmentPositionFromEncoded(uint64(parsedValue))
default:
return Error.New("unable to scan %T into SegmentPosition", val)
}
return nil
}

// EncodeSpanner implements spanner.Encoder.
func (pos SegmentPosition) EncodeSpanner() (any, error) {
return int64(pos.Encode()), nil
}

// Version is used to uniquely identify objects with the same key.
type Version int64

Expand Down
48 changes: 11 additions & 37 deletions satellite/metabase/raw.go
View file
Expand Up @@ -400,49 +400,23 @@ func (s *SpannerAdapter) TestingGetAllSegments(ctx context.Context, aliasCache *
}

var segment RawSegment

var position int64
var createdAt time.Time
var repairedAt, expiresAt spanner.NullTime
var encryptedSize, plainOffset, plainSize, placement int64
var streamID, rootPieceID, encryptedKeyNonce, encryptedKey, encryptedETag, inlineData []byte
var encryptedSize, plainSize, placement int64
var aliasPieces AliasPieces
// TODO(spanner) potentially we could use row.ToStruct but we would need to add AliasPieces to RawSegment
if err := row.Columns(
&streamID, &position,
&createdAt, &repairedAt, &expiresAt,
&rootPieceID, &encryptedKeyNonce, &encryptedKey,
&encryptedSize, &plainOffset, &plainSize,
&encryptedETag,
&segment.StreamID, &segment.Position,
&segment.CreatedAt, &segment.RepairedAt, &segment.ExpiresAt,
&segment.RootPieceID, &segment.EncryptedKeyNonce, &segment.EncryptedKey,
&encryptedSize, &segment.PlainOffset, &plainSize,
&segment.EncryptedETag,
redundancyScheme{&segment.Redundancy},
&inlineData, &aliasPieces,
&segment.InlineData, &aliasPieces,
&placement,
); err != nil {
return nil, Error.Wrap(err)
}

segment.StreamID, err = uuid.FromBytes(streamID)
if err != nil {
return nil, Error.Wrap(err)
}
segment.Position = SegmentPositionFromEncoded(uint64(position))
segment.CreatedAt = createdAt
segment.RootPieceID, err = storj.PieceIDFromBytes(rootPieceID)
if err != nil {
return nil, Error.Wrap(err)
}
segment.EncryptedKeyNonce = encryptedKeyNonce
segment.EncryptedKey = encryptedKey
if repairedAt.Valid {
segment.RepairedAt = &repairedAt.Time
}
if expiresAt.Valid {
segment.ExpiresAt = &expiresAt.Time
}
segment.EncryptedSize = int32(encryptedSize)
segment.PlainOffset = plainOffset
segment.PlainSize = int32(plainSize)
segment.EncryptedETag = encryptedETag
segment.InlineData = inlineData
segment.Placement = storj.PlacementConstraint(placement)

segment.Pieces, err = aliasCache.ConvertAliasesToPieces(ctx, aliasPieces)
Expand Down Expand Up @@ -594,14 +568,14 @@ func (s *SpannerAdapter) TestingBatchInsertSegments(ctx context.Context, aliasCa

// TODO(spanner) verify if casting is good
vals := append([]interface{}{},
segment.StreamID.Bytes(),
int64(segment.Position.Encode()),
segment.StreamID,
segment.Position,

segment.CreatedAt,
segment.RepairedAt,
segment.ExpiresAt,

segment.RootPieceID.Bytes(),
segment.RootPieceID,
segment.EncryptedKeyNonce,
segment.EncryptedKey,
segment.EncryptedETag,
Expand Down
2 changes: 1 addition & 1 deletion testsuite/playwright-ui/go.mod
View file
Expand Up @@ -10,7 +10,7 @@ require (
github.com/stretchr/testify v1.9.0
github.com/zeebo/errs v1.3.0
go.uber.org/zap v1.27.0
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5
storj.io/common v0.0.0-20240423093537-3f87f2a59098
storj.io/edge v1.74.1-0.20240320132813-10b6e5e71fe5
storj.io/storj v1.91.0-alpha.0.20240320150624-bf817ece68fd
)
Expand Down
4 changes: 2 additions & 2 deletions testsuite/playwright-ui/go.sum
View file
Expand Up @@ -2004,8 +2004,8 @@ sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
storj.io/common v0.0.0-20220719163320-cd2ef8e1b9b0/go.mod h1:mCYV6Ud5+cdbuaxdPD5Zht/HYaIn0sffnnws9ErkrMQ=
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5 h1:WyM77KVNO9+Q6Ox82iyEzYeGMAaAfiDedTIQLwOirTU=
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5/go.mod h1:MFl009RHY4tIqySVNy/6EmgRw2q60d26h9N/nb7JxGU=
storj.io/common v0.0.0-20240423093537-3f87f2a59098 h1:Xh0fk7anHRsrb9aYXro4SNas/FD0/+kEc3g9w9nkr3s=
storj.io/common v0.0.0-20240423093537-3f87f2a59098/go.mod h1:MFl009RHY4tIqySVNy/6EmgRw2q60d26h9N/nb7JxGU=
storj.io/dotworld v0.0.0-20210324183515-0d11aeccd840 h1:oqMwoF6vaOrCe92SKRyr8cc2WSjLYAd8fjpAHA7rNqY=
storj.io/dotworld v0.0.0-20210324183515-0d11aeccd840/go.mod h1:KU9YvEgRrMMiWLvH8pzn1UkoCoxggKIPvQxmNdx7aXQ=
storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
Expand Down
2 changes: 1 addition & 1 deletion testsuite/storjscan/go.mod
View file
Expand Up @@ -9,7 +9,7 @@ require (
github.com/zeebo/errs v1.3.0
go.uber.org/zap v1.27.0
golang.org/x/sync v0.6.0
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5
storj.io/common v0.0.0-20240423093537-3f87f2a59098
storj.io/storj v1.63.1
storj.io/storjscan v0.0.0-20220926140643-1623c3b391b0
storj.io/uplink v1.12.3-0.20240417071044-72c1ade6e3bd
Expand Down
4 changes: 2 additions & 2 deletions testsuite/storjscan/go.sum
View file
Expand Up @@ -1307,8 +1307,8 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
storj.io/common v0.0.0-20220719163320-cd2ef8e1b9b0/go.mod h1:mCYV6Ud5+cdbuaxdPD5Zht/HYaIn0sffnnws9ErkrMQ=
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5 h1:WyM77KVNO9+Q6Ox82iyEzYeGMAaAfiDedTIQLwOirTU=
storj.io/common v0.0.0-20240417231550-f34d2f5f48d5/go.mod h1:MFl009RHY4tIqySVNy/6EmgRw2q60d26h9N/nb7JxGU=
storj.io/common v0.0.0-20240423093537-3f87f2a59098 h1:Xh0fk7anHRsrb9aYXro4SNas/FD0/+kEc3g9w9nkr3s=
storj.io/common v0.0.0-20240423093537-3f87f2a59098/go.mod h1:MFl009RHY4tIqySVNy/6EmgRw2q60d26h9N/nb7JxGU=
storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
storj.io/drpc v0.0.34 h1:q9zlQKfJ5A7x8NQNFk8x7eKUF78FMhmAbZLnFK+og7I=
storj.io/drpc v0.0.34/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
Expand Down

0 comments on commit 4e7a3c2

Please sign in to comment.