From 6834c04539e1d49f5fc67aeb8b6a7feef7a614b6 Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Wed, 15 Nov 2023 15:11:26 +0100 Subject: [PATCH] satellite/metabase: drop pending objects table support We decided that we won't use seprate table for handling pending objects. We need to remove related code. https://github.com/storj/storj/issues/6421 Change-Id: I442b0f58da75409f725e08e2cd83d29ed4f91ec6 --- satellite/metabase/commit.go | 419 +--- satellite/metabase/commit_test.go | 1779 +---------------- satellite/metabase/delete.go | 67 - satellite/metabase/delete_bucket.go | 67 - satellite/metabase/delete_bucket_test.go | 47 - satellite/metabase/delete_objects.go | 116 -- satellite/metabase/delete_objects_test.go | 308 --- satellite/metabase/delete_test.go | 273 --- satellite/metabase/get.go | 24 +- satellite/metabase/get_test.go | 55 - satellite/metabase/list.go | 20 - satellite/metabase/metabasetest/common.go | 8 - satellite/metabase/metabasetest/create.go | 48 - satellite/metabase/metabasetest/test.go | 71 - .../metabase/pending_objects_iterator.go | 379 ---- .../metabase/pending_objects_iterator_test.go | 1283 ------------ satellite/metabase/raw.go | 81 +- satellite/metabase/zombiedeletion/chore.go | 2 +- satellite/metainfo/config.go | 1 - 19 files changed, 175 insertions(+), 4873 deletions(-) delete mode 100644 satellite/metabase/pending_objects_iterator.go delete mode 100644 satellite/metabase/pending_objects_iterator_test.go diff --git a/satellite/metabase/commit.go b/satellite/metabase/commit.go index 74408b9376c8..fb7763c420c6 100644 --- a/satellite/metabase/commit.go +++ b/satellite/metabase/commit.go @@ -47,11 +47,6 @@ type BeginObjectNextVersion struct { EncryptedMetadataEncryptedKey []byte // optional Encryption storj.EncryptionParameters - - // UsePendingObjectsTable was added to options not metabase configuration - // to be able to test scenarios with pending object in pending_objects and - // objects table with the same test case. - UsePendingObjectsTable bool } // Verify verifies get object request fields. @@ -73,7 +68,6 @@ func (opts *BeginObjectNextVersion) Verify() error { } // BeginObjectNextVersion adds a pending object to the database, with automatically assigned version. -// TODO at the end of transition to pending_objects table we can rename this metod to just BeginObject. func (db *DB) BeginObjectNextVersion(ctx context.Context, opts BeginObjectNextVersion) (object Object, err error) { defer mon.Task()(&ctx)(&err) @@ -98,34 +92,7 @@ func (db *DB) BeginObjectNextVersion(ctx context.Context, opts BeginObjectNextVe ZombieDeletionDeadline: opts.ZombieDeletionDeadline, } - if opts.UsePendingObjectsTable { - object.Status = Pending - object.Version = PendingVersion - - if err := db.db.QueryRowContext(ctx, ` - INSERT INTO pending_objects ( - project_id, bucket_name, object_key, stream_id, - expires_at, encryption, - zombie_deletion_deadline, - encrypted_metadata, encrypted_metadata_nonce, encrypted_metadata_encrypted_key - ) VALUES ( - $1, $2, $3, $4, - $5, $6, - $7, - $8, $9, $10) - RETURNING created_at - `, opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.StreamID, - opts.ExpiresAt, encryptionParameters{&opts.Encryption}, - opts.ZombieDeletionDeadline, - opts.EncryptedMetadata, opts.EncryptedMetadataNonce, opts.EncryptedMetadataEncryptedKey, - ).Scan(&object.CreatedAt); err != nil { - if code := pgerrcode.FromError(err); code == pgxerrcode.UniqueViolation { - return Object{}, Error.Wrap(ErrObjectAlreadyExists.New("")) - } - return Object{}, Error.New("unable to insert object: %w", err) - } - } else { - if err := db.db.QueryRowContext(ctx, ` + if err := db.db.QueryRowContext(ctx, ` INSERT INTO objects ( project_id, bucket_name, object_key, version, stream_id, expires_at, encryption, @@ -145,12 +112,11 @@ func (db *DB) BeginObjectNextVersion(ctx context.Context, opts BeginObjectNextVe $8, $9, $10) RETURNING status, version, created_at `, opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.StreamID, - opts.ExpiresAt, encryptionParameters{&opts.Encryption}, - opts.ZombieDeletionDeadline, - opts.EncryptedMetadata, opts.EncryptedMetadataNonce, opts.EncryptedMetadataEncryptedKey, - ).Scan(&object.Status, &object.Version, &object.CreatedAt); err != nil { - return Object{}, Error.New("unable to insert object: %w", err) - } + opts.ExpiresAt, encryptionParameters{&opts.Encryption}, + opts.ZombieDeletionDeadline, + opts.EncryptedMetadata, opts.EncryptedMetadataNonce, opts.EncryptedMetadataEncryptedKey, + ).Scan(&object.Status, &object.Version, &object.CreatedAt); err != nil { + return Object{}, Error.New("unable to insert object: %w", err) } mon.Meter("object_begin").Mark(1) @@ -258,8 +224,6 @@ type BeginSegment struct { RootPieceID storj.PieceID Pieces Pieces - - UsePendingObjectsTable bool } // BeginSegment verifies, whether a new segment upload can be started. @@ -283,24 +247,14 @@ func (db *DB) BeginSegment(ctx context.Context, opts BeginSegment) (err error) { // Verify that object exists and is partial. var exists bool - if opts.UsePendingObjectsTable { - err = db.db.QueryRowContext(ctx, ` - SELECT EXISTS ( - SELECT 1 - FROM pending_objects - WHERE (project_id, bucket_name, object_key, stream_id) = ($1, $2, $3, $4) - ) - `, opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.StreamID).Scan(&exists) - } else { - err = db.db.QueryRowContext(ctx, ` - SELECT EXISTS ( - SELECT 1 - FROM objects - WHERE (project_id, bucket_name, object_key, version, stream_id) = ($1, $2, $3, $4, $5) AND - status = `+statusPending+` - )`, - opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.Version, opts.StreamID).Scan(&exists) - } + err = db.db.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 + FROM objects + WHERE (project_id, bucket_name, object_key, version, stream_id) = ($1, $2, $3, $4, $5) AND + status = `+statusPending+` + )`, + opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.Version, opts.StreamID).Scan(&exists) if err != nil { return Error.New("unable to query object status: %w", err) } @@ -336,8 +290,6 @@ type CommitSegment struct { Pieces Pieces Placement storj.PlacementConstraint - - UsePendingObjectsTable bool } // CommitSegment commits segment to the database. @@ -378,84 +330,45 @@ func (db *DB) CommitSegment(ctx context.Context, opts CommitSegment) (err error) return Error.New("unable to convert pieces to aliases: %w", err) } - // second part will be removed when there will be no pending_objects in objects table. // Verify that object exists and is partial. - if opts.UsePendingObjectsTable { - _, err = db.db.ExecContext(ctx, ` - INSERT INTO segments ( - stream_id, position, expires_at, - root_piece_id, encrypted_key_nonce, encrypted_key, - encrypted_size, plain_offset, plain_size, encrypted_etag, - redundancy, - remote_alias_pieces, - placement - ) VALUES ( - ( - SELECT stream_id - FROM pending_objects - WHERE (project_id, bucket_name, object_key, stream_id) = ($12, $13, $14, $15) - ), $1, $2, - $3, $4, $5, - $6, $7, $8, $9, - $10, - $11, - $16 - ) - ON CONFLICT(stream_id, position) - DO UPDATE SET - expires_at = $2, - root_piece_id = $3, encrypted_key_nonce = $4, encrypted_key = $5, - encrypted_size = $6, plain_offset = $7, plain_size = $8, encrypted_etag = $9, - redundancy = $10, - remote_alias_pieces = $11, - placement = $16 - `, opts.Position, opts.ExpiresAt, - opts.RootPieceID, opts.EncryptedKeyNonce, opts.EncryptedKey, - opts.EncryptedSize, opts.PlainOffset, opts.PlainSize, opts.EncryptedETag, - redundancyScheme{&opts.Redundancy}, - aliasPieces, - opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.StreamID, - opts.Placement, - ) - } else { - _, err = db.db.ExecContext(ctx, ` - INSERT INTO segments ( - stream_id, position, expires_at, - root_piece_id, encrypted_key_nonce, encrypted_key, - encrypted_size, plain_offset, plain_size, encrypted_etag, - redundancy, - remote_alias_pieces, - placement - ) VALUES ( - ( - SELECT stream_id - FROM objects - WHERE (project_id, bucket_name, object_key, version, stream_id) = ($12, $13, $14, $15, $16) AND - status = `+statusPending+` - ), $1, $2, - $3, $4, $5, - $6, $7, $8, $9, - $10, - $11, - $17 - ) - ON CONFLICT(stream_id, position) - DO UPDATE SET - expires_at = $2, - root_piece_id = $3, encrypted_key_nonce = $4, encrypted_key = $5, - encrypted_size = $6, plain_offset = $7, plain_size = $8, encrypted_etag = $9, - redundancy = $10, - remote_alias_pieces = $11, - placement = $17 - `, opts.Position, opts.ExpiresAt, - opts.RootPieceID, opts.EncryptedKeyNonce, opts.EncryptedKey, - opts.EncryptedSize, opts.PlainOffset, opts.PlainSize, opts.EncryptedETag, - redundancyScheme{&opts.Redundancy}, - aliasPieces, - opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.Version, opts.StreamID, - opts.Placement, + + _, err = db.db.ExecContext(ctx, ` + INSERT INTO segments ( + stream_id, position, expires_at, + root_piece_id, encrypted_key_nonce, encrypted_key, + encrypted_size, plain_offset, plain_size, encrypted_etag, + redundancy, + remote_alias_pieces, + placement + ) VALUES ( + ( + SELECT stream_id + FROM objects + WHERE (project_id, bucket_name, object_key, version, stream_id) = ($12, $13, $14, $15, $16) AND + status = `+statusPending+` + ), $1, $2, + $3, $4, $5, + $6, $7, $8, $9, + $10, + $11, + $17 ) - } + ON CONFLICT(stream_id, position) + DO UPDATE SET + expires_at = $2, + root_piece_id = $3, encrypted_key_nonce = $4, encrypted_key = $5, + encrypted_size = $6, plain_offset = $7, plain_size = $8, encrypted_etag = $9, + redundancy = $10, + remote_alias_pieces = $11, + placement = $17 + `, opts.Position, opts.ExpiresAt, + opts.RootPieceID, opts.EncryptedKeyNonce, opts.EncryptedKey, + opts.EncryptedSize, opts.PlainOffset, opts.PlainSize, opts.EncryptedETag, + redundancyScheme{&opts.Redundancy}, + aliasPieces, + opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.Version, opts.StreamID, + opts.Placement, + ) if err != nil { if code := pgerrcode.FromError(err); code == pgxerrcode.NotNullViolation { return ErrPendingObjectMissing.New("") @@ -485,8 +398,6 @@ type CommitInlineSegment struct { EncryptedETag []byte InlineData []byte - - UsePendingObjectsTable bool } // CommitInlineSegment commits inline segment to the database. @@ -511,8 +422,7 @@ func (db *DB) CommitInlineSegment(ctx context.Context, opts CommitInlineSegment) return ErrInvalidRequest.New("PlainOffset negative") } - if opts.UsePendingObjectsTable { - _, err = db.db.ExecContext(ctx, ` + _, err = db.db.ExecContext(ctx, ` INSERT INTO segments ( stream_id, position, expires_at, root_piece_id, encrypted_key_nonce, encrypted_key, @@ -521,9 +431,11 @@ func (db *DB) CommitInlineSegment(ctx context.Context, opts CommitInlineSegment) ) VALUES ( ( SELECT stream_id - FROM pending_objects - WHERE (project_id, bucket_name, object_key, stream_id) = ($11, $12, $13, $14) - ), $1, $2, + FROM objects + WHERE (project_id, bucket_name, object_key, version, stream_id) = ($11, $12, $13, $14, $15) AND + status = `+statusPending+` + ), + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 @@ -535,43 +447,11 @@ func (db *DB) CommitInlineSegment(ctx context.Context, opts CommitInlineSegment) encrypted_size = $6, plain_offset = $7, plain_size = $8, encrypted_etag = $9, inline_data = $10 `, opts.Position, opts.ExpiresAt, - storj.PieceID{}, opts.EncryptedKeyNonce, opts.EncryptedKey, - len(opts.InlineData), opts.PlainOffset, opts.PlainSize, opts.EncryptedETag, - opts.InlineData, - opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.StreamID, - ) - } else { - _, err = db.db.ExecContext(ctx, ` - INSERT INTO segments ( - stream_id, position, expires_at, - root_piece_id, encrypted_key_nonce, encrypted_key, - encrypted_size, plain_offset, plain_size, encrypted_etag, - inline_data - ) VALUES ( - ( - SELECT stream_id - FROM objects - WHERE (project_id, bucket_name, object_key, version, stream_id) = ($11, $12, $13, $14, $15) AND - status = `+statusPending+` - ), - $1, $2, - $3, $4, $5, - $6, $7, $8, $9, - $10 - ) - ON CONFLICT(stream_id, position) - DO UPDATE SET - expires_at = $2, - root_piece_id = $3, encrypted_key_nonce = $4, encrypted_key = $5, - encrypted_size = $6, plain_offset = $7, plain_size = $8, encrypted_etag = $9, - inline_data = $10 - `, opts.Position, opts.ExpiresAt, - storj.PieceID{}, opts.EncryptedKeyNonce, opts.EncryptedKey, - len(opts.InlineData), opts.PlainOffset, opts.PlainSize, opts.EncryptedETag, - opts.InlineData, - opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.Version, opts.StreamID, - ) - } + storj.PieceID{}, opts.EncryptedKeyNonce, opts.EncryptedKey, + len(opts.InlineData), opts.PlainOffset, opts.PlainSize, opts.EncryptedETag, + opts.InlineData, + opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.Version, opts.StreamID, + ) if err != nil { if code := pgerrcode.FromError(err); code == pgxerrcode.NotNullViolation { return ErrPendingObjectMissing.New("") @@ -602,8 +482,6 @@ type CommitObject struct { DisallowDelete bool - UsePendingObjectsTable bool - // Versioned indicates whether an object is allowed to have multiple versions. Versioned bool } @@ -676,8 +554,6 @@ func (db *DB) CommitObject(ctx context.Context, opts CommitObject) (object Objec totalEncryptedSize += int64(seg.EncryptedSize) } - const versionArgIndex = 3 - nextStatus := committedWhereVersioned(opts.Versioned) args := []interface{}{ @@ -699,136 +575,63 @@ func (db *DB) CommitObject(ctx context.Context, opts CommitObject) (object Objec return Error.Wrap(err) } - if opts.UsePendingObjectsTable { - opts.Version = precommit.HighestVersion + 1 - args[versionArgIndex] = opts.Version + nextVersion := opts.Version + if nextVersion < precommit.HighestVersion { + nextVersion = precommit.HighestVersion + 1 + } + args = append(args, nextVersion) + opts.Version = nextVersion + metadataColumns := "" + if opts.OverrideEncryptedMetadata { args = append(args, opts.EncryptedMetadataNonce, opts.EncryptedMetadata, opts.EncryptedMetadataEncryptedKey, - opts.OverrideEncryptedMetadata, ) - - err = tx.QueryRowContext(ctx, ` - WITH delete_pending_object AS ( - DELETE FROM pending_objects - WHERE (project_id, bucket_name, object_key, stream_id) = ($1, $2, $3, $5) - RETURNING expires_at, encryption, encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key - ) - INSERT INTO objects ( - project_id, bucket_name, object_key, version, stream_id, - status, segment_count, total_plain_size, total_encrypted_size, - fixed_segment_size, zombie_deletion_deadline, expires_at, - encryption, - encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key - ) - SELECT - $1 as project_id, $2 as bucket_name, $3 as object_key, $4::INT4 as version, $5 as stream_id, - $6 as status, $7::INT4 as segment_count, $8::INT8 as total_plain_size, $9::INT8 as total_encrypted_size, - $10::INT4 as fixed_segment_size, NULL::timestamp as zombie_deletion_deadline, expires_at, - -- TODO should we allow to override existing encryption parameters or return error if don't match with opts? - CASE - WHEN encryption = 0 AND $11 <> 0 THEN $11 - WHEN encryption = 0 AND $11 = 0 THEN NULL - ELSE encryption - END as - encryption, - CASE - WHEN $15::BOOL = true THEN $12 - ELSE encrypted_metadata_nonce - END as - encrypted_metadata_nonce, - CASE - WHEN $15::BOOL = true THEN $13 - ELSE encrypted_metadata - END as - encrypted_metadata, - CASE - WHEN $15::BOOL = true THEN $14 - ELSE encrypted_metadata_encrypted_key - END as - encrypted_metadata_encrypted_key - FROM delete_pending_object - -- we don't want ON CONFLICT clause to update existign object - -- as this way we may miss removing old object segments - RETURNING - created_at, expires_at, - encrypted_metadata, encrypted_metadata_encrypted_key, encrypted_metadata_nonce, - encryption + metadataColumns = `, + encrypted_metadata_nonce = $13, + encrypted_metadata = $14, + encrypted_metadata_encrypted_key = $15 + ` + } + err = tx.QueryRowContext(ctx, ` + UPDATE objects SET + version = $12, + status = $6, + segment_count = $7, + + total_plain_size = $8, + total_encrypted_size = $9, + fixed_segment_size = $10, + zombie_deletion_deadline = NULL, + + -- TODO should we allow to override existing encryption parameters or return error if don't match with opts? + encryption = CASE + WHEN objects.encryption = 0 AND $11 <> 0 THEN $11 + WHEN objects.encryption = 0 AND $11 = 0 THEN NULL + ELSE objects.encryption + END + `+metadataColumns+` + WHERE (project_id, bucket_name, object_key, version, stream_id) = ($1, $2, $3, $4, $5) AND + status = `+statusPending+` + RETURNING + created_at, expires_at, + encrypted_metadata, encrypted_metadata_encrypted_key, encrypted_metadata_nonce, + encryption `, args...).Scan( - &object.CreatedAt, &object.ExpiresAt, - &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, &object.EncryptedMetadataNonce, - encryptionParameters{&object.Encryption}, - ) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return ErrObjectNotFound.Wrap(Error.New("object with specified version and pending status is missing")) - } else if code := pgerrcode.FromError(err); code == pgxerrcode.NotNullViolation { - // TODO maybe we should check message if 'encryption' label is there - return ErrInvalidRequest.New("Encryption is missing") - } - return Error.New("failed to update object: %w", err) - } - } else { - nextVersion := opts.Version - if nextVersion < precommit.HighestVersion { - nextVersion = precommit.HighestVersion + 1 - } - args = append(args, nextVersion) - opts.Version = nextVersion - - metadataColumns := "" - if opts.OverrideEncryptedMetadata { - args = append(args, - opts.EncryptedMetadataNonce, - opts.EncryptedMetadata, - opts.EncryptedMetadataEncryptedKey, - ) - metadataColumns = `, - encrypted_metadata_nonce = $13, - encrypted_metadata = $14, - encrypted_metadata_encrypted_key = $15 - ` - } - err = tx.QueryRowContext(ctx, ` - UPDATE objects SET - version = $12, - status = $6, - segment_count = $7, - - total_plain_size = $8, - total_encrypted_size = $9, - fixed_segment_size = $10, - zombie_deletion_deadline = NULL, - - -- TODO should we allow to override existing encryption parameters or return error if don't match with opts? - encryption = CASE - WHEN objects.encryption = 0 AND $11 <> 0 THEN $11 - WHEN objects.encryption = 0 AND $11 = 0 THEN NULL - ELSE objects.encryption - END - `+metadataColumns+` - WHERE (project_id, bucket_name, object_key, version, stream_id) = ($1, $2, $3, $4, $5) AND - status = `+statusPending+` - RETURNING - created_at, expires_at, - encrypted_metadata, encrypted_metadata_encrypted_key, encrypted_metadata_nonce, - encryption - `, args...).Scan( - &object.CreatedAt, &object.ExpiresAt, - &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, &object.EncryptedMetadataNonce, - encryptionParameters{&object.Encryption}, - ) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return ErrObjectNotFound.Wrap(Error.New("object with specified version and pending status is missing")) - } else if code := pgerrcode.FromError(err); code == pgxerrcode.NotNullViolation { - // TODO maybe we should check message if 'encryption' label is there - return ErrInvalidRequest.New("Encryption is missing") - } - return Error.New("failed to update object: %w", err) + &object.CreatedAt, &object.ExpiresAt, + &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, &object.EncryptedMetadataNonce, + encryptionParameters{&object.Encryption}, + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrObjectNotFound.Wrap(Error.New("object with specified version and pending status is missing")) + } else if code := pgerrcode.FromError(err); code == pgxerrcode.NotNullViolation { + // TODO maybe we should check message if 'encryption' label is there + return ErrInvalidRequest.New("Encryption is missing") } + return Error.New("failed to update object: %w", err) } object.StreamID = opts.StreamID diff --git a/satellite/metabase/commit_test.go b/satellite/metabase/commit_test.go index a919e3725f22..e941b7abe53e 100644 --- a/satellite/metabase/commit_test.go +++ b/satellite/metabase/commit_test.go @@ -337,184 +337,6 @@ func TestBeginObjectNextVersion(t *testing.T) { }) } -func TestBeginObjectNextVersion_PendingObjects(t *testing.T) { - // TODO when we stop storing pending objects in objects tabe we will be able - // to merge this tests with TestBeginObjectNextVersion - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj := metabasetest.RandObjectStream() - - objectStream := metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - } - - t.Run("object already exists", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now1 := time.Now() - zombieDeadline := now1.Add(24 * time.Hour) - futureTime := now1.Add(10 * 24 * time.Hour) - - objectStream.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &futureTime, - - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - ErrClass: &metabase.ErrObjectAlreadyExists, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabase.PendingObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - }, - CreatedAt: now1, - ExpiresAt: nil, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("multiple versions", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now1 := time.Now() - zombieDeadline := now1.Add(24 * time.Hour) - futureTime := now1.Add(10 * 24 * time.Hour) - - objectStream.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - now2 := time.Now() - - secondObjectStream := objectStream - secondObjectStream.StreamID = testrand.UUID() - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: secondObjectStream, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &futureTime, - - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabase.PendingObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - }, - CreatedAt: now1, - ExpiresAt: nil, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - { - PendingObjectStream: metabase.PendingObjectStream{ - ProjectID: secondObjectStream.ProjectID, - BucketName: secondObjectStream.BucketName, - ObjectKey: secondObjectStream.ObjectKey, - StreamID: secondObjectStream.StreamID, - }, - CreatedAt: now2, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &futureTime, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("begin object next version with metadata", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - objectStream.Version = metabase.NextVersion - - encryptedMetadata := testrand.BytesInt(64) - encryptedMetadataNonce := testrand.Nonce() - encryptedMetadataEncryptedKey := testrand.BytesInt(32) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadataEncryptedKey: encryptedMetadataEncryptedKey, - - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabase.PendingObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - }, - CreatedAt: now, - - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadataEncryptedKey: encryptedMetadataEncryptedKey, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - }) -} - func TestBeginObjectExactVersion(t *testing.T) { metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { obj := metabasetest.RandObjectStream() @@ -1119,158 +941,6 @@ func TestBeginSegment(t *testing.T) { }, }.Check(ctx, t, db) }) - - // TODO those test are copies of tests above with some adjustments to test pending_objects table. - // we will be able to delete those tests when we will start supporting only pending_objects table. - t.Run("use pending objects table", func(t *testing.T) { - obj.Version = metabase.NextVersion - t.Run("pending object missing", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - RootPieceID: storj.PieceID{1}, - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - ErrClass: &metabase.ErrPendingObjectMissing, - }.Check(ctx, t, db) - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("pending object missing when object committed", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - now := time.Now() - - obj := obj - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - }, - Version: 1, - }.Check(ctx, t, db) - obj.Version++ - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - }, - }.Check(ctx, t, db) - - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - RootPieceID: storj.PieceID{1}, - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - ErrClass: &metabase.ErrPendingObjectMissing, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("begin segment successfully", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - RootPieceID: storj.PieceID{1}, - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("multiple begin segment successfully", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - obj := metabasetest.RandObjectStream() - obj.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - for i := 0; i < 5; i++ { - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - RootPieceID: testrand.PieceID(), - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - } - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - }) }) } @@ -1931,358 +1601,15 @@ func TestCommitSegment(t *testing.T) { }, }.Check(ctx, t, db) }) + }) +} - // TODO those test are copies of tests above with some adjustments to test pending_objects table. - // we will be able to delete those tests when we will start supporting only pending_objects table. - t.Run("use pending objects table", func(t *testing.T) { - obj.Version = metabase.NextVersion - t.Run("duplicate", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now1 := time.Now() - zombieDeadline := now1.Add(24 * time.Hour) - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - rootPieceID := testrand.PieceID() - pieces := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}} - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - RootPieceID: rootPieceID, - Pieces: pieces, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now1, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - CreatedAt: now, - - RootPieceID: rootPieceID, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainOffset: 0, - PlainSize: 512, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("overwrite", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now1 := time.Now() - zombieDeadline := now1.Add(24 * time.Hour) - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - rootPieceID1 := testrand.PieceID() - rootPieceID2 := testrand.PieceID() - pieces1 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}} - pieces2 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}} - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - RootPieceID: rootPieceID1, - Pieces: pieces1, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - RootPieceID: rootPieceID1, - Pieces: pieces1, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - RootPieceID: rootPieceID2, - Pieces: pieces2, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now1, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - CreatedAt: now, - - RootPieceID: rootPieceID2, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainOffset: 0, - PlainSize: 512, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces2, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit segment of object with expires at", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - rootPieceID := testrand.PieceID() - pieces := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}} - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - - now := time.Now() - expectedExpiresAt := now.Add(33 * time.Hour) - zombieDeadline := now.Add(24 * time.Hour) - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - ExpiresAt: &expectedExpiresAt, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - ExpiresAt: &expectedExpiresAt, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - ExpiresAt: &expectedExpiresAt, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - CreatedAt: now, - ExpiresAt: &expectedExpiresAt, - - RootPieceID: rootPieceID, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainOffset: 0, - PlainSize: 512, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit segment of pending object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - rootPieceID := testrand.PieceID() - pieces := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}} - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - encryptedETag := testrand.Bytes(32) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - EncryptedETag: encryptedETag, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - CreatedAt: now, - - RootPieceID: rootPieceID, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainOffset: 0, - PlainSize: 512, - EncryptedETag: encryptedETag, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces, - }, - }, - }.Check(ctx, t, db) - }) - }) - }) -} - -func TestCommitInlineSegment(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj := metabasetest.RandObjectStream() - for _, test := range metabasetest.InvalidObjectStreams(obj) { - test := test - t.Run(test.Name, func(t *testing.T) { +func TestCommitInlineSegment(t *testing.T) { + metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { + obj := metabasetest.RandObjectStream() + for _, test := range metabasetest.InvalidObjectStreams(obj) { + test := test + t.Run(test.Name, func(t *testing.T) { defer metabasetest.DeleteAll{}.Check(ctx, t, db) metabasetest.CommitInlineSegment{ Opts: metabase.CommitInlineSegment{ @@ -2755,423 +2082,68 @@ func TestCommitInlineSegment(t *testing.T) { }, }.Check(ctx, t, db) }) + }) +} - // TODO those test are copies of tests above with some adjustments to test pending_objects table. - // we will be able to delete those tests when we will start supporting only pending_objects table. - t.Run("use pending objects table", func(t *testing.T) { - obj.Version = metabase.NextVersion - t.Run("duplicate", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) +func TestCommitObject(t *testing.T) { + metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { + obj := metabasetest.RandObjectStream() - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, + for _, test := range metabasetest.InvalidObjectStreams(obj) { + test := test + t.Run(test.Name, func(t *testing.T) { + defer metabasetest.DeleteAll{}.Check(ctx, t, db) + metabasetest.CommitObject{ + Opts: metabase.CommitObject{ + ObjectStream: test.ObjectStream, }, - Version: metabase.PendingVersion, + ErrClass: test.ErrClass, + ErrText: test.ErrText, }.Check(ctx, t, db) + metabasetest.Verify{}.Check(ctx, t, db) + }) + } - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - InlineData: []byte{1, 2, 3}, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, + t.Run("invalid EncryptedMetadata", func(t *testing.T) { + defer metabasetest.DeleteAll{}.Check(ctx, t, db) - PlainSize: 512, - PlainOffset: 0, + metabasetest.CommitObject{ + Opts: metabase.CommitObject{ + ObjectStream: metabase.ObjectStream{ + ProjectID: obj.ProjectID, + BucketName: obj.BucketName, + ObjectKey: obj.ObjectKey, + Version: metabase.DefaultVersion, + StreamID: obj.StreamID, + }, + OverrideEncryptedMetadata: true, + EncryptedMetadata: testrand.BytesInt(32), + }, + ErrClass: &metabase.ErrInvalidRequest, + ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be set if EncryptedMetadata is set", + }.Check(ctx, t, db) - UsePendingObjectsTable: true, + metabasetest.CommitObject{ + Opts: metabase.CommitObject{ + ObjectStream: metabase.ObjectStream{ + ProjectID: obj.ProjectID, + BucketName: obj.BucketName, + ObjectKey: obj.ObjectKey, + Version: metabase.DefaultVersion, + StreamID: obj.StreamID, }, - }.Check(ctx, t, db) + OverrideEncryptedMetadata: true, + EncryptedMetadataEncryptedKey: testrand.BytesInt(32), + }, + ErrClass: &metabase.ErrInvalidRequest, + ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be not set if EncryptedMetadata is not set", + }.Check(ctx, t, db) - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - InlineData: []byte{1, 2, 3}, + metabasetest.Verify{}.Check(ctx, t, db) + }) - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainSize: 512, - PlainOffset: 0, - - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - CreatedAt: now, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainOffset: 0, - PlainSize: 512, - - InlineData: []byte{1, 2, 3}, - EncryptedSize: 3, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("overwrite", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - InlineData: []byte{1, 2, 3}, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainSize: 512, - PlainOffset: 0, - - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - InlineData: []byte{4, 5, 6}, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainSize: 512, - PlainOffset: 0, - - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - CreatedAt: now, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainOffset: 0, - PlainSize: 512, - - InlineData: []byte{4, 5, 6}, - EncryptedSize: 3, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit empty segment of pending object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - encryptedETag := testrand.Bytes(32) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainSize: 0, - PlainOffset: 0, - EncryptedETag: encryptedETag, - - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - CreatedAt: now, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainOffset: 0, - PlainSize: 0, - - EncryptedSize: 0, - EncryptedETag: encryptedETag, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit segment of pending object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - encryptedETag := testrand.Bytes(32) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - InlineData: []byte{1, 2, 3}, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainSize: 512, - PlainOffset: 0, - EncryptedETag: encryptedETag, - - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - CreatedAt: now, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainOffset: 0, - PlainSize: 512, - - InlineData: []byte{1, 2, 3}, - EncryptedSize: 3, - EncryptedETag: encryptedETag, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit segment of object with expires at", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - encryptedETag := testrand.Bytes(32) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - expectedExpiresAt := now.Add(33 * time.Hour) - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - ExpiresAt: &expectedExpiresAt, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - ExpiresAt: &expectedExpiresAt, - InlineData: []byte{1, 2, 3}, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainSize: 512, - PlainOffset: 0, - EncryptedETag: encryptedETag, - - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - ExpiresAt: &expectedExpiresAt, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - CreatedAt: now, - ExpiresAt: &expectedExpiresAt, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - PlainOffset: 0, - PlainSize: 512, - - InlineData: []byte{1, 2, 3}, - EncryptedSize: 3, - EncryptedETag: encryptedETag, - }, - }, - }.Check(ctx, t, db) - }) - }) - }) -} - -func TestCommitObject(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj := metabasetest.RandObjectStream() - - for _, test := range metabasetest.InvalidObjectStreams(obj) { - test := test - t.Run(test.Name, func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: test.ObjectStream, - }, - ErrClass: test.ErrClass, - ErrText: test.ErrText, - }.Check(ctx, t, db) - metabasetest.Verify{}.Check(ctx, t, db) - }) - } - - t.Run("invalid EncryptedMetadata", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: metabase.DefaultVersion, - StreamID: obj.StreamID, - }, - OverrideEncryptedMetadata: true, - EncryptedMetadata: testrand.BytesInt(32), - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be set if EncryptedMetadata is set", - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: metabase.DefaultVersion, - StreamID: obj.StreamID, - }, - OverrideEncryptedMetadata: true, - EncryptedMetadataEncryptedKey: testrand.BytesInt(32), - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be not set if EncryptedMetadata is not set", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("version without pending", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) + t.Run("version without pending", func(t *testing.T) { + defer metabasetest.DeleteAll{}.Check(ctx, t, db) metabasetest.CommitObject{ Opts: metabase.CommitObject{ @@ -3726,637 +2698,6 @@ func TestCommitObject(t *testing.T) { }, }.Check(ctx, t, db) }) - - t.Run("use pending objects table", func(t *testing.T) { - obj.Version = metabase.NextVersion - t.Run("version", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - }, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - now := time.Now() - - encryptedMetadata := testrand.Bytes(1024) - encryptedMetadataNonce := testrand.Nonce() - encryptedMetadataKey := testrand.Bytes(265) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 1, - StreamID: obj.StreamID, - }, - OverrideEncryptedMetadata: true, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataEncryptedKey: encryptedMetadataKey, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - // disallow for double commit - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 1, - StreamID: obj.StreamID, - }, - UsePendingObjectsTable: true, - }, - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "metabase: object with specified version and pending status is missing", // TODO: this error message could be better - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 1, - StreamID: obj.StreamID, - }, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataEncryptedKey: encryptedMetadataKey, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("assign plain_offset", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version++ - now := time.Now() - - rootPieceID := testrand.PieceID() - pieces := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}} - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Index: 0}, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 999999, - - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Index: 1}, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 999999, - - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Index: 0}, - CreatedAt: now, - - RootPieceID: rootPieceID, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces, - }, - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Index: 1}, - CreatedAt: now, - - RootPieceID: rootPieceID, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 512, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces, - }, - }, - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - SegmentCount: 2, - FixedSegmentSize: 512, - TotalPlainSize: 2 * 512, - TotalEncryptedSize: 2 * 1024, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("large object over 2 GB", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version++ - now := time.Now() - - rootPieceID := testrand.PieceID() - pieces := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}} - encryptedKey := testrand.Bytes(32) - encryptedKeyNonce := testrand.Bytes(32) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Index: 0}, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: math.MaxInt32, - PlainSize: math.MaxInt32, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Index: 1}, - RootPieceID: rootPieceID, - Pieces: pieces, - - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: math.MaxInt32, - PlainSize: math.MaxInt32, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Index: 0}, - CreatedAt: now, - - RootPieceID: rootPieceID, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: math.MaxInt32, - PlainSize: math.MaxInt32, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces, - }, - { - StreamID: obj.StreamID, - Position: metabase.SegmentPosition{Index: 1}, - CreatedAt: now, - - RootPieceID: rootPieceID, - EncryptedKey: encryptedKey, - EncryptedKeyNonce: encryptedKeyNonce, - - EncryptedSize: math.MaxInt32, - PlainSize: math.MaxInt32, - PlainOffset: math.MaxInt32, - - Redundancy: metabasetest.DefaultRedundancy, - - Pieces: pieces, - }, - }, - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - SegmentCount: 2, - FixedSegmentSize: math.MaxInt32, - TotalPlainSize: 2 * math.MaxInt32, - TotalEncryptedSize: 2 * math.MaxInt32, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit with encryption (no override)", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version++ - now := time.Now() - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - // set different encryption than with BeginObjectExactVersion - Encryption: storj.EncryptionParameters{ - CipherSuite: storj.EncNull, - BlockSize: 512, - }, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - SegmentCount: 0, - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit with metadata (no overwrite)", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - - expectedMetadata := testrand.Bytes(memory.KiB) - expectedMetadataKey := testrand.Bytes(32) - expectedMetadataNonce := testrand.Nonce().Bytes() - - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: expectedMetadata, - EncryptedMetadataEncryptedKey: expectedMetadataKey, - EncryptedMetadataNonce: expectedMetadataNonce, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version++ - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: expectedMetadata, - EncryptedMetadataEncryptedKey: expectedMetadataKey, - EncryptedMetadataNonce: expectedMetadataNonce, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit with metadata (overwrite)", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - - expectedMetadata := testrand.Bytes(memory.KiB) - expecedMetadataKey := testrand.Bytes(32) - expecedMetadataNonce := testrand.Nonce().Bytes() - - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: testrand.Bytes(memory.KiB), - EncryptedMetadataEncryptedKey: testrand.Bytes(32), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version++ - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - OverrideEncryptedMetadata: true, - EncryptedMetadata: expectedMetadata, - EncryptedMetadataEncryptedKey: expecedMetadataKey, - EncryptedMetadataNonce: expecedMetadataNonce, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: expectedMetadata, - EncryptedMetadataEncryptedKey: expecedMetadataKey, - EncryptedMetadataNonce: expecedMetadataNonce, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("commit with empty metadata (overwrite)", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: testrand.Bytes(memory.KiB), - EncryptedMetadataEncryptedKey: testrand.Bytes(32), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version++ - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - OverrideEncryptedMetadata: true, - EncryptedMetadata: nil, - EncryptedMetadataEncryptedKey: nil, - EncryptedMetadataNonce: nil, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("overwrite object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - - obj := metabasetest.RandObjectStream() - for i := 0; i < 10; i++ { - obj.Version = metabase.NextVersion - obj.StreamID = testrand.UUID() - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: testrand.Bytes(memory.KiB), - EncryptedMetadataEncryptedKey: testrand.Bytes(32), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version = metabase.Version(i + 1) - - expectedInlineData := testrand.Bytes(8) - expectedEncryptedKey := testrand.Bytes(32) - expectedEncryptedKeyNonce := testrand.Bytes(32) - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - InlineData: expectedInlineData, - EncryptedKey: expectedEncryptedKey, - EncryptedKeyNonce: expectedEncryptedKeyNonce, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - OverrideEncryptedMetadata: true, - EncryptedMetadata: nil, - EncryptedMetadataEncryptedKey: nil, - EncryptedMetadataNonce: nil, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - SegmentCount: 1, - TotalEncryptedSize: int64(len(expectedInlineData)), - - Encryption: metabasetest.DefaultEncryption, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj.StreamID, - EncryptedKey: expectedEncryptedKey, - EncryptedKeyNonce: expectedEncryptedKeyNonce, - EncryptedSize: int32(len(expectedInlineData)), - InlineData: expectedInlineData, - CreatedAt: time.Now(), - }, - }, - }.Check(ctx, t, db) - } - }) - - t.Run("upload with expiration time", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - now := time.Now() - expectedExpiresAt := now.Add(time.Hour).Truncate(time.Millisecond) - - obj := obj - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: testrand.Bytes(memory.KiB), - EncryptedMetadataEncryptedKey: testrand.Bytes(32), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - - ExpiresAt: &expectedExpiresAt, - - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - obj.Version = metabase.DefaultVersion - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - - OverrideEncryptedMetadata: true, - EncryptedMetadata: nil, - EncryptedMetadataEncryptedKey: nil, - EncryptedMetadataNonce: nil, - - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - SegmentCount: 0, - ExpiresAt: &expectedExpiresAt, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - }) }) } diff --git a/satellite/metabase/delete.go b/satellite/metabase/delete.go index 9823dc661ee2..fdf14950fef7 100644 --- a/satellite/metabase/delete.go +++ b/satellite/metabase/delete.go @@ -205,47 +205,6 @@ func (db *DB) DeletePendingObject(ctx context.Context, opts DeletePendingObject) return result, nil } -// DeletePendingObjectNew deletes a pending object. -// TODO DeletePendingObjectNew will replace DeletePendingObject when objects table will be free from pending objects. -func (db *DB) DeletePendingObjectNew(ctx context.Context, opts DeletePendingObject) (result DeleteObjectResult, err error) { - defer mon.Task()(&ctx)(&err) - - if err := opts.Verify(); err != nil { - return DeleteObjectResult{}, err - } - - err = withRows(db.db.QueryContext(ctx, ` - WITH deleted_objects AS ( - DELETE FROM pending_objects - WHERE - (project_id, bucket_name, object_key, stream_id) = ($1, $2, $3, $4) - RETURNING - stream_id, created_at, expires_at, - encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, - encryption - ), deleted_segments AS ( - DELETE FROM segments - WHERE segments.stream_id IN (SELECT deleted_objects.stream_id FROM deleted_objects) - RETURNING segments.stream_id - ) - SELECT * FROM deleted_objects - `, opts.ProjectID, []byte(opts.BucketName), opts.ObjectKey, opts.StreamID))(func(rows tagsql.Rows) error { - result.Removed, err = db.scanPendingObjectDeletion(ctx, opts.Location(), rows) - return err - }) - if err != nil { - return DeleteObjectResult{}, err - } - - if len(result.Removed) == 0 { - return DeleteObjectResult{}, ErrObjectNotFound.Wrap(Error.New("no rows deleted")) - } - - mon.Meter("object_delete").Mark(len(result.Removed)) - - return result, nil -} - // DeleteObjectsAllVersions deletes all versions of multiple objects from the same bucket. func (db *DB) DeleteObjectsAllVersions(ctx context.Context, opts DeleteObjectsAllVersions) (result DeleteObjectResult, err error) { defer mon.Task()(&ctx)(&err) @@ -375,32 +334,6 @@ func (db *DB) scanMultipleObjectsDeletion(ctx context.Context, rows tagsql.Rows) return objects, nil } -func (db *DB) scanPendingObjectDeletion(ctx context.Context, location ObjectLocation, rows tagsql.Rows) (objects []Object, err error) { - defer mon.Task()(&ctx)(&err) - - objects = make([]Object, 0, 10) - - var object Object - for rows.Next() { - object.ProjectID = location.ProjectID - object.BucketName = location.BucketName - object.ObjectKey = location.ObjectKey - - err = rows.Scan(&object.StreamID, - &object.CreatedAt, &object.ExpiresAt, - &object.EncryptedMetadataNonce, &object.EncryptedMetadata, &object.EncryptedMetadataEncryptedKey, - encryptionParameters{&object.Encryption}, - ) - if err != nil { - return nil, Error.New("unable to delete pending object: %w", err) - } - - object.Status = Pending - objects = append(objects, object) - } - return objects, nil -} - // DeleteObjectLastCommitted contains arguments necessary for deleting last committed version of object. type DeleteObjectLastCommitted struct { ObjectLocation diff --git a/satellite/metabase/delete_bucket.go b/satellite/metabase/delete_bucket.go index 5db07278c5cb..976144cd7ba9 100644 --- a/satellite/metabase/delete_bucket.go +++ b/satellite/metabase/delete_bucket.go @@ -32,7 +32,6 @@ func (db *DB) DeleteBucketObjects(ctx context.Context, opts DeleteBucketObjects) deleteBatchSizeLimit.Ensure(&opts.BatchSize) - // TODO we may think about doing pending and committed objects in parallel deletedBatchCount := int64(opts.BatchSize) for deletedBatchCount > 0 { if err := ctx.Err(); err != nil { @@ -47,20 +46,6 @@ func (db *DB) DeleteBucketObjects(ctx context.Context, opts DeleteBucketObjects) } } - deletedBatchCount = int64(opts.BatchSize) - for deletedBatchCount > 0 { - if err := ctx.Err(); err != nil { - return deletedObjectCount, err - } - - deletedBatchCount, err = db.deleteBucketPendingObjects(ctx, opts) - deletedObjectCount += deletedBatchCount - - if err != nil { - return deletedObjectCount, err - } - } - return deletedObjectCount, nil } @@ -116,55 +101,3 @@ func (db *DB) deleteBucketObjects(ctx context.Context, opts DeleteBucketObjects) return deletedObjectCount, nil } - -func (db *DB) deleteBucketPendingObjects(ctx context.Context, opts DeleteBucketObjects) (deletedObjectCount int64, err error) { - defer mon.Task()(&ctx)(&err) - - var query string - - // TODO handle number of deleted segments - switch db.impl { - case dbutil.Cockroach: - query = ` - WITH deleted_objects AS ( - DELETE FROM pending_objects - WHERE (project_id, bucket_name) = ($1, $2) - LIMIT $3 - RETURNING pending_objects.stream_id - ), deleted_segments AS ( - DELETE FROM segments - WHERE segments.stream_id IN (SELECT deleted_objects.stream_id FROM deleted_objects) - RETURNING segments.stream_id - ) - SELECT COUNT(1) FROM deleted_objects - ` - case dbutil.Postgres: - query = ` - WITH deleted_objects AS ( - DELETE FROM pending_objects - WHERE stream_id IN ( - SELECT stream_id FROM pending_objects - WHERE (project_id, bucket_name) = ($1, $2) - LIMIT $3 - ) - RETURNING pending_objects.stream_id - ), deleted_segments AS ( - DELETE FROM segments - WHERE segments.stream_id IN (SELECT deleted_objects.stream_id FROM deleted_objects) - RETURNING segments.stream_id - ) - SELECT COUNT(1) FROM deleted_objects - ` - default: - return 0, Error.New("unhandled database: %v", db.impl) - } - - err = db.db.QueryRowContext(ctx, query, opts.Bucket.ProjectID, []byte(opts.Bucket.BucketName), opts.BatchSize).Scan(&deletedObjectCount) - if err != nil { - return 0, Error.Wrap(err) - } - - mon.Meter("object_delete").Mark64(deletedObjectCount) - - return deletedObjectCount, nil -} diff --git a/satellite/metabase/delete_bucket_test.go b/satellite/metabase/delete_bucket_test.go index 4d3402fa4406..1403487aa1aa 100644 --- a/satellite/metabase/delete_bucket_test.go +++ b/satellite/metabase/delete_bucket_test.go @@ -238,53 +238,6 @@ func TestDeleteBucketObjects(t *testing.T) { metabasetest.Verify{}.Check(ctx, t, db) }) - - t.Run("pending and committed objects", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.CreateObject(ctx, t, db, obj1, 2) - - obj1.ObjectKey = "some key" - obj1.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj1, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.DeleteBucketObjects{ - Opts: metabase.DeleteBucketObjects{ - Bucket: obj1.Location().Bucket(), - BatchSize: 2, - }, - Deleted: 2, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - - // object only in pending_objects table - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj1, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.DeleteBucketObjects{ - Opts: metabase.DeleteBucketObjects{ - Bucket: obj1.Location().Bucket(), - BatchSize: 2, - }, - Deleted: 1, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) }) } diff --git a/satellite/metabase/delete_objects.go b/satellite/metabase/delete_objects.go index c234b16a714d..7678b349e38d 100644 --- a/satellite/metabase/delete_objects.go +++ b/satellite/metabase/delete_objects.go @@ -290,119 +290,3 @@ func (db *DB) deleteInactiveObjectsAndSegments(ctx context.Context, objects []Ob return nil } - -// DeleteInactivePendingObjects deletes all pending objects that are inactive. Inactive means that zombie deletion deadline passed -// and no new segmets were uploaded after opts.InactiveDeadline. -func (db *DB) DeleteInactivePendingObjects(ctx context.Context, opts DeleteZombieObjects) (err error) { - defer mon.Task()(&ctx)(&err) - - return db.deleteObjectsAndSegmentsBatch(ctx, opts.BatchSize, func(startAfter ObjectStream, batchsize int) (last ObjectStream, err error) { - defer mon.Task()(&ctx)(&err) - - query := ` - SELECT - project_id, bucket_name, object_key, stream_id - FROM pending_objects - ` + db.impl.AsOfSystemInterval(opts.AsOfSystemInterval) + ` - WHERE - (project_id, bucket_name, object_key, stream_id) > ($1, $2, $3, $4) - AND (zombie_deletion_deadline IS NULL OR zombie_deletion_deadline < $5) - ORDER BY project_id, bucket_name, object_key, stream_id - LIMIT $6;` - - objects := make([]ObjectStream, 0, batchsize) - - scanErrClass := errs.Class("DB rows scan has failed") - err = withRows(db.db.QueryContext(ctx, query, - startAfter.ProjectID, []byte(startAfter.BucketName), []byte(startAfter.ObjectKey), startAfter.StreamID, - opts.DeadlineBefore, - batchsize), - )(func(rows tagsql.Rows) error { - for rows.Next() { - err = rows.Scan(&last.ProjectID, &last.BucketName, &last.ObjectKey, &last.StreamID) - if err != nil { - return scanErrClass.Wrap(err) - } - - db.log.Debug("selected zombie object for deleting it", - zap.Stringer("Project", last.ProjectID), - zap.String("Bucket", last.BucketName), - zap.String("Object Key", string(last.ObjectKey)), - zap.Stringer("StreamID", last.StreamID), - ) - objects = append(objects, last) - } - - return nil - }) - if err != nil { - if scanErrClass.Has(err) { - return ObjectStream{}, Error.New("unable to select zombie objects for deletion: %w", err) - } - - db.log.Warn("unable to select zombie objects for deletion", zap.Error(Error.Wrap(err))) - return ObjectStream{}, nil - } - - err = db.deleteInactiveObjectsAndSegmentsNew(ctx, objects, opts) - if err != nil { - db.log.Warn("delete from DB zombie objects", zap.Error(err)) - return ObjectStream{}, nil - } - - return last, nil - }) -} - -func (db *DB) deleteInactiveObjectsAndSegmentsNew(ctx context.Context, objects []ObjectStream, opts DeleteZombieObjects) (err error) { - defer mon.Task()(&ctx)(&err) - - if len(objects) == 0 { - return nil - } - - err = pgxutil.Conn(ctx, db.db, func(conn *pgx.Conn) error { - var batch pgx.Batch - for _, obj := range objects { - batch.Queue(` - WITH check_segments AS ( - SELECT 1 FROM segments - WHERE stream_id = $4::BYTEA AND created_at > $5 - ), deleted_objects AS ( - DELETE FROM pending_objects - WHERE - (project_id, bucket_name, object_key, stream_id) = ($1::BYTEA, $2::BYTEA, $3::BYTEA, $4) AND - NOT EXISTS (SELECT 1 FROM check_segments) - RETURNING stream_id - ) - DELETE FROM segments - WHERE segments.stream_id IN (SELECT stream_id FROM deleted_objects) - `, obj.ProjectID, []byte(obj.BucketName), []byte(obj.ObjectKey), obj.StreamID, opts.InactiveDeadline) - } - - results := conn.SendBatch(ctx, &batch) - defer func() { err = errs.Combine(err, results.Close()) }() - - var segmentsDeleted int64 - var errlist errs.Group - for i := 0; i < batch.Len(); i++ { - result, err := results.Exec() - errlist.Add(err) - - if err == nil { - segmentsDeleted += result.RowsAffected() - } - } - - // TODO calculate deleted objects - mon.Meter("zombie_segment_delete").Mark64(segmentsDeleted) - mon.Meter("segment_delete").Mark64(segmentsDeleted) - - return errlist.Err() - }) - if err != nil { - return Error.New("unable to delete zombie objects: %w", err) - } - - return nil -} diff --git a/satellite/metabase/delete_objects_test.go b/satellite/metabase/delete_objects_test.go index 7606c1aa31bc..7f09ceab3298 100644 --- a/satellite/metabase/delete_objects_test.go +++ b/satellite/metabase/delete_objects_test.go @@ -482,311 +482,3 @@ func TestDeleteZombieObjects(t *testing.T) { }) }) } - -func TestDeleteInactivePendingObjects(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj1 := metabasetest.RandObjectStream() - obj1.Version = metabase.NextVersion - obj2 := metabasetest.RandObjectStream() - obj2.Version = metabase.NextVersion - obj3 := metabasetest.RandObjectStream() - obj3.Version = metabase.NextVersion - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - pastTime := now.Add(-1 * time.Hour) - futureTime := now.Add(1 * time.Hour) - - t.Run("none", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.DeleteInactivePendingObjects{ - Opts: metabase.DeleteZombieObjects{ - DeadlineBefore: now, - }, - }.Check(ctx, t, db) - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("partial objects", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - // zombie object with default deadline - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj1, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - // zombie object with deadline time in the past - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj2, - ZombieDeletionDeadline: &pastTime, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - // pending object with expiration time in the future - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj3, - ZombieDeletionDeadline: &futureTime, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.DeleteInactivePendingObjects{ - Opts: metabase.DeleteZombieObjects{ - DeadlineBefore: now, - InactiveDeadline: now, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ // the object with zombie deadline time in the past is gone - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj1), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj3), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &futureTime, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("partial object with segment", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj1, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &now, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj1, - RootPieceID: storj.PieceID{1}, - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj1, - RootPieceID: storj.PieceID{1}, - Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}}, - - EncryptedKey: []byte{3}, - EncryptedKeyNonce: []byte{4}, - EncryptedETag: []byte{5}, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - // object will be checked if is inactive but inactive time is in future - metabasetest.DeleteInactivePendingObjects{ - Opts: metabase.DeleteZombieObjects{ - DeadlineBefore: now.Add(1 * time.Hour), - InactiveDeadline: now.Add(-1 * time.Hour), - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj1), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &now, - }, - }, - Segments: []metabase.RawSegment{ - { - StreamID: obj1.StreamID, - RootPieceID: storj.PieceID{1}, - Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}}, - CreatedAt: now, - - EncryptedKey: []byte{3}, - EncryptedKeyNonce: []byte{4}, - EncryptedETag: []byte{5}, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - }, - }, - }.Check(ctx, t, db) - - // object will be checked if is inactive and will be deleted with segment - metabasetest.DeleteInactivePendingObjects{ - Opts: metabase.DeleteZombieObjects{ - DeadlineBefore: now.Add(1 * time.Hour), - InactiveDeadline: now.Add(2 * time.Hour), - AsOfSystemInterval: -1 * time.Microsecond, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("batch size", func(t *testing.T) { - for i := 0; i < 33; i++ { - obj := metabasetest.RandObjectStream() - obj.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - // use default 24h zombie deletion deadline - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - for i := byte(0); i < 3; i++ { - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)}, - RootPieceID: storj.PieceID{i + 1}, - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)}, - RootPieceID: storj.PieceID{1}, - Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}}, - - EncryptedKey: []byte{3}, - EncryptedKeyNonce: []byte{4}, - EncryptedETag: []byte{5}, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - } - } - - metabasetest.DeleteInactivePendingObjects{ - Opts: metabase.DeleteZombieObjects{ - DeadlineBefore: now.Add(25 * time.Hour), - InactiveDeadline: now.Add(48 * time.Hour), - BatchSize: 4, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("committed objects", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj1 := obj1 - obj1.Version = metabase.DefaultVersion - object1, _ := metabasetest.CreateTestObject{}.Run(ctx, t, db, obj1, 1) - - obj2 := obj2 - obj2.Version = metabase.DefaultVersion - object2 := object1 - object2.ObjectStream = obj2 - metabasetest.CreateTestObject{ - BeginObjectExactVersion: &metabase.BeginObjectExactVersion{ - ObjectStream: object2.ObjectStream, - ZombieDeletionDeadline: &pastTime, - Encryption: metabasetest.DefaultEncryption, - }, - }.Run(ctx, t, db, object2.ObjectStream, 1) - - obj3 := obj3 - obj3.Version = metabase.DefaultVersion - object3, _ := metabasetest.CreateTestObject{ - BeginObjectExactVersion: &metabase.BeginObjectExactVersion{ - ObjectStream: obj3, - ZombieDeletionDeadline: &futureTime, - Encryption: metabasetest.DefaultEncryption, - }, - }.Run(ctx, t, db, obj3, 1) - - expectedObj1Segment := metabase.Segment{ - StreamID: obj1.StreamID, - RootPieceID: storj.PieceID{1}, - CreatedAt: now, - EncryptedKey: []byte{3}, - EncryptedKeyNonce: []byte{4}, - EncryptedETag: []byte{5}, - EncryptedSize: 1060, - PlainSize: 512, - Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}}, - Redundancy: metabasetest.DefaultRedundancy, - } - - expectedObj2Segment := expectedObj1Segment - expectedObj2Segment.StreamID = object2.StreamID - expectedObj3Segment := expectedObj1Segment - expectedObj3Segment.StreamID = object3.StreamID - - metabasetest.DeleteInactivePendingObjects{ - Opts: metabase.DeleteZombieObjects{ - DeadlineBefore: now, - InactiveDeadline: now.Add(1 * time.Hour), - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ // all committed objects should NOT be deleted - Objects: []metabase.RawObject{ - metabase.RawObject(object1), - metabase.RawObject(object2), - metabase.RawObject(object3), - }, - Segments: []metabase.RawSegment{ - metabase.RawSegment(expectedObj1Segment), - metabase.RawSegment(expectedObj2Segment), - metabase.RawSegment(expectedObj3Segment), - }, - }.Check(ctx, t, db) - }) - }) -} diff --git a/satellite/metabase/delete_test.go b/satellite/metabase/delete_test.go index b114181a7aee..056a159c282d 100644 --- a/satellite/metabase/delete_test.go +++ b/satellite/metabase/delete_test.go @@ -259,279 +259,6 @@ func TestDeletePendingObject(t *testing.T) { }) } -func TestDeletePendingObjectNew(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj := metabasetest.RandObjectStream() - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - for _, test := range metabasetest.InvalidObjectStreams(obj) { - test := test - t.Run(test.Name, func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: test.ObjectStream, - }, - ErrClass: test.ErrClass, - ErrText: test.ErrText, - }.Check(ctx, t, db) - metabasetest.Verify{}.Check(ctx, t, db) - }) - } - - t.Run("object missing", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: obj, - }, - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "metabase: no rows deleted", - }.Check(ctx, t, db) - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("non existing object version", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.BeginObjectExactVersion{ - Opts: metabase.BeginObjectExactVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - }, - }.Check(ctx, t, db) - - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 33, - StreamID: obj.StreamID, - }, - }, - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "metabase: no rows deleted", - }.Check(ctx, t, db) - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.Pending, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("delete committed object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - object := metabasetest.CreateObject(ctx, t, db, obj, 0) - - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: object.ObjectStream, - }, - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "metabase: no rows deleted", - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.CommittedUnversioned, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("without segments with wrong StreamID", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.BeginObjectExactVersion{ - Opts: metabase.BeginObjectExactVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - }, - }.Check(ctx, t, db) - - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: obj.Version, - StreamID: uuid.UUID{33}, - }, - }, - Result: metabase.DeleteObjectResult{}, - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "metabase: no rows deleted", - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.Pending, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("without segments", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj := obj - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - object := metabase.RawObject{ - ObjectStream: obj, - CreatedAt: now, - Status: metabase.Pending, - Encryption: metabasetest.DefaultEncryption, - } - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: obj, - }, - Result: metabase.DeleteObjectResult{ - Removed: []metabase.Object{metabase.Object(object)}, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("with segments", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj := obj - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - RootPieceID: testrand.PieceID(), - Pieces: metabase.Pieces{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - - EncryptedKey: testrand.Bytes(32), - EncryptedKeyNonce: testrand.Bytes(32), - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: obj, - }, - Result: metabase.DeleteObjectResult{ - Removed: []metabase.Object{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.Pending, - Encryption: metabasetest.DefaultEncryption, - }, - }, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("with inline segment", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj := obj - obj.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - metabasetest.CommitInlineSegment{ - Opts: metabase.CommitInlineSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: 0}, - - EncryptedKey: testrand.Bytes(32), - EncryptedKeyNonce: testrand.Bytes(32), - - InlineData: testrand.Bytes(1024), - - PlainSize: 512, - PlainOffset: 0, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.DeletePendingObjectNew{ - Opts: metabase.DeletePendingObject{ - ObjectStream: obj, - }, - Result: metabase.DeleteObjectResult{ - Removed: []metabase.Object{ - { - ObjectStream: obj, - CreatedAt: now, - Status: metabase.Pending, - Encryption: metabasetest.DefaultEncryption, - }, - }, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - }) -} - func TestDeleteObjectExactVersion(t *testing.T) { metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { obj := metabasetest.RandObjectStream() diff --git a/satellite/metabase/get.go b/satellite/metabase/get.go index ea6bfc007853..3d25a5cde326 100644 --- a/satellite/metabase/get.go +++ b/satellite/metabase/get.go @@ -30,9 +30,6 @@ func (obj *Object) IsMigrated() bool { return obj.TotalPlainSize <= 0 } -// PendingObject pending object metadata. -type PendingObject RawPendingObject - // Segment segment metadata. // TODO define separated struct. type Segment RawSegment @@ -320,9 +317,7 @@ func (db *DB) BucketEmpty(ctx context.Context, opts BucketEmpty) (empty bool, er var value bool err = db.db.QueryRowContext(ctx, ` - SELECT - (SELECT EXISTS (SELECT 1 FROM objects WHERE (project_id, bucket_name) = ($1, $2))) OR - (SELECT EXISTS (SELECT 1 FROM pending_objects WHERE (project_id, bucket_name) = ($1, $2))) + SELECT EXISTS (SELECT 1 FROM objects WHERE (project_id, bucket_name) = ($1, $2)) `, opts.ProjectID, []byte(opts.BucketName)).Scan(&value) if err != nil { return false, Error.New("unable to query objects: %w", err) @@ -404,23 +399,6 @@ func (db *DB) TestingAllObjects(ctx context.Context) (objects []Object, err erro return objects, nil } -// TestingAllPendingObjects gets all pending objects. -// Use only for testing purposes. -func (db *DB) TestingAllPendingObjects(ctx context.Context) (objects []PendingObject, err error) { - defer mon.Task()(&ctx)(&err) - - rawObjects, err := db.testingGetAllPendingObjects(ctx) - if err != nil { - return nil, Error.Wrap(err) - } - - for _, o := range rawObjects { - objects = append(objects, PendingObject(o)) - } - - return objects, nil -} - // TestingAllSegments gets all segments. // Use only for testing purposes. func (db *DB) TestingAllSegments(ctx context.Context) (segments []Segment, err error) { diff --git a/satellite/metabase/get_test.go b/satellite/metabase/get_test.go index 0400e926ce6d..a88849069eb6 100644 --- a/satellite/metabase/get_test.go +++ b/satellite/metabase/get_test.go @@ -1863,60 +1863,5 @@ func TestBucketEmpty(t *testing.T) { }, }.Check(ctx, t, db) }) - - t.Run("object in pending_objects", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.CreatePendingObjectNew(ctx, t, db, obj, 0) - - metabasetest.BucketEmpty{ - Opts: metabase.BucketEmpty{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - }, - Result: false, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("object in pending_objects and in object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - object := metabasetest.CreateObject(ctx, t, db, obj, 0) - - metabasetest.CreatePendingObjectNew(ctx, t, db, obj, 0) - - metabasetest.BucketEmpty{ - Opts: metabase.BucketEmpty{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - }, - Result: false, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(object), - }, - PendingObjects: []metabase.RawPendingObject{ - { - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) }) } diff --git a/satellite/metabase/list.go b/satellite/metabase/list.go index 6610b0ec566d..71e583ce4a02 100644 --- a/satellite/metabase/list.go +++ b/satellite/metabase/list.go @@ -169,23 +169,3 @@ func (opts *IteratePendingObjects) Verify() error { } return nil } - -// IteratePendingObjects iterates through all pending objects. -func (db *DB) IteratePendingObjects(ctx context.Context, opts IteratePendingObjects, fn func(context.Context, PendingObjectsIterator) error) (err error) { - defer mon.Task()(&ctx)(&err) - if err = opts.Verify(); err != nil { - return err - } - return iterateAllPendingObjects(ctx, db, opts, fn) -} - -// IteratePendingObjectsByKeyNew iterates through all streams of pending objects with the same ObjectKey. -// TODO should be refactored to IteratePendingObjectsByKey after full transition to pending_objects table. -func (db *DB) IteratePendingObjectsByKeyNew(ctx context.Context, opts IteratePendingObjectsByKey, fn func(context.Context, PendingObjectsIterator) error) (err error) { - defer mon.Task()(&ctx)(&err) - - if err := opts.Verify(); err != nil { - return err - } - return iteratePendingObjectsByKeyNew(ctx, db, opts, fn) -} diff --git a/satellite/metabase/metabasetest/common.go b/satellite/metabase/metabasetest/common.go index c1825511bd58..c5e59a070173 100644 --- a/satellite/metabase/metabasetest/common.go +++ b/satellite/metabase/metabasetest/common.go @@ -36,8 +36,6 @@ func (step Verify) Check(ctx *testcontext.Context, t testing.TB, db *metabase.DB sortRawObjects(state.Objects) sortRawObjects(step.Objects) - sortRawPendingObjects(state.PendingObjects) - sortRawPendingObjects(step.PendingObjects) sortRawSegments(state.Segments) sortRawSegments(step.Segments) @@ -68,12 +66,6 @@ func sortRawObjects(objects []metabase.RawObject) { }) } -func sortRawPendingObjects(objects []metabase.RawPendingObject) { - sort.Slice(objects, func(i, j int) bool { - return objects[i].StreamID.Less(objects[j].StreamID) - }) -} - func sortRawSegments(segments []metabase.RawSegment) { sort.Slice(segments, func(i, j int) bool { if segments[i].StreamID == segments[j].StreamID { diff --git a/satellite/metabase/metabasetest/create.go b/satellite/metabase/metabasetest/create.go index f34f71c13418..56be47263ec8 100644 --- a/satellite/metabase/metabasetest/create.go +++ b/satellite/metabase/metabasetest/create.go @@ -54,54 +54,6 @@ func CreatePendingObject(ctx *testcontext.Context, t testing.TB, db *metabase.DB return object } -// CreatePendingObjectNew creates a new pending object with the specified number of segments. -// TODO CreatePendingObject will be removed when transition to pending_objects table will be complete. -func CreatePendingObjectNew(ctx *testcontext.Context, t testing.TB, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte) { - obj.Version = metabase.NextVersion - BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - for i := byte(0); i < numberOfSegments; i++ { - BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)}, - RootPieceID: storj.PieceID{i + 1}, - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)}, - RootPieceID: storj.PieceID{1}, - Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}}, - - EncryptedKey: []byte{3}, - EncryptedKeyNonce: []byte{4}, - EncryptedETag: []byte{5}, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - } -} - // CreateObject creates a new committed object with the specified number of segments. func CreateObject(ctx *testcontext.Context, t testing.TB, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte) metabase.Object { CreatePendingObject(ctx, t, db, obj, numberOfSegments) diff --git a/satellite/metabase/metabasetest/test.go b/satellite/metabase/metabasetest/test.go index c6edae322dfa..875475391c20 100644 --- a/satellite/metabase/metabasetest/test.go +++ b/satellite/metabase/metabasetest/test.go @@ -468,21 +468,6 @@ func (step DeletePendingObject) Check(ctx *testcontext.Context, t testing.TB, db compareDeleteObjectResult(t, result, step.Result) } -// DeletePendingObjectNew is for testing metabase.DeletePendingObjectNew. -type DeletePendingObjectNew struct { - Opts metabase.DeletePendingObject - Result metabase.DeleteObjectResult - ErrClass *errs.Class - ErrText string -} - -// Check runs the test. -func (step DeletePendingObjectNew) Check(ctx *testcontext.Context, t testing.TB, db *metabase.DB) { - result, err := db.DeletePendingObjectNew(ctx, step.Opts) - checkError(t, err, step.ErrClass, step.ErrText) - compareDeleteObjectResult(t, result, step.Result) -} - // DeleteObjectsAllVersions is for testing metabase.DeleteObjectsAllVersions. type DeleteObjectsAllVersions struct { Opts metabase.DeleteObjectsAllVersions @@ -526,20 +511,6 @@ func (step DeleteZombieObjects) Check(ctx *testcontext.Context, t testing.TB, db checkError(t, err, step.ErrClass, step.ErrText) } -// DeleteInactivePendingObjects is for testing metabase.DeleteInactivePendingObjects. -type DeleteInactivePendingObjects struct { - Opts metabase.DeleteZombieObjects - - ErrClass *errs.Class - ErrText string -} - -// Check runs the test. -func (step DeleteInactivePendingObjects) Check(ctx *testcontext.Context, t testing.TB, db *metabase.DB) { - err := db.DeleteInactivePendingObjects(ctx, step.Opts) - checkError(t, err, step.ErrClass, step.ErrText) -} - // IterateCollector is for testing metabase.IterateCollector. type IterateCollector []metabase.ObjectEntry @@ -601,28 +572,6 @@ func (step IteratePendingObjectsByKey) Check(ctx *testcontext.Context, t *testin require.Zero(t, diff) } -// IteratePendingObjectsByKeyNew is for testing metabase.IteratePendingObjectsByKeyNew. -type IteratePendingObjectsByKeyNew struct { - Opts metabase.IteratePendingObjectsByKey - - Result []metabase.PendingObjectEntry - ErrClass *errs.Class - ErrText string -} - -// Check runs the test. -func (step IteratePendingObjectsByKeyNew) Check(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - var collector PendingObjectsCollector - - err := db.IteratePendingObjectsByKeyNew(ctx, step.Opts, collector.Add) - checkError(t, err, step.ErrClass, step.ErrText) - - result := []metabase.PendingObjectEntry(collector) - - diff := cmp.Diff(step.Result, result, DefaultTimeDiff()) - require.Zero(t, diff) -} - // IterateObjectsWithStatus is for testing metabase.IterateObjectsWithStatus. type IterateObjectsWithStatus struct { Opts metabase.IterateObjectsWithStatus @@ -643,26 +592,6 @@ func (step IterateObjectsWithStatus) Check(ctx *testcontext.Context, t testing.T require.Zero(t, diff) } -// IteratePendingObjects is for testing metabase.IteratePendingObjects. -type IteratePendingObjects struct { - Opts metabase.IteratePendingObjects - - Result []metabase.PendingObjectEntry - ErrClass *errs.Class - ErrText string -} - -// Check runs the test. -func (step IteratePendingObjects) Check(ctx *testcontext.Context, t testing.TB, db *metabase.DB) { - var result PendingObjectsCollector - - err := db.IteratePendingObjects(ctx, step.Opts, result.Add) - checkError(t, err, step.ErrClass, step.ErrText) - - diff := cmp.Diff(step.Result, []metabase.PendingObjectEntry(result), DefaultTimeDiff()) - require.Zero(t, diff) -} - // IterateLoopObjects is for testing metabase.IterateLoopObjects. type IterateLoopObjects struct { Opts metabase.IterateLoopObjects diff --git a/satellite/metabase/pending_objects_iterator.go b/satellite/metabase/pending_objects_iterator.go deleted file mode 100644 index 53f53b61fbae..000000000000 --- a/satellite/metabase/pending_objects_iterator.go +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright (C) 2023 Storj Labs, Inc. -// See LICENSE for copying information. - -package metabase - -import ( - "bytes" - "context" - "strings" - - "github.com/zeebo/errs" - - "storj.io/common/uuid" - "storj.io/private/tagsql" -) - -// pendingObjectsIterator enables iteration on pending objects in a bucket. -type pendingObjectsIterator struct { - db *DB - - projectID uuid.UUID - bucketName []byte - prefix ObjectKey - prefixLimit ObjectKey - batchSize int - recursive bool - includeCustomMetadata bool - includeSystemMetadata bool - - curIndex int - curRows tagsql.Rows - cursor pendingObjectIterateCursor // not relative to prefix - - skipPrefix ObjectKey // relative to prefix - doNextQuery func(context.Context, *pendingObjectsIterator) (_ tagsql.Rows, err error) - - // failErr is set when either scan or next query fails during iteration. - failErr error -} - -type pendingObjectIterateCursor struct { - Key ObjectKey - StreamID uuid.UUID - Inclusive bool -} - -func iterateAllPendingObjects(ctx context.Context, db *DB, opts IteratePendingObjects, fn func(context.Context, PendingObjectsIterator) error) (err error) { - defer mon.Task()(&ctx)(&err) - - it := &pendingObjectsIterator{ - db: db, - - projectID: opts.ProjectID, - bucketName: []byte(opts.BucketName), - prefix: opts.Prefix, - prefixLimit: prefixLimit(opts.Prefix), - batchSize: opts.BatchSize, - recursive: opts.Recursive, - includeCustomMetadata: opts.IncludeCustomMetadata, - includeSystemMetadata: opts.IncludeSystemMetadata, - - curIndex: 0, - cursor: firstPendingObjectIterateCursor(opts.Recursive, opts.Cursor, opts.Prefix), - - doNextQuery: doNextQueryAllPendingObjects, - } - - // start from either the cursor or prefix, depending on which is larger - if lessKey(it.cursor.Key, opts.Prefix) { - it.cursor.Key = opts.Prefix - it.cursor.Inclusive = true - } - - return iteratePendingObjects(ctx, it, fn) -} - -func iteratePendingObjects(ctx context.Context, it *pendingObjectsIterator, fn func(context.Context, PendingObjectsIterator) error) (err error) { - batchsizeLimit.Ensure(&it.batchSize) - - it.curRows, err = it.doNextQuery(ctx, it) - if err != nil { - return err - } - it.cursor.Inclusive = false - - defer func() { - if rowsErr := it.curRows.Err(); rowsErr != nil { - err = errs.Combine(err, rowsErr) - } - err = errs.Combine(err, it.failErr, it.curRows.Close()) - }() - - return fn(ctx, it) -} - -// Next returns true if there was another item and copy it in item. -func (it *pendingObjectsIterator) Next(ctx context.Context, item *PendingObjectEntry) bool { - if it.recursive { - return it.next(ctx, item) - } - - // TODO: implement this on the database side - - // skip until we are past the prefix we returned before. - if it.skipPrefix != "" { - for strings.HasPrefix(string(item.ObjectKey), string(it.skipPrefix)) { - if !it.next(ctx, item) { - return false - } - } - it.skipPrefix = "" - } else { - ok := it.next(ctx, item) - if !ok { - return false - } - } - - // should this be treated as a prefix? - p := strings.IndexByte(string(item.ObjectKey), Delimiter) - if p >= 0 { - it.skipPrefix = item.ObjectKey[:p+1] - *item = PendingObjectEntry{ - IsPrefix: true, - ObjectKey: item.ObjectKey[:p+1], - } - } - - return true -} - -// next returns true if there was another item and copy it in item. -func (it *pendingObjectsIterator) next(ctx context.Context, item *PendingObjectEntry) bool { - next := it.curRows.Next() - if !next { - if it.curIndex < it.batchSize { - return false - } - - if it.curRows.Err() != nil { - return false - } - - if !it.recursive { - afterPrefix := it.cursor.Key[len(it.prefix):] - p := bytes.IndexByte([]byte(afterPrefix), Delimiter) - if p >= 0 { - it.cursor.Key = it.prefix + prefixLimit(afterPrefix[:p+1]) - it.cursor.StreamID = uuid.UUID{} - } - } - - rows, err := it.doNextQuery(ctx, it) - if err != nil { - it.failErr = errs.Combine(it.failErr, err) - return false - } - - if closeErr := it.curRows.Close(); closeErr != nil { - it.failErr = errs.Combine(it.failErr, closeErr, rows.Close()) - return false - } - - it.curRows = rows - it.curIndex = 0 - if !it.curRows.Next() { - return false - } - } - - err := it.scanItem(item) - if err != nil { - it.failErr = errs.Combine(it.failErr, err) - return false - } - - it.curIndex++ - it.cursor.Key = it.prefix + item.ObjectKey - it.cursor.StreamID = item.StreamID - - return true -} - -func doNextQueryAllPendingObjects(ctx context.Context, it *pendingObjectsIterator) (_ tagsql.Rows, err error) { - defer mon.Task()(&ctx)(&err) - - cursorCompare := ">" - if it.cursor.Inclusive { - cursorCompare = ">=" - } - - if it.prefixLimit == "" { - querySelectFields := pendingObjectsQuerySelectorFields("object_key", it) - return it.db.db.QueryContext(ctx, ` - SELECT - `+querySelectFields+` - FROM pending_objects - WHERE - (project_id, bucket_name, object_key, stream_id) `+cursorCompare+` ($1, $2, $3, $4) - AND (project_id, bucket_name) < ($1, $6) - AND (expires_at IS NULL OR expires_at > now()) - ORDER BY (project_id, bucket_name, object_key, stream_id) ASC - LIMIT $5 - `, it.projectID, it.bucketName, - []byte(it.cursor.Key), it.cursor.StreamID, - it.batchSize, - nextBucket(it.bucketName), - ) - } - - fromSubstring := 1 - if it.prefix != "" { - fromSubstring = len(it.prefix) + 1 - } - - querySelectFields := pendingObjectsQuerySelectorFields("SUBSTRING(object_key FROM $7)", it) - return it.db.db.QueryContext(ctx, ` - SELECT - `+querySelectFields+` - FROM pending_objects - WHERE - (project_id, bucket_name, object_key, stream_id) `+cursorCompare+` ($1, $2, $3, $4) - AND (project_id, bucket_name, object_key) < ($1, $2, $5) - AND (expires_at IS NULL OR expires_at > now()) - ORDER BY (project_id, bucket_name, object_key, stream_id) ASC - LIMIT $6 - `, it.projectID, it.bucketName, - []byte(it.cursor.Key), it.cursor.StreamID, - []byte(it.prefixLimit), - it.batchSize, - fromSubstring, - ) -} - -func pendingObjectsQuerySelectorFields(objectKeyColumn string, it *pendingObjectsIterator) string { - querySelectFields := objectKeyColumn + ` - ,stream_id - ,encryption` - - if it.includeSystemMetadata { - querySelectFields += ` - ,created_at - ,expires_at` - } - - if it.includeCustomMetadata { - querySelectFields += ` - ,encrypted_metadata_nonce - ,encrypted_metadata - ,encrypted_metadata_encrypted_key` - } - - return querySelectFields -} - -// scanItem scans doNextQuery results into PendingObjectEntry. -func (it *pendingObjectsIterator) scanItem(item *PendingObjectEntry) (err error) { - item.IsPrefix = false - - fields := []interface{}{ - &item.ObjectKey, - &item.StreamID, - encryptionParameters{&item.Encryption}, - } - - if it.includeSystemMetadata { - fields = append(fields, - &item.CreatedAt, - &item.ExpiresAt, - ) - } - - if it.includeCustomMetadata { - fields = append(fields, - &item.EncryptedMetadataNonce, - &item.EncryptedMetadata, - &item.EncryptedMetadataEncryptedKey, - ) - } - - return it.curRows.Scan(fields...) -} - -// firstPendingObjectIterateCursor adjust the cursor for a non-recursive iteration. -// The cursor is non-inclusive and we need to adjust to handle prefix as cursor properly. -// We return the next possible key from the prefix. -func firstPendingObjectIterateCursor(recursive bool, cursor PendingObjectsCursor, prefix ObjectKey) pendingObjectIterateCursor { - if recursive { - return pendingObjectIterateCursor{ - Key: cursor.Key, - StreamID: cursor.StreamID, - } - } - - // when the cursor does not match the prefix, we'll return the original cursor. - if !strings.HasPrefix(string(cursor.Key), string(prefix)) { - return pendingObjectIterateCursor{ - Key: cursor.Key, - StreamID: cursor.StreamID, - } - } - - // handle case where: - // prefix: x/y/ - // cursor: x/y/z/w - // In this case, we want the skip prefix to be `x/y/z` + string('/' + 1). - - cursorWithoutPrefix := cursor.Key[len(prefix):] - p := strings.IndexByte(string(cursorWithoutPrefix), Delimiter) - if p < 0 { - // The cursor is not a prefix, but instead a path inside the prefix, - // so we can use it directly. - return pendingObjectIterateCursor{ - Key: cursor.Key, - StreamID: cursor.StreamID, - } - } - - // return the next prefix given a scoped path - return pendingObjectIterateCursor{ - Key: cursor.Key[:len(prefix)+p] + ObjectKey(Delimiter+1), - StreamID: cursor.StreamID, - Inclusive: true, - } -} - -func iteratePendingObjectsByKeyNew(ctx context.Context, db *DB, opts IteratePendingObjectsByKey, fn func(context.Context, PendingObjectsIterator) error) (err error) { - defer mon.Task()(&ctx)(&err) - - cursor := opts.Cursor - - if cursor.StreamID.IsZero() { - cursor.StreamID = uuid.UUID{} - } - - it := &pendingObjectsIterator{ - db: db, - - projectID: opts.ProjectID, - bucketName: []byte(opts.BucketName), - prefix: "", - prefixLimit: "", - batchSize: opts.BatchSize, - recursive: true, - includeCustomMetadata: true, - includeSystemMetadata: true, - - curIndex: 0, - cursor: pendingObjectIterateCursor{ - Key: opts.ObjectKey, - StreamID: opts.Cursor.StreamID, - }, - doNextQuery: doNextQueryPendingStreamsByKey, - } - - return iteratePendingObjects(ctx, it, fn) - -} - -func doNextQueryPendingStreamsByKey(ctx context.Context, it *pendingObjectsIterator) (_ tagsql.Rows, err error) { - defer mon.Task()(&ctx)(&err) - - return it.db.db.QueryContext(ctx, ` - SELECT - object_key, stream_id, encryption, - created_at, expires_at, - encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key - FROM pending_objects - WHERE - (project_id, bucket_name, object_key) = ($1, $2, $3) AND - stream_id > $4::BYTEA - ORDER BY stream_id ASC - LIMIT $5 - `, it.projectID, it.bucketName, - []byte(it.cursor.Key), - it.cursor.StreamID, - it.batchSize, - ) -} diff --git a/satellite/metabase/pending_objects_iterator_test.go b/satellite/metabase/pending_objects_iterator_test.go deleted file mode 100644 index 542e612990ce..000000000000 --- a/satellite/metabase/pending_objects_iterator_test.go +++ /dev/null @@ -1,1283 +0,0 @@ -// Copyright (C) 2023 Storj Labs, Inc. -// See LICENSE for copying information. - -package metabase_test - -import ( - "sort" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "storj.io/common/storj" - "storj.io/common/testcontext" - "storj.io/common/testrand" - "storj.io/common/uuid" - "storj.io/storj/satellite/metabase" - "storj.io/storj/satellite/metabase/metabasetest" -) - -func TestIteratePendingObjects(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - t.Run("invalid arguments", func(t *testing.T) { - t.Run("ProjectID missing", func(t *testing.T) { - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{}, - BucketName: "sj://mybucket", - Recursive: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "ProjectID missing", - }.Check(ctx, t, db) - }) - t.Run("BucketName missing", func(t *testing.T) { - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{1}, - BucketName: "", - Recursive: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "BucketName missing", - }.Check(ctx, t, db) - }) - t.Run("Limit is negative", func(t *testing.T) { - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{1}, - BucketName: "mybucket", - BatchSize: -1, - Recursive: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "BatchSize is negative", - }.Check(ctx, t, db) - }) - }) - - t.Run("empty bucket", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - objects := createObjects(ctx, t, db, 2, uuid.UUID{1}, "mybucket") - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{1}, - BucketName: "myemptybucket", - BatchSize: 10, - Recursive: true, - }, - Result: nil, - }.Check(ctx, t, db) - metabasetest.Verify{Objects: objects}.Check(ctx, t, db) - }) - - t.Run("success", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - - pending := metabasetest.RandObjectStream() - pending.Version = metabase.NextVersion - committed := metabasetest.RandObjectStream() - committed.ProjectID = pending.ProjectID - committed.BucketName = pending.BucketName - committed.Version = metabase.NextVersion - - projectID := pending.ProjectID - bucketName := pending.BucketName - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: pending, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - encryptedMetadata := testrand.Bytes(1024) - encryptedMetadataNonce := testrand.Nonce() - encryptedMetadataKey := testrand.Bytes(265) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: committed, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - committed.Version++ - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: committed, - OverrideEncryptedMetadata: true, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataEncryptedKey: encryptedMetadataKey, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - // IteratePendingObjects should find only pending object - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, - Result: []metabase.PendingObjectEntry{{ - ObjectKey: pending.ObjectKey, - StreamID: pending.StreamID, - CreatedAt: now, - Encryption: metabasetest.DefaultEncryption, - }}, - }.Check(ctx, t, db) - }) - - t.Run("less objects than limit", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - numberOfObjects := 3 - limit := 10 - expected := make([]metabase.PendingObjectEntry, numberOfObjects) - expectedObjects := createPendingObjects(ctx, t, db, numberOfObjects, 0, uuid.UUID{1}, "mybucket") - for i, object := range expectedObjects { - expected[i] = pendingObjectEntryFromRaw(object) - } - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{1}, - BucketName: "mybucket", - Recursive: true, - BatchSize: limit, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, - Result: expected, - }.Check(ctx, t, db) - metabasetest.Verify{PendingObjects: expectedObjects}.Check(ctx, t, db) - }) - - t.Run("more objects than limit", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - numberOfObjects := 10 - limit := 3 - expected := make([]metabase.PendingObjectEntry, numberOfObjects) - expectedObjects := createPendingObjects(ctx, t, db, numberOfObjects, 0, uuid.UUID{1}, "mybucket") - for i, object := range expectedObjects { - expected[i] = pendingObjectEntryFromRaw(object) - } - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{1}, - BucketName: "mybucket", - Recursive: true, - BatchSize: limit, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, - Result: expected, - }.Check(ctx, t, db) - metabasetest.Verify{PendingObjects: expectedObjects}.Check(ctx, t, db) - }) - - t.Run("objects in one bucket in project with 2 buckets", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - numberOfObjectsPerBucket := 5 - - expected := make([]metabase.PendingObjectEntry, numberOfObjectsPerBucket) - - objectsBucketA := createPendingObjects(ctx, t, db, numberOfObjectsPerBucket, 0, uuid.UUID{1}, "bucket-a") - objectsBucketB := createPendingObjects(ctx, t, db, numberOfObjectsPerBucket, 0, uuid.UUID{1}, "bucket-b") - for i, obj := range objectsBucketA { - expected[i] = pendingObjectEntryFromRaw(obj) - } - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{1}, - BucketName: "bucket-a", - Recursive: true, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, - Result: expected, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: append(objectsBucketA, objectsBucketB...), - }.Check(ctx, t, db) - }) - - t.Run("objects in one bucket with same bucketName in another project", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - numberOfObjectsPerBucket := 5 - - expected := make([]metabase.PendingObjectEntry, numberOfObjectsPerBucket) - - objectsProject1 := createPendingObjects(ctx, t, db, numberOfObjectsPerBucket, 0, uuid.UUID{1}, "mybucket") - objectsProject2 := createPendingObjects(ctx, t, db, numberOfObjectsPerBucket, 0, uuid.UUID{2}, "mybucket") - for i, obj := range objectsProject1 { - expected[i] = pendingObjectEntryFromRaw(obj) - } - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: uuid.UUID{1}, - BucketName: "mybucket", - Recursive: true, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, - Result: expected, - }.Check(ctx, t, db) - - metabasetest.Verify{ - PendingObjects: append(objectsProject1, objectsProject2...), - }.Check(ctx, t, db) - }) - - t.Run("recursive", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - projectID, bucketName := uuid.UUID{1}, "bucky" - - objects := createPendingObjectsWithKeys(ctx, t, db, projectID, bucketName, []metabase.ObjectKey{ - "a", - "b/1", - "b/2", - "b/3", - "c", - "c/", - "c//", - "c/1", - "g", - }) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, - Result: []metabase.PendingObjectEntry{ - objects["a"], - objects["b/1"], - objects["b/2"], - objects["b/3"], - objects["c"], - objects["c/"], - objects["c//"], - objects["c/1"], - objects["g"], - }, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Cursor: metabase.PendingObjectsCursor{Key: "a", StreamID: uuid.Max()}, - }, - Result: []metabase.PendingObjectEntry{ - objects["b/1"], - objects["b/2"], - objects["b/3"], - objects["c"], - objects["c/"], - objects["c//"], - objects["c/1"], - objects["g"], - }, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Cursor: metabase.PendingObjectsCursor{Key: "b"}, - }, - Result: []metabase.PendingObjectEntry{ - objects["b/1"], - objects["b/2"], - objects["b/3"], - objects["c"], - objects["c/"], - objects["c//"], - objects["c/1"], - objects["g"], - }, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - }, - Result: pendingWithoutPrefix("b/", - objects["b/1"], - objects["b/2"], - objects["b/3"], - ), - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - Cursor: metabase.PendingObjectsCursor{Key: "a"}, - }, - Result: pendingWithoutPrefix("b/", - objects["b/1"], - objects["b/2"], - objects["b/3"], - ), - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - Cursor: metabase.PendingObjectsCursor{Key: "b/2", StreamID: uuid.UUID{1}}, - }, - Result: pendingWithoutPrefix("b/", - objects["b/2"], - objects["b/3"], - ), - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - Cursor: metabase.PendingObjectsCursor{Key: "c/"}, - }, - Result: nil, - }.Check(ctx, t, db) - }) - - t.Run("non-recursive", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - projectID, bucketName := uuid.UUID{1}, "bucky" - - objects := createPendingObjectsWithKeys(ctx, t, db, projectID, bucketName, []metabase.ObjectKey{ - "a", - "b/1", - "b/2", - "b/3", - "c", - "c/", - "c//", - "c/1", - "g", - }) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, - Result: []metabase.PendingObjectEntry{ - objects["a"], - pendingPrefixEntry("b/"), - objects["c"], - pendingPrefixEntry("c/"), - objects["g"], - }, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Cursor: metabase.PendingObjectsCursor{Key: "a", StreamID: uuid.Max()}, - }, - Result: []metabase.PendingObjectEntry{ - pendingPrefixEntry("b/"), - objects["c"], - pendingPrefixEntry("c/"), - objects["g"], - }, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Cursor: metabase.PendingObjectsCursor{Key: "b"}, - }, - Result: []metabase.PendingObjectEntry{ - pendingPrefixEntry("b/"), - objects["c"], - pendingPrefixEntry("c/"), - objects["g"], - }, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - }, - Result: pendingWithoutPrefix("b/", - objects["b/1"], - objects["b/2"], - objects["b/3"], - ), - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - Cursor: metabase.PendingObjectsCursor{Key: "a"}, - }, - Result: pendingWithoutPrefix("b/", - objects["b/1"], - objects["b/2"], - objects["b/3"], - ), - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - Cursor: metabase.PendingObjectsCursor{Key: "b/2", StreamID: uuid.UUID{15: 1}}, - }, - Result: pendingWithoutPrefix("b/", - objects["b/2"], - objects["b/3"], - ), - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "b/", - Cursor: metabase.PendingObjectsCursor{Key: "c/"}, - }, - Result: nil, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "c/", - Cursor: metabase.PendingObjectsCursor{Key: "c/", StreamID: uuid.UUID{15: 1}}, - }, - Result: pendingWithoutPrefix("c/", - objects["c/"], - pendingPrefixEntry("c//"), - objects["c/1"], - ), - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - - Prefix: "c//", - }, - Result: pendingWithoutPrefix("c//", - objects["c//"], - ), - }.Check(ctx, t, db) - }) - - t.Run("boundaries", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - projectID, bucketName := uuid.UUID{1}, "bucky" - - queries := []metabase.ObjectKey{""} - for a := 0; a <= 0xFF; a++ { - if 3 < a && a < 252 { - continue - } - queries = append(queries, metabase.ObjectKey([]byte{byte(a)})) - for b := 0; b <= 0xFF; b++ { - if 4 < b && b < 251 { - continue - } - queries = append(queries, metabase.ObjectKey([]byte{byte(a), byte(b)})) - } - } - - createPendingObjectsWithKeys(ctx, t, db, projectID, bucketName, queries[1:]) - - var collector metabasetest.PendingObjectsCollector - for _, cursor := range queries { - for _, prefix := range queries { - collector = collector[:0] - err := db.IteratePendingObjects(ctx, metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Cursor: metabase.PendingObjectsCursor{ - Key: cursor, - StreamID: uuid.UUID{}, - }, - Prefix: prefix, - IncludeCustomMetadata: true, - }, collector.Add) - require.NoError(t, err) - - collector = collector[:0] - err = db.IteratePendingObjects(ctx, metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Cursor: metabase.PendingObjectsCursor{ - Key: cursor, - StreamID: uuid.UUID{}, - }, - Prefix: prefix, - Recursive: true, - IncludeCustomMetadata: true, - }, collector.Add) - require.NoError(t, err) - } - } - }) - - t.Run("verify-iterator-boundary", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - projectID, bucketName := uuid.UUID{1}, "bucky" - queries := []metabase.ObjectKey{"\x00\xFF"} - createObjectsWithKeys(ctx, t, db, projectID, bucketName, queries) - var collector metabasetest.PendingObjectsCollector - err := db.IteratePendingObjects(ctx, metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Cursor: metabase.PendingObjectsCursor{ - Key: metabase.ObjectKey([]byte{}), - StreamID: uuid.UUID{}, - }, - Prefix: metabase.ObjectKey([]byte{1}), - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, collector.Add) - require.NoError(t, err) - }) - - t.Run("include/exclude custom metadata", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj1 := metabasetest.RandObjectStream() - obj1.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj1, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: []byte{3}, - EncryptedMetadataEncryptedKey: []byte{4}, - EncryptedMetadataNonce: []byte{5}, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - // include custom metadata - var collector metabasetest.PendingObjectsCollector - err := db.IteratePendingObjects(ctx, metabase.IteratePendingObjects{ - ProjectID: obj1.ProjectID, - BucketName: obj1.BucketName, - Recursive: true, - IncludeCustomMetadata: true, - IncludeSystemMetadata: true, - }, collector.Add) - require.NoError(t, err) - require.Len(t, collector, 1) - - for _, entry := range collector { - require.Equal(t, entry.EncryptedMetadata, []byte{3}) - require.Equal(t, entry.EncryptedMetadataEncryptedKey, []byte{4}) - require.Equal(t, entry.EncryptedMetadataNonce, []byte{5}) - } - - // exclude custom metadata - collector = collector[:0] - err = db.IteratePendingObjects(ctx, metabase.IteratePendingObjects{ - ProjectID: obj1.ProjectID, - BucketName: obj1.BucketName, - Recursive: true, - IncludeCustomMetadata: false, - IncludeSystemMetadata: true, - }, collector.Add) - require.NoError(t, err) - require.Len(t, collector, 1) - - for _, entry := range collector { - require.Nil(t, entry.EncryptedMetadataNonce) - require.Nil(t, entry.EncryptedMetadata) - require.Nil(t, entry.EncryptedMetadataEncryptedKey) - } - }) - - t.Run("exclude system metadata", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj1 := metabasetest.RandObjectStream() - obj1.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj1, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: []byte{3}, - EncryptedMetadataEncryptedKey: []byte{4}, - EncryptedMetadataNonce: []byte{5}, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - var collector metabasetest.PendingObjectsCollector - err := db.IteratePendingObjects(ctx, metabase.IteratePendingObjects{ - ProjectID: obj1.ProjectID, - BucketName: obj1.BucketName, - Recursive: true, - IncludeCustomMetadata: true, - IncludeSystemMetadata: false, - }, collector.Add) - - require.NoError(t, err) - require.Len(t, collector, 1) - - for _, entry := range collector { - // fields that should always be set - require.NotEmpty(t, entry.ObjectKey) - require.NotEmpty(t, entry.StreamID) - require.False(t, entry.Encryption.IsZero()) - - require.True(t, entry.CreatedAt.IsZero()) - require.Nil(t, entry.ExpiresAt) - - require.NotNil(t, entry.EncryptedMetadataNonce) - require.NotNil(t, entry.EncryptedMetadata) - require.NotNil(t, entry.EncryptedMetadataEncryptedKey) - } - }) - - t.Run("verify-cursor-continuation", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - projectID, bucketName := uuid.UUID{1}, "bucky" - createPendingObjectsWithKeys(ctx, t, db, projectID, bucketName, []metabase.ObjectKey{ - "1", - "a/a", - "a/0", - }) - var collector metabasetest.PendingObjectsCollector - err := db.IteratePendingObjects(ctx, metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Prefix: metabase.ObjectKey("a/"), - BatchSize: 1, - }, collector.Add) - require.NoError(t, err) - require.Len(t, collector, 2) - }) - - t.Run("skip-expired-objects", func(t *testing.T) { - now := time.Now() - type test struct { - notExpired []metabase.ObjectKey - expired []metabase.ObjectKey - } - testCases := []test{ - { - notExpired: []metabase.ObjectKey{"1"}, - expired: []metabase.ObjectKey{"2"}, - }, - { - notExpired: []metabase.ObjectKey{"2"}, - expired: []metabase.ObjectKey{"1"}, - }, - { - notExpired: []metabase.ObjectKey{"2"}, - expired: []metabase.ObjectKey{"1", "3"}, - }, - { - notExpired: []metabase.ObjectKey{"2", "4"}, - expired: []metabase.ObjectKey{"1", "3"}, - }, - { - expired: []metabase.ObjectKey{"1", "2", "3", "4"}, - }, - } - - stream := metabase.ObjectStream{ - ProjectID: testrand.UUID(), - BucketName: "bucket", - } - for i, tc := range testCases { - tc := tc - t.Run(strconv.Itoa(i), func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - expectedResult := []metabase.PendingObjectEntry{} - if len(tc.notExpired) == 0 { - expectedResult = nil - } - for _, key := range tc.notExpired { - stream.ObjectKey = key - stream.StreamID = testrand.UUID() - metabasetest.CreatePendingObjectNew(ctx, t, db, stream, 0) - expectedResult = append(expectedResult, metabase.PendingObjectEntry{ - ObjectKey: key, - StreamID: stream.StreamID, - CreatedAt: now, - Encryption: metabasetest.DefaultEncryption, - }) - } - for _, key := range tc.expired { - stream := stream - stream.ObjectKey = key - stream.StreamID = testrand.UUID() - stream.Version = metabase.NextVersion - - expiresAt := now.Add(-2 * time.Hour) - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: stream, - Encryption: metabasetest.DefaultEncryption, - - ExpiresAt: &expiresAt, - - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - } - for _, batchSize := range []int{1, 2, 3} { - opts := metabase.IteratePendingObjects{ - ProjectID: stream.ProjectID, - BucketName: stream.BucketName, - BatchSize: batchSize, - IncludeSystemMetadata: true, - } - metabasetest.IteratePendingObjects{ - Opts: opts, - Result: expectedResult, - }.Check(ctx, t, db) - { - opts := opts - opts.Recursive = true - metabasetest.IteratePendingObjects{ - Opts: opts, - Result: expectedResult, - }.Check(ctx, t, db) - } - } - }) - } - }) - - t.Run("prefix longer than key", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - projectID, bucketName := uuid.UUID{1}, "bucky" - objects := createPendingObjectsWithKeys(ctx, t, db, projectID, bucketName, []metabase.ObjectKey{ - "aaaa/a", - "aaaa/b", - "aaaa/c", - }) - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: false, - Prefix: "aaaa/", - BatchSize: 2, - IncludeSystemMetadata: true, - }, - Result: pendingWithoutPrefix("aaaa/", - objects["aaaa/a"], - objects["aaaa/b"], - objects["aaaa/c"], - ), - }.Check(ctx, t, db) - }) - - t.Run("two objects the same object key", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - projectID, bucketName := uuid.UUID{2}, "bucky" - - id1 := metabasetest.RandObjectStream() - id1.ProjectID = projectID - id1.BucketName = bucketName - - id2 := metabasetest.RandObjectStream() - id2.ProjectID = projectID - id2.BucketName = bucketName - id2.ObjectKey = id1.ObjectKey - - if id2.StreamID.Less(id1.StreamID) { - id1.StreamID, id2.StreamID = id2.StreamID, id1.StreamID - } - - var objs []metabase.PendingObjectEntry - for _, id := range []metabase.ObjectStream{id1, id2} { - id.Version = metabase.NextVersion - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: id, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - - objs = append(objs, metabase.PendingObjectEntry{ - ObjectKey: id.ObjectKey, - StreamID: id.StreamID, - CreatedAt: time.Now(), - Encryption: metabasetest.DefaultEncryption, - }) - } - - metabasetest.IteratePendingObjects{ - Opts: metabase.IteratePendingObjects{ - ProjectID: projectID, - BucketName: bucketName, - Recursive: true, - BatchSize: 1, - IncludeSystemMetadata: true, - }, - Result: objs, - }.Check(ctx, t, db) - }) - }) -} - -func createPendingObjects(ctx *testcontext.Context, t *testing.T, db *metabase.DB, numberOfObjects int, numberOfSegments int, projectID uuid.UUID, bucketName string) []metabase.RawPendingObject { - objects := make([]metabase.RawPendingObject, numberOfObjects) - for i := 0; i < numberOfObjects; i++ { - obj := metabasetest.RandObjectStream() - obj.ProjectID = projectID - obj.BucketName = bucketName - obj.Version = metabase.NextVersion - now := time.Now() - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: obj, - Encryption: metabasetest.DefaultEncryption, - UsePendingObjectsTable: true, - }, - Version: metabase.PendingVersion, - }.Check(ctx, t, db) - obj.Version++ - - for i := 0; i < numberOfSegments; i++ { - metabasetest.BeginSegment{ - Opts: metabase.BeginSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)}, - RootPieceID: storj.PieceID{byte(i) + 1}, - Pieces: []metabase.Piece{{ - Number: 1, - StorageNode: testrand.NodeID(), - }}, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - - metabasetest.CommitSegment{ - Opts: metabase.CommitSegment{ - ObjectStream: obj, - Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)}, - RootPieceID: storj.PieceID{1}, - Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}}, - - EncryptedKey: []byte{3}, - EncryptedKeyNonce: []byte{4}, - EncryptedETag: []byte{5}, - - EncryptedSize: 1024, - PlainSize: 512, - PlainOffset: 0, - Redundancy: metabasetest.DefaultRedundancy, - UsePendingObjectsTable: true, - }, - }.Check(ctx, t, db) - } - - zombieDeletionDeadline := time.Now().Add(24 * time.Hour) - objects[i] = metabase.RawPendingObject{ - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeletionDeadline, - } - } - sort.SliceStable(objects, func(i, j int) bool { - return objects[i].ObjectKey < objects[j].ObjectKey - }) - return objects -} - -func pendingObjectEntryFromRaw(obj metabase.RawPendingObject) metabase.PendingObjectEntry { - return metabase.PendingObjectEntry{ - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - ExpiresAt: obj.ExpiresAt, - CreatedAt: obj.CreatedAt, - Encryption: obj.Encryption, - EncryptedMetadataNonce: obj.EncryptedMetadataNonce, - EncryptedMetadata: obj.EncryptedMetadataNonce, - EncryptedMetadataEncryptedKey: obj.EncryptedMetadataEncryptedKey, - } -} - -func createPendingObjectsWithKeys(ctx *testcontext.Context, t *testing.T, db *metabase.DB, projectID uuid.UUID, bucketName string, keys []metabase.ObjectKey) map[metabase.ObjectKey]metabase.PendingObjectEntry { - objects := make(map[metabase.ObjectKey]metabase.PendingObjectEntry, len(keys)) - for _, key := range keys { - obj := metabasetest.RandObjectStream() - obj.ProjectID = projectID - obj.BucketName = bucketName - obj.ObjectKey = key - now := time.Now() - - metabasetest.CreatePendingObjectNew(ctx, t, db, obj, 0) - - objects[key] = metabase.PendingObjectEntry{ - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - CreatedAt: now, - Encryption: metabasetest.DefaultEncryption, - } - } - - return objects -} - -func pendingWithoutPrefix(prefix metabase.ObjectKey, entries ...metabase.PendingObjectEntry) []metabase.PendingObjectEntry { - xs := make([]metabase.PendingObjectEntry, len(entries)) - for i, e := range entries { - xs[i] = e - xs[i].ObjectKey = entries[i].ObjectKey[len(prefix):] - } - return xs -} - -func pendingPrefixEntry(key metabase.ObjectKey) metabase.PendingObjectEntry { - return metabase.PendingObjectEntry{ - IsPrefix: true, - ObjectKey: key, - } -} - -func TestIteratePendingObjectsByKey(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj := metabasetest.RandObjectStream() - - location := obj.Location() - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - for _, test := range metabasetest.InvalidObjectLocations(location) { - test := test - t.Run(test.Name, func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: test.ObjectLocation, - }, - ErrClass: test.ErrClass, - ErrText: test.ErrText, - }.Check(ctx, t, db) - metabasetest.Verify{}.Check(ctx, t, db) - }) - } - - t.Run("committed object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj := metabasetest.RandObjectStream() - - metabasetest.CreateObject(ctx, t, db, obj, 0) - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: obj.Location(), - BatchSize: 10, - }, - Result: nil, - }.Check(ctx, t, db) - }) - t.Run("non existing object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - pending := metabasetest.RandObjectStream() - metabasetest.CreatePendingObjectNew(ctx, t, db, pending, 0) - - object := metabase.RawPendingObject{ - PendingObjectStream: metabasetest.ObjectStreamToPending(pending), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - } - - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: metabase.ObjectLocation{ - ProjectID: pending.ProjectID, - BucketName: pending.BucketName, - ObjectKey: pending.Location().ObjectKey + "other", - }, - BatchSize: 10, - }, - Result: nil, - }.Check(ctx, t, db) - - metabasetest.Verify{PendingObjects: []metabase.RawPendingObject{object}}.Check(ctx, t, db) - }) - - t.Run("less and more objects than limit", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - pending := []metabase.ObjectStream{metabasetest.RandObjectStream(), metabasetest.RandObjectStream(), metabasetest.RandObjectStream()} - - location := pending[0].Location() - objects := make([]metabase.RawPendingObject, 3) - expected := make([]metabase.PendingObjectEntry, 3) - - for i, obj := range pending { - obj.ProjectID = location.ProjectID - obj.BucketName = location.BucketName - obj.ObjectKey = location.ObjectKey - obj.Version = metabase.Version(i + 1) - - metabasetest.CreatePendingObjectNew(ctx, t, db, obj, 0) - - objects[i] = metabase.RawPendingObject{ - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - } - expected[i] = pendingObjectEntryFromRaw(objects[i]) - } - - sort.Slice(expected, func(i, j int) bool { - return expected[i].StreamID.Less(expected[j].StreamID) - }) - - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: location, - BatchSize: 10, - }, - Result: expected, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: location, - BatchSize: 2, - }, - Result: expected, - }.Check(ctx, t, db) - - metabasetest.Verify{PendingObjects: objects}.Check(ctx, t, db) - }) - - t.Run("prefixed object key", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - pending := metabasetest.RandObjectStream() - pending.ObjectKey = metabase.ObjectKey("a/prefixed/" + string(location.ObjectKey)) - metabasetest.CreatePendingObjectNew(ctx, t, db, pending, 0) - - object := metabase.RawPendingObject{ - PendingObjectStream: metabasetest.ObjectStreamToPending(pending), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - } - - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: pending.Location(), - }, - Result: []metabase.PendingObjectEntry{pendingObjectEntryFromRaw(object)}, - }.Check(ctx, t, db) - - metabasetest.Verify{PendingObjects: []metabase.RawPendingObject{object}}.Check(ctx, t, db) - }) - - t.Run("using streamID cursor", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - pending := []metabase.ObjectStream{metabasetest.RandObjectStream(), metabasetest.RandObjectStream(), metabasetest.RandObjectStream()} - - location := pending[0].Location() - objects := make([]metabase.RawPendingObject, 3) - expected := make([]metabase.PendingObjectEntry, 3) - - for i, obj := range pending { - obj.ProjectID = location.ProjectID - obj.BucketName = location.BucketName - obj.ObjectKey = location.ObjectKey - obj.Version = metabase.Version(i + 1) - - metabasetest.CreatePendingObjectNew(ctx, t, db, obj, 0) - - objects[i] = metabase.RawPendingObject{ - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - } - expected[i] = pendingObjectEntryFromRaw(objects[i]) - } - - sort.Slice(expected, func(i, j int) bool { - return expected[i].StreamID.Less(expected[j].StreamID) - }) - - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: location, - BatchSize: 10, - Cursor: metabase.StreamIDCursor{ - StreamID: expected[0].StreamID, - }, - }, - Result: expected[1:], - }.Check(ctx, t, db) - - metabasetest.Verify{PendingObjects: objects}.Check(ctx, t, db) - }) - - t.Run("same key different versions", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj1 := metabasetest.RandObjectStream() - obj2 := obj1 - obj2.StreamID = testrand.UUID() - obj2.Version = 2 - - pending := []metabase.ObjectStream{obj1, obj2} - - location := pending[0].Location() - objects := make([]metabase.RawPendingObject, 2) - expected := make([]metabase.PendingObjectEntry, 2) - - for i, obj := range pending { - obj.ProjectID = location.ProjectID - obj.BucketName = location.BucketName - obj.ObjectKey = location.ObjectKey - obj.Version = metabase.Version(i + 1) - - metabasetest.CreatePendingObjectNew(ctx, t, db, obj, 0) - - objects[i] = metabase.RawPendingObject{ - PendingObjectStream: metabasetest.ObjectStreamToPending(obj), - CreatedAt: now, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - } - expected[i] = pendingObjectEntryFromRaw(objects[i]) - } - - sort.Slice(expected, func(i, j int) bool { - return expected[i].StreamID.Less(expected[j].StreamID) - }) - - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: location, - BatchSize: 1, - }, - Result: expected, - }.Check(ctx, t, db) - - metabasetest.IteratePendingObjectsByKeyNew{ - Opts: metabase.IteratePendingObjectsByKey{ - ObjectLocation: location, - BatchSize: 3, - }, - Result: expected, - }.Check(ctx, t, db) - - metabasetest.Verify{PendingObjects: objects}.Check(ctx, t, db) - }) - }) -} diff --git a/satellite/metabase/raw.go b/satellite/metabase/raw.go index d2c19b684ae5..0e4e986d36a8 100644 --- a/satellite/metabase/raw.go +++ b/satellite/metabase/raw.go @@ -41,25 +41,6 @@ type RawObject struct { ZombieDeletionDeadline *time.Time } -// RawPendingObject defines the full pending object that is stored in the database. It should be rarely used directly. -type RawPendingObject struct { - PendingObjectStream - - CreatedAt time.Time - ExpiresAt *time.Time - - EncryptedMetadataNonce []byte - EncryptedMetadata []byte - EncryptedMetadataEncryptedKey []byte - - Encryption storj.EncryptionParameters - - // ZombieDeletionDeadline defines when the pending raw object should be deleted from the database. - // This is as a safeguard against objects that failed to upload and the client has not indicated - // whether they want to continue uploading or delete the already uploaded data. - ZombieDeletionDeadline *time.Time -} - // RawSegment defines the full segment that is stored in the database. It should be rarely used directly. type RawSegment struct { StreamID uuid.UUID @@ -96,9 +77,8 @@ type RawCopy struct { // RawState contains full state of a table. type RawState struct { - Objects []RawObject - PendingObjects []RawPendingObject - Segments []RawSegment + Objects []RawObject + Segments []RawSegment } // TestingGetState returns the state of the database. @@ -110,11 +90,6 @@ func (db *DB) TestingGetState(ctx context.Context) (_ *RawState, err error) { return nil, Error.New("GetState: %w", err) } - state.PendingObjects, err = db.testingGetAllPendingObjects(ctx) - if err != nil { - return nil, Error.New("GetState: %w", err) - } - state.Segments, err = db.testingGetAllSegments(ctx) if err != nil { return nil, Error.New("GetState: %w", err) @@ -127,7 +102,6 @@ func (db *DB) TestingGetState(ctx context.Context) (_ *RawState, err error) { func (db *DB) TestingDeleteAll(ctx context.Context) (err error) { _, err = db.db.ExecContext(ctx, ` WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM objects; - WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM pending_objects; WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM segments; WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM node_aliases; WITH ignore_full_scan_for_test AS (SELECT 1) SELECT setval('node_alias_seq', 1, false); @@ -198,57 +172,6 @@ func (db *DB) testingGetAllObjects(ctx context.Context) (_ []RawObject, err erro return objs, nil } -// testingGetAllPendingObjects returns the state of the database. -func (db *DB) testingGetAllPendingObjects(ctx context.Context) (_ []RawPendingObject, err error) { - objs := []RawPendingObject{} - - rows, err := db.db.QueryContext(ctx, ` - WITH ignore_full_scan_for_test AS (SELECT 1) - SELECT - project_id, bucket_name, object_key, stream_id, - created_at, expires_at, - encrypted_metadata_nonce, encrypted_metadata, encrypted_metadata_encrypted_key, - encryption, zombie_deletion_deadline - FROM pending_objects - ORDER BY project_id ASC, bucket_name ASC, object_key ASC, stream_id ASC - `) - if err != nil { - return nil, Error.New("testingGetAllPendingObjects query: %w", err) - } - defer func() { err = errs.Combine(err, rows.Close()) }() - for rows.Next() { - var obj RawPendingObject - err := rows.Scan( - &obj.ProjectID, - &obj.BucketName, - &obj.ObjectKey, - &obj.StreamID, - - &obj.CreatedAt, - &obj.ExpiresAt, - - &obj.EncryptedMetadataNonce, - &obj.EncryptedMetadata, - &obj.EncryptedMetadataEncryptedKey, - - encryptionParameters{&obj.Encryption}, - &obj.ZombieDeletionDeadline, - ) - if err != nil { - return nil, Error.New("testingGetAllPendingObjects scan failed: %w", err) - } - objs = append(objs, obj) - } - if err := rows.Err(); err != nil { - return nil, Error.New("testingGetAllPendingObjects scan failed: %w", err) - } - - if len(objs) == 0 { - return nil, nil - } - return objs, nil -} - // testingGetAllSegments returns the state of the database. func (db *DB) testingGetAllSegments(ctx context.Context) (_ []RawSegment, err error) { segs := []RawSegment{} diff --git a/satellite/metabase/zombiedeletion/chore.go b/satellite/metabase/zombiedeletion/chore.go index 523e44e51ee3..88b11fb8f6ca 100644 --- a/satellite/metabase/zombiedeletion/chore.go +++ b/satellite/metabase/zombiedeletion/chore.go @@ -92,5 +92,5 @@ func (chore *Chore) deleteZombieObjects(ctx context.Context) (err error) { return err } - return chore.metabase.DeleteInactivePendingObjects(ctx, opts) + return nil } diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 62aeb0d1975a..99a58e0a242b 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -167,7 +167,6 @@ func (c Config) Metabase(applicationName string) metabase.Config { } // ExtendedConfig extended config keeps additional helper fields and methods around Config. -// TODO potentially can be removed when UsePendingObjectsTableProjects won't be used anymore. type ExtendedConfig struct { Config