satellite/metainfo/objectdeletion: Object deletion implementation

Improve our delete logic to require fewer database requests.
This PR creates a new objectdeletion package

Change-Id: I0500173bb9b8c771accb350f076329ede6dbb42c

satellite/metainfo/objectdeletion/identifier.go

@@ -0,0 +1,86 @@
+// Copyright (C) 2020 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package objectdeletion
+
+import (
+	"errors"
+	"strconv"
+	"strings"
+
+	"github.com/zeebo/errs"
+
+	"storj.io/common/storj"
+	"storj.io/common/uuid"
+)
+
+// ObjectIdentifier contains information about an object
+// that are needed for delete operation.
+type ObjectIdentifier struct {
+	ProjectID     uuid.UUID
+	Bucket        []byte
+	EncryptedPath []byte
+}
+
+// SegmentPath returns a raw path for a specific segment index.
+func (id *ObjectIdentifier) SegmentPath(segmentIndex int64) ([]byte, error) {
+	if segmentIndex < lastSegmentIndex {
+		return nil, errors.New("invalid segment index")
+	}
+	segment := "l"
+	if segmentIndex > lastSegmentIndex {
+		segment = "s" + strconv.FormatInt(segmentIndex, 10)
+	}
+
+	return []byte(storj.JoinPaths(
+		id.ProjectID.String(),
+		segment,
+		string(id.Bucket),
+		string(id.EncryptedPath),
+	)), nil
+}
+
+// ParseSegmentPath parses a raw path and returns an
+// object identifier from that path along with the path's segment index.
+// example: <project-id>/01/<bucket-name>/<encrypted-path>
+func ParseSegmentPath(rawPath []byte) (ObjectIdentifier, int64, error) {
+	elements := storj.SplitPath(string(rawPath))
+	if len(elements) < 4 {
+		return ObjectIdentifier{}, -1, errs.New("invalid path %q", string(rawPath))
+	}
+
+	projectID, err := uuid.FromString(elements[0])
+	if err != nil {
+		return ObjectIdentifier{}, -1, errs.Wrap(err)
+	}
+	var segmentIndex int64
+	if elements[1] == "l" {
+		segmentIndex = lastSegmentIndex
+	} else {
+		segmentIndex, err = strconv.ParseInt(elements[1][1:], 10, 64) // remove the strng `s` from segment index we got
+
+		if err != nil {
+			return ObjectIdentifier{}, -1, errs.Wrap(err)
+		}
+	}
+
+	return ObjectIdentifier{
+		ProjectID:     projectID,
+		Bucket:        []byte(elements[2]),
+		EncryptedPath: []byte(storj.JoinPaths(elements[3:]...)),
+	}, segmentIndex, nil
+}
+
+// Key returns a string concatenated by all object identifier fields plus 0.
+// It's a unique string used to identify an object.
+// It's not a valid key for retrieving pointers from metainfo database.
+func (id *ObjectIdentifier) Key() string {
+	builder := strings.Builder{}
+	// we don't need the return value here
+	// Write will always return the length of the argument and nil error
+	_, _ = builder.Write(id.ProjectID[:])
+	_, _ = builder.Write(id.Bucket)
+	_, _ = builder.Write(id.EncryptedPath)
+
+	return builder.String()
+}

satellite/metainfo/objectdeletion/report.go

@@ -0,0 +1,54 @@
+// Copyright (C) 2020 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package objectdeletion
+
+import (
+	"context"
+
+	"go.uber.org/zap"
+)
+
+// Report represents the deleteion status report.
+type Report struct {
+	Deleted []*ObjectIdentifier
+	Failed  []*ObjectIdentifier
+}
+
+// HasFailures returns wether a delete operation has failures.
+func (r Report) HasFailures() bool {
+	return len(r.Failed) > 0
+}
+
+// GenerateReport returns the result of a delete, success, or failure.
+func GenerateReport(ctx context.Context, log *zap.Logger, requests []*ObjectIdentifier, deletedPaths [][]byte) Report {
+	defer mon.Task()(&ctx)(nil)
+
+	report := Report{}
+	deletedObjects := make(map[string]*ObjectIdentifier)
+	for _, path := range deletedPaths {
+		if path == nil {
+			continue
+		}
+		id, _, err := ParseSegmentPath(path)
+		if err != nil {
+			log.Debug("failed to parse deleted segmnt path for report",
+				zap.String("Raw Segment Path", string(path)),
+			)
+			continue
+		}
+		if _, ok := deletedObjects[id.Key()]; !ok {
+			deletedObjects[id.Key()] = &id
+		}
+	}
+
+	// populate report with failed and deleted objects
+	for _, req := range requests {
+		if _, ok := deletedObjects[req.Key()]; !ok {
+			report.Failed = append(report.Failed, req)
+		} else {
+			report.Deleted = append(report.Deleted, req)
+		}
+	}
+	return report
+}

satellite/metainfo/objectdeletion/report_test.go

@@ -0,0 +1,75 @@
+// Copyright (C) 2020 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package objectdeletion_test
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zeebo/errs"
+	"go.uber.org/zap/zaptest"
+
+	"storj.io/common/testcontext"
+	"storj.io/common/testrand"
+	"storj.io/storj/satellite/metainfo/objectdeletion"
+)
+
+func TestReport(t *testing.T) {
+	logger := zaptest.NewLogger(t)
+
+	var testCases = []struct {
+		description     string
+		numRequests     int
+		numDeletedPaths int
+		expectedFailure bool
+	}{
+		{"has-failure", 2, 1, true},
+		{"all-deleted", 2, 2, false},
+	}
+
+	for _, tt := range testCases {
+		tt := tt
+		t.Run(tt.description, func(t *testing.T) {
+			ctx := testcontext.New(t)
+			defer ctx.Cleanup()
+			requests := createRequests(tt.numRequests)
+			deletedSegmentPaths, err := createDeletedSegmentPaths(requests, tt.numDeletedPaths)
+			require.NoError(t, err)
+			report := objectdeletion.GenerateReport(ctx, logger, requests, deletedSegmentPaths)
+			require.Equal(t, tt.expectedFailure, report.HasFailures())
+		})
+	}
+
+}
+
+func createDeletedSegmentPaths(requests []*objectdeletion.ObjectIdentifier, numDeleted int) ([][]byte, error) {
+	if numDeleted > len(requests) {
+		return nil, errs.New("invalid argument")
+	}
+	deletedSegmentPaths := make([][]byte, 0, numDeleted)
+	for i := 0; i < numDeleted; i++ {
+		path, err := requests[i].SegmentPath(int64(testrand.Intn(10)))
+		if err != nil {
+			return nil, err
+		}
+		deletedSegmentPaths = append(deletedSegmentPaths, path)
+	}
+	return deletedSegmentPaths, nil
+}
+
+func createRequests(numRequests int) []*objectdeletion.ObjectIdentifier {
+	requests := make([]*objectdeletion.ObjectIdentifier, 0, numRequests)
+
+	for i := 0; i < numRequests; i++ {
+		obj := objectdeletion.ObjectIdentifier{
+			ProjectID:     testrand.UUID(),
+			Bucket:        []byte("test"),
+			EncryptedPath: []byte(strconv.Itoa(i) + "test"),
+		}
+		requests = append(requests, &obj)
+	}
+
+	return requests
+}