Merge pull request #22 from brave/ttl

Set TTL attribute when soft-delete a sync item

datastore/datastoretest/dynamo.go

@@ -150,3 +150,41 @@ func ScanClientItemCounts(dynamo *datastore.Dynamo) ([]datastore.ClientItemCount
 
 	return clientItemCounts, nil
 }
+
+// TTL is used to unmarshal result of TTL attribute from dynamoDB.
+type TTL struct {
+	TTL *int64 `dynamodbav:"ttl,omitempty"`
+}
+
+// IsTTLSet checks if TTL is set for an item.
+func IsTTLSet(dynamo *datastore.Dynamo, clientID string, id string) (bool, error) {
+	primaryKey := datastore.PrimaryKey{ClientID: clientID, ID: id}
+	key, err := dynamodbattribute.MarshalMap(primaryKey)
+	if err != nil {
+		return false, fmt.Errorf("error marshalling primary key: %w", err)
+	}
+
+	proj := expression.NamesList(expression.Name("ttl"))
+	expr, err := expression.NewBuilder().WithProjection(proj).Build()
+	if err != nil {
+		return false, fmt.Errorf("error building expr: %w", err)
+	}
+
+	input := &dynamodb.GetItemInput{
+		Key:                      key,
+		ExpressionAttributeNames: expr.Names(),
+		ProjectionExpression:     expr.Projection(),
+		TableName:                aws.String(datastore.Table),
+	}
+	out, err := dynamo.GetItem(input)
+	if err != nil {
+		return false, fmt.Errorf("error doing get TTL for an item: %w", err)
+	}
+
+	ttl := TTL{}
+	err = dynamodbattribute.UnmarshalMap(out.Item, &ttl)
+	if err != nil {
+		return false, fmt.Errorf("error unmarshalling TTL: %w", err)
+	}
+	return ttl.TTL != nil, nil
+}

datastore/sync_entity.go

@@ -25,6 +25,8 @@ const (
 	clientTagItemPrefix    = "Client#"
 	serverTagItemPrefix    = "Server#"
 	conditionalCheckFailed = "ConditionalCheckFailed"
+	ttlAttrName            = "ttl"
+	ttl                    = time.Hour * 24 * 90 // 90 days
 )
 
 // SyncEntity is used to marshal and unmarshal sync items in dynamoDB.
@@ -292,6 +294,12 @@ func (dynamo *Dynamo) UpdateSyncEntity(entity *SyncEntity) (bool, bool, error) {
 	}
 	if entity.Deleted != nil {
 		update = update.Set(expression.Name("Deleted"), expression.Value(entity.Deleted))
+
+		// Set TTL attribute when soft-deleting an item which will remove obsolete
+		// items automatically by dynamoDB.
+		if *entity.Deleted {
+			update = update.Set(expression.Name(ttlAttrName), expression.Value(time.Now().Add(ttl).Unix()))
+		}
 	}
 	if entity.Folder != nil {
 		update = update.Set(expression.Name("Folder"), expression.Value(entity.Folder))

datastore/sync_entity_test.go

@@ -311,6 +311,46 @@ func (suite *SyncEntityTestSuite) TestUpdateSyncEntity_Basic() {
 	suite.Assert().Equal(syncItems, []datastore.SyncEntity{updateEntity1, updateEntity2, updateEntity3})
 }
 
+func (suite *SyncEntityTestSuite) TestUpdateSyncEntity_AddTTLWhenSoftDelete() {
+	entity1 := datastore.SyncEntity{
+		ClientID:      "client1",
+		ID:            "id1",
+		Version:       aws.Int64(1),
+		Ctime:         aws.Int64(12345678),
+		Mtime:         aws.Int64(12345678),
+		DataType:      aws.Int(123),
+		Folder:        aws.Bool(false),
+		Deleted:       aws.Bool(false),
+		DataTypeMtime: aws.String("123#12345678"),
+		Specifics:     []byte{1, 2},
+	}
+	suite.Require().NoError(
+		suite.dynamo.InsertSyncEntity(&entity1), "InsertSyncEntity should succeed")
+
+	// Do a non-delete update.
+	entity1.Version = aws.Int64(2)
+	conflict, delete, err := suite.dynamo.UpdateSyncEntity(&entity1)
+	suite.Require().NoError(err, "UpdateSyncEntity should succeed")
+	suite.Assert().False(conflict, "Successful update should not have conflict")
+	suite.Assert().False(delete, "Non-delete operation should return false")
+
+	isTTLSet, err := datastoretest.IsTTLSet(suite.dynamo, entity1.ClientID, entity1.ID)
+	suite.Require().NoError(err, "IsTTLSet should succeed")
+	suite.Assert().False(isTTLSet, "TTL should not be set for non-delete operations")
+
+	// Do a soft-delete.
+	entity1.Version = aws.Int64(3)
+	entity1.Deleted = aws.Bool(true)
+	conflict, delete, err = suite.dynamo.UpdateSyncEntity(&entity1)
+	suite.Require().NoError(err, "UpdateSyncEntity should succeed")
+	suite.Assert().False(conflict, "Successful update should not have conflict")
+	suite.Assert().True(delete, "Delete operation should return false")
+
+	isTTLSet, err = datastoretest.IsTTLSet(suite.dynamo, entity1.ClientID, entity1.ID)
+	suite.Require().NoError(err, "IsTTLSet should succeed")
+	suite.Assert().True(isTTLSet, "TTL should be set when soft-delete")
+}
+
 func (suite *SyncEntityTestSuite) TestUpdateSyncEntity_ReuseClientTag() {
 	// Insert an item with client tag.
 	entity1 := datastore.SyncEntity{

dynamo.Dockerfile

@@ -18,6 +18,9 @@ RUN mkdir -p ${DB_LOCATION} && \
       DYNAMO_PID=$! && \
       aws dynamodb create-table --cli-input-json file://table.json \
       --endpoint-url ${AWS_ENDPOINT} --region ${AWS_REGION} && \
+      aws dynamodb update-time-to-live --table-name ${TABLE_NAME} \
+      --endpoint-url ${AWS_ENDPOINT} --region ${AWS_REGION} \
+      --time-to-live-specification Enabled=true,AttributeName=ttl && \
       kill $DYNAMO_PID
 
 FROM amazon/dynamodb-local:1.12.0