GO-2827 Unbind user files (reclaim / reconcile space usage)

.mockery.yaml

@@ -56,6 +56,10 @@ packages:
   github.com/anyproto/anytype-heart/core/filestorage/filesync:
     interfaces:
       FileSync:
+  github.com/anyproto/anytype-heart/core/filestorage/rpcstore:
+    interfaces:
+      Service:
+      RpcStore:
   github.com/anyproto/anytype-heart/core/block/import/common/objectid:
     interfaces:
       IdGetterProvider:
@@ -165,6 +169,9 @@ packages:
   github.com/anyproto/anytype-heart/core/block/object/idresolver:
     interfaces:
       Resolver:
+  github.com/anyproto/anytype-heart/core/filestorage:
+    interfaces:
+      FileStorage:
   github.com/anyproto/anytype-heart/util/linkpreview:
     interfaces:
       LinkPreview:

core/anytype/bootstrap.go

@@ -57,6 +57,7 @@ import (
 	"github.com/anyproto/anytype-heart/core/files/fileacl"
 	"github.com/anyproto/anytype-heart/core/files/fileobject"
 	"github.com/anyproto/anytype-heart/core/files/fileuploader"
+	"github.com/anyproto/anytype-heart/core/files/reconciler"
 	"github.com/anyproto/anytype-heart/core/filestorage"
 	"github.com/anyproto/anytype-heart/core/filestorage/filesync"
 	"github.com/anyproto/anytype-heart/core/filestorage/rpcstore"
@@ -241,6 +242,7 @@ func Bootstrap(a *app.App, components ...app.Component) {
 		Register(deletioncontroller.New()).
 		Register(invitestore.New()).
 		Register(filesync.New()).
+		Register(reconciler.New()).
 		Register(fileobject.New(200*time.Millisecond, 2*time.Second)).
 		Register(inviteservice.New()).
 		Register(acl.New()).

core/block/delete.go

@@ -4,7 +4,7 @@
 	"context"
 	"fmt"
 
 	"go.uber.org/zap"
 
 	"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
 	"github.com/anyproto/anytype-heart/core/domain"
@@ -52,7 +52,7 @@
 		}
 		err = s.OnDelete(id, nil)
 		if err != nil {
 			return fmt.Errorf("on delete: %w", err)
 		}
 		if sbType == coresb.SmartBlockTypeRelation {
 			err := s.deleteRelationOptions(relationKey)
@@ -67,11 +67,9 @@
 		if err != nil {
 			return fmt.Errorf("delete file data: %w", err)
 		}
-		// this will call DeleteTree asynchronously in the end
-		return spc.DeleteTree(context.Background(), id.ObjectID)
+		err = spc.DeleteTree(context.Background(), id.ObjectID)
 	default:
-		// this will call DeleteTree asynchronously in the end
-		return spc.DeleteTree(context.Background(), id.ObjectID)
+		err = spc.DeleteTree(context.Background(), id.ObjectID)
 	}
 	if err != nil {
 		return
@@ -137,7 +135,7 @@
 		return nil
 	})
 	if err != nil {
-		log.Error("failed to perform delete operation on object", zap.Error(err))
+		log.With("error", err, "objectId", id.ObjectID).Error("failed to perform delete operation on object")
 	}
 	if err := s.objectStore.DeleteObject(id); err != nil {
 		return fmt.Errorf("delete object from local store: %w", err)

core/block/editor/factory.go

@@ -21,6 +21,7 @@ import (
 	"github.com/anyproto/anytype-heart/core/files"
 	"github.com/anyproto/anytype-heart/core/files/fileobject"
 	"github.com/anyproto/anytype-heart/core/files/fileuploader"
+	"github.com/anyproto/anytype-heart/core/files/reconciler"
 	"github.com/anyproto/anytype-heart/pkg/lib/core"
 	coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
 	"github.com/anyproto/anytype-heart/pkg/lib/localstore/filestore"
@@ -58,6 +59,7 @@ type ObjectFactory struct {
 	fileObjectService   fileobject.Service
 	processService      process.Service
 	fileUploaderService fileuploader.Service
+	fileReconciler      reconciler.Reconciler
 	objectDeleter       ObjectDeleter
 }
 
@@ -85,6 +87,7 @@ func (f *ObjectFactory) Init(a *app.App) (err error) {
 	f.processService = app.MustComponent[process.Service](a)
 	f.fileUploaderService = app.MustComponent[fileuploader.Service](a)
 	f.objectDeleter = app.MustComponent[ObjectDeleter](a)
+	f.fileReconciler = app.MustComponent[reconciler.Reconciler](a)
 	return nil
 }
 

core/block/editor/files.go

@@ -12,17 +12,20 @@ import (
 	"github.com/anyproto/anytype-heart/core/block/source"
 	"github.com/anyproto/anytype-heart/core/domain"
 	"github.com/anyproto/anytype-heart/core/files/fileobject"
+	"github.com/anyproto/anytype-heart/core/files/reconciler"
 	"github.com/anyproto/anytype-heart/core/filestorage"
 	coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
 )
 
 func (f *ObjectFactory) newFile(sb smartblock.SmartBlock) *File {
 	basicComponent := basic.NewBasic(sb, f.objectStore, f.layoutConverter)
 	return &File{
-		SmartBlock:     sb,
-		ChangeReceiver: sb.(source.ChangeReceiver),
-		AllOperations:  basicComponent,
-		Text:           stext.NewText(sb, f.objectStore, f.eventSender), fileObjectService: f.fileObjectService,
+		SmartBlock:        sb,
+		ChangeReceiver:    sb.(source.ChangeReceiver),
+		AllOperations:     basicComponent,
+		Text:              stext.NewText(sb, f.objectStore, f.eventSender),
+		fileObjectService: f.fileObjectService,
+		reconciler:        f.fileReconciler,
 	}
 }
 
@@ -32,6 +35,7 @@ type File struct {
 	basic.AllOperations
 	stext.Text
 	fileObjectService fileobject.Service
+	reconciler        reconciler.Reconciler
 }
 
 func (f *File) CreationStateMigration(ctx *smartblock.InitContext) migration.Migration {
@@ -70,6 +74,8 @@ func (f *File) Init(ctx *smartblock.InitContext) error {
 		return err
 	}
 
+	f.SmartBlock.AddHook(f.reconciler.FileObjectHook(domain.FullID{SpaceID: f.SpaceID(), ObjectID: f.Id()}), smartblock.HookBeforeApply)
+
 	if !ctx.IsNewObject {
 		err = f.fileObjectService.EnsureFileAddedToSyncQueue(domain.FullID{ObjectID: f.Id(), SpaceID: f.SpaceID()}, ctx.State.Details())
 		if err != nil {

core/block/editor/smartblock/smartblock.go

@@ -50,7 +50,6 @@ type ApplyFlag int
 var (
 	ErrSimpleBlockNotFound                         = errors.New("simple block not found")
 	ErrCantInitExistingSmartblockWithNonEmptyState = errors.New("can't init existing smartblock with non-empty state")
-	ErrIsDeleted                                   = errors.New("smartblock is deleted")
 )
 
 const (
@@ -550,7 +549,7 @@ func (sb *smartBlock) EnabledRelationAsDependentObjects() {
 func (sb *smartBlock) Apply(s *state.State, flags ...ApplyFlag) (err error) {
 	startTime := time.Now()
 	if sb.IsDeleted() {
-		return ErrIsDeleted
+		return domain.ErrObjectIsDeleted
 	}
 	var (
 		sendEvent           = true
@@ -968,7 +967,7 @@ func (sb *smartBlock) RemoveExtraRelations(ctx session.Context, relationIds []st
 
 func (sb *smartBlock) StateAppend(f func(d state.Doc) (s *state.State, changes []*pb.ChangeContent, err error)) error {
 	if sb.IsDeleted() {
-		return ErrIsDeleted
+		return domain.ErrObjectIsDeleted
 	}
 	s, changes, err := f(sb.Doc)
 	if err != nil {
@@ -1001,7 +1000,7 @@ func (sb *smartBlock) StateAppend(f func(d state.Doc) (s *state.State, changes [
 // TODO: need to test StateRebuild
 func (sb *smartBlock) StateRebuild(d state.Doc) (err error) {
 	if sb.IsDeleted() {
-		return ErrIsDeleted
+		return domain.ErrObjectIsDeleted
 	}
 	sb.updateRestrictions()
 	sb.injectDerivedDetails(d.(*state.State), sb.SpaceID(), sb.Type())

core/domain/errors.go

@@ -3,3 +3,4 @@ package domain
 import "errors"
 
 var ErrValidationFailed = errors.New("validation failed")
+var ErrObjectIsDeleted = errors.New("object is deleted")

core/file.go

@@ -10,6 +10,7 @@ import (
 	"github.com/anyproto/anytype-heart/core/domain/objectorigin"
 	"github.com/anyproto/anytype-heart/core/files"
 	"github.com/anyproto/anytype-heart/core/files/fileobject"
+	"github.com/anyproto/anytype-heart/core/files/reconciler"
 	"github.com/anyproto/anytype-heart/pb"
 )
 
@@ -194,3 +195,16 @@ func (mw *Middleware) FileNodeUsage(ctx context.Context, req *pb.RpcFileNodeUsag
 
 	return resp
 }
+
+func (mw *Middleware) FileReconcile(ctx context.Context, req *pb.RpcFileReconcileRequest) *pb.RpcFileReconcileResponse {
+	err := getService[reconciler.Reconciler](mw).Start(ctx)
+	if err != nil {
+		return &pb.RpcFileReconcileResponse{
+			Error: &pb.RpcFileReconcileResponseError{
+				Code:        mapErrorCode[pb.RpcFileReconcileResponseErrorCode](err),
+				Description: err.Error(),
+			},
+		}
+	}
+	return &pb.RpcFileReconcileResponse{}
+}

core/files/files_test.go

@@ -89,7 +89,7 @@ func TestFileAdd(t *testing.T) {
 	ctx := context.Background()
 
 	uploaded := make(chan struct{})
-	fx.fileSyncService.OnUploaded(func(objectId string) error {
+	fx.fileSyncService.OnUploaded(func(objectId string, fileId domain.FullFileId) error {
 		close(uploaded)
 		return nil
 	})
@@ -199,7 +199,7 @@ func TestFileAddWithCustomKeys(t *testing.T) {
 		ctx := context.Background()
 
 		uploaded := make(chan struct{})
-		fx.fileSyncService.OnUploaded(func(objectId string) error {
+		fx.fileSyncService.OnUploaded(func(objectId string, fileId domain.FullFileId) error {
 			close(uploaded)
 			return nil
 		})
@@ -237,7 +237,7 @@ func TestFileAddWithCustomKeys(t *testing.T) {
 				ctx := context.Background()
 
 				uploaded := make(chan struct{})
-				fx.fileSyncService.OnUploaded(func(objectId string) error {
+				fx.fileSyncService.OnUploaded(func(objectId string, fileId domain.FullFileId) error {
 					close(uploaded)
 					return nil
 				})