Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GO-2827 Unbind user files (reclaim / reconcile space usage) #1226

Merged
merged 18 commits into from
Jun 13, 2024
Merged

GO-2827 Unbind user files (reclaim / reconcile space usage) #1226

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
8f4c003
GO-2827: Persistent queue: add RemoveWait method
deff7 May 3, 2024
0f90551
GO-2827: WIP: File reconciler
deff7 May 6, 2024
47d7751
GO-2827: Rewrite reconciler
deff7 May 8, 2024
0f770e3
GO-3446: Tree storage: sqlite: Extract deletion status to another table
deff7 May 8, 2024
b46d1e1
GO-2827 Merge branch 'go-3446-sqlite-tree-storage-race-condition-when…
deff7 May 8, 2024
b012590
GO-2827: Fix IterateFiles: handle streams properly
deff7 May 9, 2024
c5c01c6
GO-2827: Check isDeleted flag just in case
deff7 May 9, 2024
3346906
GO-2827: Filestorage: test
deff7 May 9, 2024
8a6842b
GO-2827: More tests
deff7 May 9, 2024
b1d7b24
GO-2827: Remove debug code
deff7 May 9, 2024
4eafba9
GO-2827: Fix CancelDeletion; add test
deff7 May 13, 2024
368b80a
GO-2827: Comment
deff7 May 13, 2024
07f1cda
GO-2827 Merge branch 'go-3446-sqlite-tree-storage-race-condition-when…
deff7 May 15, 2024
431a455
GO-2827 Merge branch 'main' of github.com:anyproto/anytype-heart into…
deff7 Jun 10, 2024
9ef26cf
GO-2827: Fix linter
deff7 Jun 10, 2024
a9d457a
GO-2827: Use std errors.Join
deff7 Jun 12, 2024
a790d69
Merge branch 'main' of github.com:anyproto/anytype-heart into go-2827…
deff7 Jun 13, 2024
bcad681
GO-2827: Fix merge
deff7 Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .mockery.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,3 +159,6 @@ packages:
github.com/anyproto/anytype-heart/core/block/object/idresolver:
interfaces:
Resolver:
github.com/anyproto/anytype-heart/core/filestorage:
interfaces:
FileStorage:
640 changes: 320 additions & 320 deletions clientlibrary/service/service.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions core/anytype/bootstrap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import (
"github.com/anyproto/anytype-heart/core/files/fileacl"
"github.com/anyproto/anytype-heart/core/files/fileobject"
"github.com/anyproto/anytype-heart/core/files/fileuploader"
"github.com/anyproto/anytype-heart/core/files/reconciler"
"github.com/anyproto/anytype-heart/core/filestorage"
"github.com/anyproto/anytype-heart/core/filestorage/filesync"
"github.com/anyproto/anytype-heart/core/filestorage/rpcstore"
Expand Down Expand Up @@ -243,6 +244,7 @@ func Bootstrap(a *app.App, components ...app.Component) {
Register(deletioncontroller.New()).
Register(invitestore.New()).
Register(filesync.New()).
Register(reconciler.New()).
Register(fileobject.New(200*time.Millisecond, 2*time.Second)).
Register(inviteservice.New()).
Register(acl.New()).
Expand Down
8 changes: 3 additions & 5 deletions core/block/delete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,9 @@ func (s *Service) DeleteObjectByFullID(id domain.FullID) (err error) {
if err != nil {
return fmt.Errorf("delete file data: %w", err)
}
// this will call DeleteTree asynchronously in the end
return spc.DeleteTree(context.Background(), id.ObjectID)
err = spc.DeleteTree(context.Background(), id.ObjectID)
default:
// this will call DeleteTree asynchronously in the end
return spc.DeleteTree(context.Background(), id.ObjectID)
err = spc.DeleteTree(context.Background(), id.ObjectID)
}
if err != nil {
return
Expand Down Expand Up @@ -137,7 +135,7 @@ func (s *Service) OnDelete(id domain.FullID, workspaceRemove func() error) error
return nil
})
if err != nil {
log.Error("failed to perform delete operation on object", zap.Error(err))
log.With("error", err, "objectId", id.ObjectID).Error("failed to perform delete operation on object")
}
if err := s.objectStore.DeleteObject(id); err != nil {
return fmt.Errorf("delete object from local store: %w", err)
Expand Down
3 changes: 3 additions & 0 deletions core/block/editor/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/anyproto/anytype-heart/core/files"
"github.com/anyproto/anytype-heart/core/files/fileobject"
"github.com/anyproto/anytype-heart/core/files/fileuploader"
"github.com/anyproto/anytype-heart/core/files/reconciler"
"github.com/anyproto/anytype-heart/pkg/lib/core"
coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/filestore"
Expand Down Expand Up @@ -58,6 +59,7 @@ type ObjectFactory struct {
fileObjectService fileobject.Service
processService process.Service
fileUploaderService fileuploader.Service
fileReconciler reconciler.Reconciler
objectDeleter ObjectDeleter
}

Expand Down Expand Up @@ -85,6 +87,7 @@ func (f *ObjectFactory) Init(a *app.App) (err error) {
f.processService = app.MustComponent[process.Service](a)
f.fileUploaderService = app.MustComponent[fileuploader.Service](a)
f.objectDeleter = app.MustComponent[ObjectDeleter](a)
f.fileReconciler = app.MustComponent[reconciler.Reconciler](a)
return nil
}

Expand Down
14 changes: 10 additions & 4 deletions core/block/editor/files.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,20 @@ import (
"github.com/anyproto/anytype-heart/core/block/source"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/core/files/fileobject"
"github.com/anyproto/anytype-heart/core/files/reconciler"
"github.com/anyproto/anytype-heart/core/filestorage"
coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
)

func (f *ObjectFactory) newFile(sb smartblock.SmartBlock) *File {
basicComponent := basic.NewBasic(sb, f.objectStore, f.layoutConverter)
return &File{
SmartBlock: sb,
ChangeReceiver: sb.(source.ChangeReceiver),
AllOperations: basicComponent,
Text: stext.NewText(sb, f.objectStore, f.eventSender), fileObjectService: f.fileObjectService,
SmartBlock: sb,
ChangeReceiver: sb.(source.ChangeReceiver),
AllOperations: basicComponent,
Text: stext.NewText(sb, f.objectStore, f.eventSender),
fileObjectService: f.fileObjectService,
reconciler: f.fileReconciler,
}
}

Expand All @@ -32,6 +35,7 @@ type File struct {
basic.AllOperations
stext.Text
fileObjectService fileobject.Service
reconciler reconciler.Reconciler
}

func (f *File) CreationStateMigration(ctx *smartblock.InitContext) migration.Migration {
Expand Down Expand Up @@ -70,6 +74,8 @@ func (f *File) Init(ctx *smartblock.InitContext) error {
return err
}

f.SmartBlock.AddHook(f.reconciler.FileObjectHook(domain.FullID{SpaceID: f.SpaceID(), ObjectID: f.Id()}), smartblock.HookBeforeApply)

if !ctx.IsNewObject {
err = f.fileObjectService.EnsureFileAddedToSyncQueue(domain.FullID{ObjectID: f.Id(), SpaceID: f.SpaceID()}, ctx.State.Details())
if err != nil {
Expand Down
7 changes: 3 additions & 4 deletions core/block/editor/smartblock/smartblock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ type ApplyFlag int
var (
ErrSimpleBlockNotFound = errors.New("simple block not found")
ErrCantInitExistingSmartblockWithNonEmptyState = errors.New("can't init existing smartblock with non-empty state")
ErrIsDeleted = errors.New("smartblock is deleted")
)

const (
Expand Down Expand Up @@ -550,7 +549,7 @@ func (sb *smartBlock) EnabledRelationAsDependentObjects() {
func (sb *smartBlock) Apply(s *state.State, flags ...ApplyFlag) (err error) {
startTime := time.Now()
if sb.IsDeleted() {
return ErrIsDeleted
return domain.ErrObjectIsDeleted
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider refactoring the Apply method to improve readability and maintainability.

The Apply method is quite lengthy and handles multiple responsibilities. It would be beneficial to break this method into smaller, more focused methods. This can improve readability, make the code easier to maintain, and simplify unit testing.

}
var (
sendEvent = true
Expand Down Expand Up @@ -968,7 +967,7 @@ func (sb *smartBlock) RemoveExtraRelations(ctx session.Context, relationIds []st

func (sb *smartBlock) StateAppend(f func(d state.Doc) (s *state.State, changes []*pb.ChangeContent, err error)) error {
if sb.IsDeleted() {
return ErrIsDeleted
return domain.ErrObjectIsDeleted
}
s, changes, err := f(sb.Doc)
if err != nil {
Expand Down Expand Up @@ -1001,7 +1000,7 @@ func (sb *smartBlock) StateAppend(f func(d state.Doc) (s *state.State, changes [
// TODO: need to test StateRebuild
func (sb *smartBlock) StateRebuild(d state.Doc) (err error) {
if sb.IsDeleted() {
return ErrIsDeleted
return domain.ErrObjectIsDeleted
}
sb.updateRestrictions()
sb.injectDerivedDetails(d.(*state.State), sb.SpaceID(), sb.Type())
Expand Down
1 change: 1 addition & 0 deletions core/domain/errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ package domain
import "errors"

var ErrValidationFailed = errors.New("validation failed")
var ErrObjectIsDeleted = errors.New("object is deleted")
14 changes: 14 additions & 0 deletions core/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/anyproto/anytype-heart/core/domain/objectorigin"
"github.com/anyproto/anytype-heart/core/files"
"github.com/anyproto/anytype-heart/core/files/fileobject"
"github.com/anyproto/anytype-heart/core/files/reconciler"
"github.com/anyproto/anytype-heart/pb"
)

Expand Down Expand Up @@ -194,3 +195,16 @@ func (mw *Middleware) FileNodeUsage(ctx context.Context, req *pb.RpcFileNodeUsag

return resp
}

func (mw *Middleware) FileReconcile(ctx context.Context, req *pb.RpcFileReconcileRequest) *pb.RpcFileReconcileResponse {
err := getService[reconciler.Reconciler](mw).Start(ctx)
if err != nil {
return &pb.RpcFileReconcileResponse{
Error: &pb.RpcFileReconcileResponseError{
Code: mapErrorCode[pb.RpcFileReconcileResponseErrorCode](err),
Description: err.Error(),
},
}
}
return &pb.RpcFileReconcileResponse{}
}
3 changes: 3 additions & 0 deletions core/files/fileobject/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/core/domain/objectorigin"
"github.com/anyproto/anytype-heart/core/files"
"github.com/anyproto/anytype-heart/core/filestorage"
"github.com/anyproto/anytype-heart/core/filestorage/filesync"
"github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus"
"github.com/anyproto/anytype-heart/pb"
Expand Down Expand Up @@ -77,6 +78,7 @@ type service struct {
fileService files.Service
fileSync filesync.FileSync
fileStore filestore.FileStore
fileStorage filestorage.FileStorage
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ensure proper error handling for fileStorage initialization.

The initialization of fileStorage is critical for the functionality of this service. It's important to add error handling for the case where fileStorage cannot be retrieved or initialized correctly. This will prevent runtime panics and provide clearer error messages for troubleshooting.

- s.fileStorage = app.MustComponent[filestorage.FileStorage](a)
+ s.fileStorage, err = app.MustComponent[filestorage.FileStorage](a)
+ if err != nil {
+     return fmt.Errorf("failed to initialize fileStorage: %w", err)
+ }

Committable suggestion was skipped due to low confidence.

objectStore objectstore.ObjectStore
spaceIdResolver idresolver.Resolver
migrationQueue *persistentqueue.Queue[*migrationItem]
Expand Down Expand Up @@ -109,6 +111,7 @@ func (s *service) Init(a *app.App) error {
s.objectStore = app.MustComponent[objectstore.ObjectStore](a)
s.fileStore = app.MustComponent[filestore.FileStore](a)
s.spaceIdResolver = app.MustComponent[idresolver.Resolver](a)
s.fileStorage = app.MustComponent[filestorage.FileStorage](a)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the Init method, the initialization of fileStorage is critical. Add error handling for the case where fileStorage cannot be retrieved or initialized correctly.


s.indexer = s.newIndexer()

Expand Down
6 changes: 3 additions & 3 deletions core/files/files_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func TestFileAdd(t *testing.T) {
ctx := context.Background()

uploaded := make(chan struct{})
fx.fileSyncService.OnUploaded(func(objectId string) error {
fx.fileSyncService.OnUploaded(func(objectId string, fileId domain.FullFileId) error {
close(uploaded)
return nil
})
Expand Down Expand Up @@ -199,7 +199,7 @@ func TestFileAddWithCustomKeys(t *testing.T) {
ctx := context.Background()

uploaded := make(chan struct{})
fx.fileSyncService.OnUploaded(func(objectId string) error {
fx.fileSyncService.OnUploaded(func(objectId string, fileId domain.FullFileId) error {
close(uploaded)
return nil
})
Expand Down Expand Up @@ -237,7 +237,7 @@ func TestFileAddWithCustomKeys(t *testing.T) {
ctx := context.Background()

uploaded := make(chan struct{})
fx.fileSyncService.OnUploaded(func(objectId string) error {
fx.fileSyncService.OnUploaded(func(objectId string, fileId domain.FullFileId) error {
close(uploaded)
return nil
})
Expand Down
Loading
Loading