GO-2827 Merge branch 'go-3446-sqlite-tree-storage-race-condition-when…

…-updating-new' into go-2827-migration-unbind-user-files

.mockery.yaml

@@ -67,6 +67,9 @@ packages:
         config:
           dir: "{{.InterfaceDir}}/mock_blockservice"
           outpkg: "mock_blockservice"
+  github.com/anyproto/anytype-heart/core/block/import/common/syncer:
+    interfaces:
+      BlockService:
   github.com/anyproto/anytype-heart/core/subscription:
     interfaces:
       CollectionService:

core/acl/aclservice.go

@@ -13,6 +13,7 @@ import (
 	"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
 	"github.com/anyproto/any-sync/coordinator/coordinatorclient"
 	"github.com/anyproto/any-sync/coordinator/coordinatorproto"
+	"github.com/anyproto/any-sync/nodeconf"
 	"github.com/anyproto/any-sync/util/crypto"
 	"github.com/gogo/protobuf/types"
 	"github.com/ipfs/go-cid"
@@ -35,6 +36,10 @@ var log = logging.Logger(CName).Desugar()
 
 var sleepTime = time.Millisecond * 500
 
+type NodeConfGetter interface {
+	GetNodeConf() (conf nodeconf.Configuration)
+}
+
 type AccountPermissions struct {
 	Account     crypto.PubKey
 	Permissions model.ParticipantPermissions
@@ -46,7 +51,7 @@ type AclService interface {
 	RevokeInvite(ctx context.Context, spaceId string) error
 	GetCurrentInvite(ctx context.Context, spaceId string) (domain.InviteInfo, error)
 	ViewInvite(ctx context.Context, inviteCid cid.Cid, inviteFileKey crypto.SymKey) (domain.InviteView, error)
-	Join(ctx context.Context, spaceId string, inviteCid cid.Cid, inviteFileKey crypto.SymKey) error
+	Join(ctx context.Context, spaceId, networkId string, inviteCid cid.Cid, inviteFileKey crypto.SymKey) error
 	ApproveLeave(ctx context.Context, spaceId string, identities []crypto.PubKey) error
 	MakeShareable(ctx context.Context, spaceId string) error
 	StopSharing(ctx context.Context, spaceId string) error
@@ -63,14 +68,16 @@ func New() AclService {
 }
 
 type aclService struct {
-	joiningClient  aclclient.AclJoiningClient
-	spaceService   space.Service
-	inviteService  inviteservice.InviteService
-	accountService account.Service
-	coordClient    coordinatorclient.CoordinatorClient
+	nodeConfigGetter NodeConfGetter
+	joiningClient    aclclient.AclJoiningClient
+	spaceService     space.Service
+	inviteService    inviteservice.InviteService
+	accountService   account.Service
+	coordClient      coordinatorclient.CoordinatorClient
 }
 
 func (a *aclService) Init(ap *app.App) (err error) {
+	a.nodeConfigGetter = app.MustComponent[NodeConfGetter](ap)
 	a.joiningClient = app.MustComponent[aclclient.AclJoiningClient](ap)
 	a.spaceService = app.MustComponent[space.Service](ap)
 	a.accountService = app.MustComponent[account.Service](ap)
@@ -325,7 +332,10 @@ func (a *aclService) StopSharing(ctx context.Context, spaceId string) error {
 	return nil
 }
 
-func (a *aclService) Join(ctx context.Context, spaceId string, inviteCid cid.Cid, inviteFileKey crypto.SymKey) error {
+func (a *aclService) Join(ctx context.Context, spaceId, networkId string, inviteCid cid.Cid, inviteFileKey crypto.SymKey) error {
+	if a.nodeConfigGetter.GetNodeConf().NetworkId != networkId {
+		return fmt.Errorf("%w. Local network: '%s', network of space to join: '%s'", ErrDifferentNetwork, a.nodeConfigGetter.GetNodeConf().NetworkId, networkId)
+	}
 	invitePayload, err := a.inviteService.GetPayload(ctx, inviteCid, inviteFileKey)
 	if err != nil {
 		return convertedOrInternalError("get invite payload", err)

core/acl/aclservice_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/anyproto/any-sync/commonspace/spacesyncproto"
 	"github.com/anyproto/any-sync/coordinator/coordinatorclient/mock_coordinatorclient"
 	"github.com/anyproto/any-sync/coordinator/coordinatorproto"
+	"github.com/anyproto/any-sync/nodeconf"
 	"github.com/anyproto/any-sync/util/cidutil"
 	"github.com/anyproto/any-sync/util/crypto"
 	"github.com/ipfs/go-cid"
@@ -41,6 +42,22 @@ import (
 
 var ctx = context.Background()
 
+type mockConfig struct {
+	Config nodeconf.Configuration
+}
+
+func (c *mockConfig) Name() string {
+	return ""
+}
+
+func (c *mockConfig) Init(*app.App) (err error) {
+	return nil
+}
+
+func (c *mockConfig) GetNodeConf() (conf nodeconf.Configuration) {
+	return c.Config
+}
+
 type fixture struct {
 	*aclService
 	a                     *app.App
@@ -56,6 +73,7 @@ type fixture struct {
 	mockCommonSpace       *mock_commonspace.MockSpace
 	mockSpaceClient       *mock_aclclient.MockAclSpaceClient
 	mockAcl               *mock_list.MockAclList
+	mockConfig            *mockConfig
 }
 
 func newFixture(t *testing.T) *fixture {
@@ -75,12 +93,14 @@ func newFixture(t *testing.T) *fixture {
 		mockCommonSpace:       mock_commonspace.NewMockSpace(ctrl),
 		mockSpaceClient:       mock_aclclient.NewMockAclSpaceClient(ctrl),
 		mockAcl:               mock_list.NewMockAclList(ctrl),
+		mockConfig:            &mockConfig{},
 	}
 	fx.a.Register(testutil.PrepareMock(ctx, fx.a, fx.mockAccountService)).
 		Register(testutil.PrepareMock(ctx, fx.a, fx.mockJoiningClient)).
 		Register(testutil.PrepareMock(ctx, fx.a, fx.mockSpaceService)).
 		Register(testutil.PrepareMock(ctx, fx.a, fx.mockInviteService)).
 		Register(testutil.PrepareMock(ctx, fx.a, fx.mockCoordinatorClient)).
+		Register(fx.mockConfig).
 		Register(fx.aclService)
 	require.NoError(t, fx.a.Start(ctx))
 	return fx
@@ -374,7 +394,7 @@ func TestService_Join(t *testing.T) {
 		fx.mockSpaceService.EXPECT().Join(ctx, "spaceId", "aclHeadId").Return(nil)
 		fx.mockSpaceService.EXPECT().TechSpace().Return(&clientspace.TechSpace{TechSpace: fx.mockTechSpace})
 		fx.mockTechSpace.EXPECT().SpaceViewSetData(ctx, "spaceId", mock.Anything).Return(nil)
-		err = fx.Join(ctx, "spaceId", realCid, key)
+		err = fx.Join(ctx, "spaceId", "", realCid, key)
 		require.NoError(t, err)
 	})
 	t.Run("join fail, space is deleted", func(t *testing.T) {
@@ -399,7 +419,7 @@ func TestService_Join(t *testing.T) {
 			InviteKey: inviteKey,
 			Metadata:  metadataPayload,
 		}).Return("", coordinatorproto.ErrSpaceIsDeleted)
-		err = fx.Join(ctx, "spaceId", realCid, key)
+		err = fx.Join(ctx, "spaceId", "", realCid, key)
 		require.Equal(t, space.ErrSpaceDeleted, err)
 	})
 	t.Run("join success, already member", func(t *testing.T) {
@@ -426,9 +446,21 @@ func TestService_Join(t *testing.T) {
 		}).Return("", list.ErrInsufficientPermissions)
 		fx.mockJoiningClient.EXPECT().CancelRemoveSelf(ctx, "spaceId").Return(nil)
 		fx.mockSpaceService.EXPECT().CancelLeave(ctx, "spaceId").Return(nil)
-		err = fx.Join(ctx, "spaceId", realCid, key)
+		err = fx.Join(ctx, "spaceId", "", realCid, key)
 		require.NoError(t, err)
 	})
+	t.Run("join fail, different network", func(t *testing.T) {
+		// given
+		fx := newFixture(t)
+		defer fx.finish(t)
+		fx.mockConfig.Config = nodeconf.Configuration{NetworkId: "net1"}
+
+		// when
+		err := fx.Join(ctx, "spaceId", "net2", cid.Cid{}, nil)
+
+		// then
+		require.True(t, errors.Is(err, ErrDifferentNetwork))
+	})
 }
 
 func TestService_Leave(t *testing.T) {

core/acl/errors.go

@@ -19,6 +19,7 @@ var (
 	ErrAclRequestFailed     = errors.New("acl request failed")
 	ErrNotShareable         = errors.New("space is not shareable")
 	ErrLimitReached         = errors.New("limit reached")
+	ErrDifferentNetwork     = errors.New("different network")
 	ErrInternal             = errors.New("internal error")
 )
 

core/application/sessions.go

@@ -16,7 +16,11 @@ func (s *Service) CreateSession(req *pb.RpcWalletCreateSessionRequest) (token st
 	appKey := req.GetAppKey()
 
 	if appKey != "" {
-		wallet := s.app.Component(walletComp.CName)
+		app := s.GetApp()
+		if app == nil {
+			return "", "", ErrApplicationIsNotRunning
+		}
+		wallet := app.Component(walletComp.CName)
 		if wallet == nil {
 			return "", "", fmt.Errorf("appToken auth not yet supported for the main app")
 		}

core/application/sessions_test.go

@@ -0,0 +1,25 @@
+package application
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/anyproto/anytype-heart/pb"
+)
+
+func TestCreateSession(t *testing.T) {
+	t.Run("with app key", func(t *testing.T) {
+		t.Run("with not initialized app expect error", func(t *testing.T) {
+			s := New()
+
+			_, _, err := s.CreateSession(&pb.RpcWalletCreateSessionRequest{
+				Auth: &pb.RpcWalletCreateSessionRequestAuthOfAppKey{
+					AppKey: "appKey",
+				},
+			})
+
+			require.Error(t, err)
+		})
+	})
+}

core/block/bookmark/bookmark_service.go

@@ -123,7 +123,7 @@ func (s *service) CreateBookmarkObject(ctx context.Context, spaceID string, deta
 	}
 	url := pbtypes.GetString(details, bundle.RelationKeySource.String())
 
-	records, _, err := s.store.Query(database.Query{
+	records, err := s.store.Query(database.Query{
 		Sorts: []*model.BlockContentDataviewSort{
 			{
 				RelationKey: bundle.RelationKeyLastModifiedDate.String(),

core/block/editor.go

@@ -691,7 +691,7 @@ func (s *Service) ListConvertToObjects(
 	ctx session.Context, req pb.RpcBlockListConvertToObjectsRequest,
 ) (linkIds []string, err error) {
 	err = cache.Do(s, req.ContextId, func(b basic.CommonOperations) error {
-		linkIds, err = b.ExtractBlocksToObjects(ctx, s.objectCreator, req)
+		linkIds, err = b.ExtractBlocksToObjects(ctx, s.objectCreator, s.templateService, req)
 		return err
 	})
 	return

core/block/editor/archive.go

@@ -73,7 +73,7 @@ func (p *Archive) updateObjects(info smartblock.ApplyInfo) (err error) {
 		return
 	}
 
-	records, _, err := p.objectStore.Query(database.Query{
+	records, err := p.objectStore.Query(database.Query{
 		Filters: []*model.BlockContentDataviewFilter{
 			{
 				RelationKey: bundle.RelationKeyIsArchived.String(),

core/block/editor/basic/basic.go

@@ -54,7 +54,7 @@ type CommonOperations interface {
 
 	PasteBlocks(s *state.State, targetBlockId string, position model.BlockPosition, blocks []simple.Block) (err error)
 	ReplaceLink(oldId, newId string) error
-	ExtractBlocksToObjects(ctx session.Context, s ObjectCreator, req pb.RpcBlockListConvertToObjectsRequest) (linkIds []string, err error)
+	ExtractBlocksToObjects(ctx session.Context, oc ObjectCreator, tsc TemplateStateCreator, req pb.RpcBlockListConvertToObjectsRequest) (linkIds []string, err error)
 
 	SetObjectTypes(ctx session.Context, objectTypeKeys []domain.TypeKey, ignoreRestrictions bool) (err error)
 	SetObjectTypesInState(s *state.State, objectTypeKeys []domain.TypeKey, ignoreRestrictions bool) (err error)

core/block/editor/basic/extract_objects.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/anyproto/anytype-heart/core/block/editor/state"
 	"github.com/anyproto/anytype-heart/core/block/simple"
-	"github.com/anyproto/anytype-heart/core/block/simple/base"
 	"github.com/anyproto/anytype-heart/core/domain"
 	"github.com/anyproto/anytype-heart/core/session"
 	"github.com/anyproto/anytype-heart/pb"
@@ -22,9 +21,18 @@ type ObjectCreator interface {
 	CreateSmartBlockFromState(ctx context.Context, spaceID string, objectTypeKeys []domain.TypeKey, createState *state.State) (id string, newDetails *types.Struct, err error)
 }
 
+type TemplateStateCreator interface {
+	CreateTemplateStateWithDetails(templateId string, details *types.Struct) (*state.State, error)
+}
+
 // ExtractBlocksToObjects extracts child blocks from the object to separate objects and
 // replaces these blocks to the links to these objects
-func (bs *basic) ExtractBlocksToObjects(ctx session.Context, objectCreator ObjectCreator, req pb.RpcBlockListConvertToObjectsRequest) (linkIds []string, err error) {
+func (bs *basic) ExtractBlocksToObjects(
+	ctx session.Context,
+	objectCreator ObjectCreator,
+	templateStateCreator TemplateStateCreator,
+	req pb.RpcBlockListConvertToObjectsRequest,
+) (linkIds []string, err error) {
 	typeUniqueKey, err := domain.UnmarshalUniqueKey(req.ObjectTypeUniqueKey)
 	if err != nil {
 		return nil, fmt.Errorf("unmarshal unique key: %w", err)
@@ -37,14 +45,12 @@ func (bs *basic) ExtractBlocksToObjects(ctx session.Context, objectCreator Objec
 	for _, rootID := range rootIds {
 		rootBlock := newState.Pick(rootID)
 
-		objState := prepareTargetObjectState(newState, rootID, rootBlock, req)
-
-		details, err := bs.prepareTargetObjectDetails(bs.SpaceID(), req, typeUniqueKey, rootBlock, objectCreator)
+		objState, err := bs.prepareObjectState(typeUniqueKey, rootBlock, templateStateCreator, req)
 		if err != nil {
-			return nil, fmt.Errorf("extract blocks to objects: %w", err)
+			return nil, err
 		}
 
-		objState.SetDetails(details)
+		insertBlocksToState(newState, rootBlock, objState)
 
 		objectID, _, err := objectCreator.CreateSmartBlockFromState(
 			context.Background(),
@@ -67,12 +73,20 @@ func (bs *basic) ExtractBlocksToObjects(ctx session.Context, objectCreator Objec
 	return linkIds, bs.Apply(newState)
 }
 
+func (bs *basic) prepareObjectState(
+	uk domain.UniqueKey, root simple.Block, creator TemplateStateCreator, req pb.RpcBlockListConvertToObjectsRequest,
+) (*state.State, error) {
+	details, err := bs.prepareTargetObjectDetails(bs.SpaceID(), uk, root)
+	if err != nil {
+		return nil, fmt.Errorf("prepare target details: %w", err)
+	}
+	return creator.CreateTemplateStateWithDetails(req.TemplateId, details)
+}
+
 func (bs *basic) prepareTargetObjectDetails(
 	spaceID string,
-	req pb.RpcBlockListConvertToObjectsRequest,
 	typeUniqueKey domain.UniqueKey,
 	rootBlock simple.Block,
-	objectCreator ObjectCreator,
 ) (*types.Struct, error) {
 	objType, err := bs.objectStore.GetObjectByUniqueKey(spaceID, typeUniqueKey)
 	if err != nil {
@@ -83,15 +97,24 @@ func (bs *basic) prepareTargetObjectDetails(
 	return details, nil
 }
 
-func prepareTargetObjectState(newState *state.State, rootID string, rootBlock simple.Block, req pb.RpcBlockListConvertToObjectsRequest) *state.State {
+func insertBlocksToState(
+	newState *state.State,
+	rootBlock simple.Block,
+	objState *state.State,
+) {
+	rootID := rootBlock.Model().Id
 	descendants := newState.Descendants(rootID)
 	newRoot, newBlocks := reassignSubtreeIds(rootID, append(descendants, rootBlock))
+
+	// remove descendant blocks from source object
 	removeBlocks(newState, descendants)
 
-	objState := buildStateFromBlocks(newBlocks)
-	fixStateForNoteLayout(objState, req, newRoot)
-	injectSmartBlockContentToRootBlock(objState)
-	return objState
+	for _, b := range newBlocks {
+		objState.Add(b)
+	}
+	rootB := objState.Pick(objState.RootId()).Model()
+	rootB.ChildrenIds = append(rootB.ChildrenIds, newRoot)
+	objState.Set(simple.New(rootB))
 }
 
 func (bs *basic) changeToBlockWithLink(newState *state.State, blockToChange simple.Block, objectID string) (string, error) {
@@ -109,38 +132,6 @@ func (bs *basic) changeToBlockWithLink(newState *state.State, blockToChange simp
 	})
 }
 
-func injectSmartBlockContentToRootBlock(objState *state.State) {
-	rootID := objState.RootId()
-	rootBlock := objState.Get(rootID).Model()
-	rootBlock.Content = &model.BlockContentOfSmartblock{
-		Smartblock: &model.BlockContentSmartblock{},
-	}
-	objState.Set(simple.New(rootBlock))
-}
-
-func fixStateForNoteLayout(
-	objState *state.State,
-	req pb.RpcBlockListConvertToObjectsRequest,
-	newRoot string,
-) {
-	// todo: add check or remove this. It supposed to be run only for note
-	{
-		objState.Add(base.NewBase(&model.Block{
-			// This id will be replaced by id of the new object
-			Id:          "_root",
-			ChildrenIds: []string{newRoot},
-		}))
-	}
-}
-
-func buildStateFromBlocks(newBlocks []simple.Block) *state.State {
-	objState := state.NewDoc("", nil).NewState()
-	for _, b := range newBlocks {
-		objState.Add(b)
-	}
-	return objState
-}
-
 func removeBlocks(state *state.State, descendants []simple.Block) {
 	for _, b := range descendants {
 		state.Unlink(b.Model().Id)