#275 Remove full index loading for next rank in link index

cybercongress · Feb 27, 2019 · 9d42d9c · 9d42d9c
1 parent 5dcddd2
commit 9d42d9c
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 12 deletions.
diff --git a/x/link/keeper/link.go b/x/link/keeper/link.go
@@ -14,10 +14,15 @@ const (
 	LinksCountBytesSize = uint64(8)
 )
 
+type LinkFilter func(l CompactLink) bool
+
+var DefaultLinkFilter = func(l CompactLink) bool { return true }
+
 type LinkKeeper interface {
 	PutLink(ctx sdk.Context, link CompactLink)
 	IsLinkExist(ctx sdk.Context, link CompactLink) bool
 	GetAllLinks(ctx sdk.Context) (Links, Links, error)
+	GetAllLinksFiltered(ctx sdk.Context, filter LinkFilter) (Links, Links, error)
 	GetLinksCount(ctx sdk.Context) uint64
 	Iterate(ctx sdk.Context, process func(link CompactLink))
 	WriteLinks(ctx sdk.Context, writer io.Writer) (err error)
@@ -50,13 +55,19 @@ func (lk BaseLinkKeeper) IsLinkExist(ctx sdk.Context, link CompactLink) bool {
 }
 
 func (lk BaseLinkKeeper) GetAllLinks(ctx sdk.Context) (Links, Links, error) {
+	return lk.GetAllLinksFiltered(ctx, DefaultLinkFilter)
+}
+
+func (lk BaseLinkKeeper) GetAllLinksFiltered(ctx sdk.Context, filter LinkFilter) (Links, Links, error) {
 
 	inLinks := make(map[CidNumber]CidLinks)
 	outLinks := make(map[CidNumber]CidLinks)
 
 	lk.Iterate(ctx, func(link CompactLink) {
-		Links(outLinks).Put(link.From(), link.To(), link.Acc())
-		Links(inLinks).Put(link.To(), link.From(), link.Acc())
+		if filter(link) {
+			Links(outLinks).Put(link.From(), link.To(), link.Acc())
+			Links(inLinks).Put(link.To(), link.From(), link.Acc())
+		}
 	})
 
 	return inLinks, outLinks, nil

diff --git a/x/link/keeper/link_index.go b/x/link/keeper/link_index.go
@@ -17,8 +17,6 @@ type LinkIndexedKeeper struct {
 	currentRankOutLinks Links
 
 	// New links for the next rank calculation.
-	// Actually, do we need them in memory?
-	// TODO: optimize to not store whole index (store just new links)
 	nextRankInLinks  Links
 	nextRankOutLinks Links
 
@@ -38,7 +36,10 @@ func (i *LinkIndexedKeeper) Load(rankCtx sdk.Context, freshCtx sdk.Context) {
 	i.currentRankInLinks = inLinks
 	i.currentRankOutLinks = outLinks
 
-	newInLinks, newOutLinks, err := i.LinkKeeper.GetAllLinks(freshCtx)
+	newInLinks, newOutLinks, err := i.LinkKeeper.GetAllLinksFiltered(freshCtx, func(l CompactLink) bool {
+		return !i.currentRankOutLinks.IsLinkExist(l.From(), l.To(), l.Acc())
+	})
+
 	if err != nil {
 		cmn.Exit(err.Error())
 	}
@@ -48,17 +49,18 @@ func (i *LinkIndexedKeeper) Load(rankCtx sdk.Context, freshCtx sdk.Context) {
 }
 
 func (i *LinkIndexedKeeper) FixLinks() {
-	// todo state copied
-	i.currentRankInLinks = Links(i.nextRankInLinks).Copy()
-	i.currentRankOutLinks = Links(i.nextRankOutLinks).Copy()
+	i.currentRankInLinks.PutAll(i.nextRankInLinks)
+	i.currentRankOutLinks.PutAll(i.nextRankOutLinks)
+	i.nextRankInLinks = make(Links)
+	i.nextRankOutLinks = make(Links)
 }
 
 // return true if this block has new links
 func (i *LinkIndexedKeeper) EndBlocker() bool {
 	hasNewLinks := len(i.currentBlockLinks) > 0
 	for _, link := range i.currentBlockLinks {
-		Links(i.nextRankOutLinks).Put(link.From(), link.To(), link.Acc())
-		Links(i.nextRankInLinks).Put(link.To(), link.From(), link.Acc())
+		i.nextRankOutLinks.Put(link.From(), link.To(), link.Acc())
+		i.nextRankInLinks.Put(link.To(), link.From(), link.Acc())
 	}
 	i.currentBlockLinks = make([]CompactLink, 0, 1000) // todo: 1000 hardcoded value
 	return hasNewLinks
@@ -102,11 +104,12 @@ func (i *LinkIndexedKeeper) GetCurrentBlockNewLinks() []CompactLink {
 }
 
 func (i *LinkIndexedKeeper) IsAnyLinkExist(from CidNumber, to CidNumber) bool {
-	return i.nextRankOutLinks.IsAnyLinkExist(from, to)
+	return i.currentRankOutLinks.IsAnyLinkExist(from, to) || i.nextRankOutLinks.IsAnyLinkExist(from, to)
 }
 
 func (i *LinkIndexedKeeper) IsLinkExist(link CompactLink) bool {
-	return i.nextRankOutLinks.IsLinkExist(link.From(), link.To(), link.Acc())
+	return i.currentRankOutLinks.IsLinkExist(link.From(), link.To(), link.Acc()) ||
+		i.nextRankOutLinks.IsLinkExist(link.From(), link.To(), link.Acc())
 }
 
 //todo: remove duplicated method (BaseLinksKeeper)

diff --git a/x/link/types/cid.go b/x/link/types/cid.go
@@ -26,6 +26,16 @@ func (links Links) Put(from CidNumber, to CidNumber, acc AccNumber) {
 	links[from] = cidLinks
 }
 
+func (links Links) PutAll(newLinks Links) {
+	for from := range newLinks {
+		for to := range newLinks[from] {
+			for u := range newLinks[from][to] {
+				links.Put(from, to, u)
+			}
+		}
+	}
+}
+
 func (links Links) Copy() Links {
 
 	linksCopy := make(Links, len(links))