En 7527 index accounts #2304
Merged
En 7527 index accounts #2304
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Conflicts: # core/indexer/elasticProcessor_test.go
miiu96
requested changes
Sep 22, 2020
core/indexer/elasticProcessor.go
Outdated
| Refresh: "true", | ||
| } | ||
|
|
||
| return ei.elasticClient.DoRequest(req) |
There was a problem hiding this comment.
instead of DoRequest you have to use DoBulkRequest to save more than one account per request. You should create a request with size smaller than 1MB and send it.
| enabledIndexesMap[index] = struct{}{} | ||
| } | ||
| if len(enabledIndexesMap) == 0 { | ||
| return nil, fmt.Errorf("empty elastic search enabled indexes map") |
core/indexer/processTransactions.go
Outdated
| @@ -259,6 +264,19 @@ func (tdp *txDatabaseProcessor) setTransactionSearchOrder(transactions map[strin | |||
| return transactions | |||
| } | |||
|
|
|||
| func addToAlteredAddresses(tx *Transaction, alteredAddresses map[string]struct{}, miniBlock *block.MiniBlock, selfShardID uint32) { | |||
| if selfShardID == miniBlock.SenderShardID { | |||
There was a problem hiding this comment.
you can refactor here to be more clear.
isIntraShard ---> save sender and receiver address
isCrossShardDstMe ---> save receiver
else if isNotRewardMB --> save sender
There was a problem hiding this comment.
refactored after Iulian's suggestion
core/indexer/common.go
Outdated
| @@ -391,7 +392,21 @@ func GetElasticTemplatesAndPolicies() (map[string]*bytes.Buffer, map[string]*byt | |||
|
|
|||
| func getTemplateByIndex(index string) *bytes.Buffer { | |||
| indexTemplate := &bytes.Buffer{} | |||
| _ = core.LoadJsonFile(&indexTemplate, "./config/elasticIndexTemplates/"+index+".json") | |||
|
|
|||
| // TODO: (maybe) un-do this code before merging. for some reason, in local tests, the older version did not work | |||
There was a problem hiding this comment.
Remove TODO or code?
| @@ -126,3 +126,47 @@ func TestMaxIntShouldReturnB(t *testing.T) { | |||
| b := 11 | |||
| assert.Equal(t, b, core.MaxInt(a, b)) | |||
| } | |||
|
|
|||
| func TestMaxFloat64(t *testing.T) { | |||
| selfShardID uint32, | ||
| isRewardTx bool, | ||
| ) { | ||
| if selfShardID == miniBlock.SenderShardID && !isRewardTx { |
miiu96
reviewed
Sep 25, 2020
| } | ||
| } | ||
| } | ||
| } |
| @@ -641,8 +689,10 @@ func indexGenesisBlocks(args *processComponentsFactoryArgs, genesisBlocks map[ui | |||
| return err | |||
| } | |||
|
|
|||
| log.Info("indexGenesisBlocks(): indexer.SaveBlock", "hash", genesisBlockHash) | |||
| args.indexer.SaveBlock(&dataBlock.Body{}, genesisBlockHeader, nil, nil, nil) | |||
| if !args.indexer.IsNilIndexer() { | |||
core/indexer/errors.go
Outdated
| @@ -30,3 +30,9 @@ var ErrNilOptions = errors.New("nil options") | |||
|
|
|||
| // ErrNegativeCacheSize signals that a invalid cache size has been provided | |||
| var ErrNegativeCacheSize = errors.New("negative cache size") | |||
|
|
|||
| // ErrNilAccountsDB signals that a nil accounts db has been provided | |||
There was a problem hiding this comment.
dont use shortcuts in comments
iulianpascalau
approved these changes
Sep 30, 2020
iulianpascalau
approved these changes
Oct 5, 2020
miiu96
reviewed
Oct 5, 2020
core/indexer/common.go
Outdated
| serializedData = append(serializedData, "\n"...) | ||
|
|
||
| buffLenWithCurrentAcc := buff.Len() + len(meta) + len(serializedData) | ||
| if buffLenWithCurrentAcc > txsBulkSizeThreshold && buff.Len() != 0 { |
There was a problem hiding this comment.
maybe you can rename this const txsBulkSizeThreshold
core/indexer/common.go
Outdated
| } | ||
| _, err = buff.Write(serializedData) | ||
| if err != nil { | ||
| log.Warn("elastic search: serialize bulk accounts history, write serialized tx", "error", err.Error()) |
There was a problem hiding this comment.
`write serialized account hisotory1
core/indexer/common.go
Outdated
| } | ||
| _, err = buff.Write(serializedData) | ||
| if err != nil { | ||
| log.Warn("elastic search: serialize bulk accounts, write serialized tx", "error", err.Error()) |
core/indexer/common.go
Outdated
| // append a newline for each element | ||
| serializedData = append(serializedData, "\n"...) | ||
|
|
||
| buffLenWithCurrentAcc := buff.Len() + len(meta) + len(serializedData) |
There was a problem hiding this comment.
buffLenWithCurrentAccHistory
core/indexer/elasticProcessor.go
Outdated
| return ei.saveAccountsHistory(accountsMap) | ||
| } | ||
|
|
||
| func (ei *elasticProcessor) saveAccountsHistory(accountInfoMap map[string]*AccountInfo) error { |
miiu96
approved these changes
Oct 6, 2020
miiu96
approved these changes
Oct 6, 2020
iulianpascalau
approved these changes
Oct 6, 2020
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This PR adds 2 functionalities:
transactionsthere was the option to index or not. Now, all indexes can be configured.accountsindex that will contain data about used accounts (from genesis and from transactions).Testing procedure:
elasticURL/accounts/_searchwhere you should see entries for each used account (genesis + from transactions). The format of an account looks like this:{ "_index": "accounts", "_type": "_doc", "_id": "erd19j8u0kstdgwmgaqm23dh8lj0g5vxm6skg80yrp237rxwjneyz3qqqjmglw", "_score": null, "_source": { "nonce": 0, "balance": "2220555555555555555555560", "balanceNum": 2220555.5555555555 } }/address/endpoint for a few samples.