Merge pull request #4246 from ElrondNetwork/merge-dev-into-liquid-sta…

…king-30-08-2022

Merge dev into liquid staking 30 08 2022

cmd/node/config/config.toml

@@ -32,8 +32,8 @@
     # than this value, then the validators rating decrease should stop so they won't get jailed
     MaxConsecutiveRoundsOfRatingDecrease = 600
 
-   # SyncProcessTimeInMillis is the value in milliseconds used when processing blocks while synchronizing blocks
-   SyncProcessTimeInMillis = 12000
+    # SyncProcessTimeInMillis is the value in milliseconds used when processing blocks while synchronizing blocks
+    SyncProcessTimeInMillis = 12000
 
 [Versions]
     DefaultVersion = "default"
@@ -703,9 +703,10 @@
         ]
 
     [VirtualMachine.GasConfig]
-        # MaxGasPerVmQuery defines the maximum amount of gas to be allocated for VM Queries coming from API
+        # The following values define the maximum amount of gas to be allocated for VM Queries coming from API
         # If set to 0, then MaxUInt64 will be used
-        MaxGasPerVmQuery = 1500000000  #1.5b
+        ShardMaxGasPerVmQuery = 1500000000  #1.5b
+        MetaMaxGasPerVmQuery = 0  #unlimited
 
 [BuiltInFunctions]
     AutomaticCrawlerAddress = "erd1fpkcgel4gcmh8zqqdt043yfcn5tyx8373kg6q2qmkxzu4dqamc0swts65c"

cmd/node/config/enableEpochs.toml

@@ -194,14 +194,14 @@
     # DisableExecByCallerEnableEpoch represents the epoch when the check on value is disabled on exec by caller
     DisableExecByCallerEnableEpoch = 1
 
-   	# FailExecutionOnEveryAPIErrorEnableEpoch represent the epoch when new protection in VM is enabled to fail all wrong API calls
-   	FailExecutionOnEveryAPIErrorEnableEpoch = 3
+    # FailExecutionOnEveryAPIErrorEnableEpoch represent the epoch when new protection in VM is enabled to fail all wrong API calls
+    FailExecutionOnEveryAPIErrorEnableEpoch = 3
 
     # ManagedCryptoAPIsEnableEpoch represents the epoch when new managed crypto APIs are enabled in the wasm VM
     ManagedCryptoAPIsEnableEpoch = 4
 
     # RefactorContextEnableEpoch represents the epoch when refactoring/simplifying is enabled in contexts
-    RefactorContextEnableEpoch = 2
+    RefactorContextEnableEpoch = 1
 
     # ESDTMetadataContinuousCleanupEnableEpoch represents the epoch when esdt metadata is automatically deleted according to inshard liquidity
     ESDTMetadataContinuousCleanupEnableEpoch = 4

cmd/node/config/external.toml

@@ -14,6 +14,7 @@
     # EnabledIndexes represents a slice of indexes that will be enabled for indexing. Full list is:
     # ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations"]
     EnabledIndexes    = ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations"]
+
 # EventNotifierConnector defines settings needed to configure and launch the event notifier component
 [EventNotifierConnector]
     # Enabled will turn on or off the event notifier connector

cmd/node/flags.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/ElrondNetwork/elrond-go-core/core"
 	logger "github.com/ElrondNetwork/elrond-go-logger"
 	"github.com/ElrondNetwork/elrond-go/common"
 	"github.com/ElrondNetwork/elrond-go/config"
@@ -140,6 +139,12 @@ var (
 		Name:  "use-health-service",
 		Usage: "Boolean option for enabling the health service.",
 	}
+	// memoryUsageToCreateProfiles is used to set a custom value for the memory usage threshold (in bytes)
+	memoryUsageToCreateProfiles = cli.Uint64Flag{
+		Name:  "memory-usage-to-create-profiles",
+		Usage: "Integer value to be used to set the memory usage thresholds (in bytes)",
+		Value: 2415919104, // 2.25GB
+	}
 	// validatorKeyIndex defines a flag that specifies the 0-th based index of the private key to be used from validatorKey.pem file
 	validatorKeyIndex = cli.IntFlag{
 		Name:  "sk-index",
@@ -189,7 +194,7 @@ var (
 		Value: "",
 	}
 
-	//useLogView is a deprecated flag, but kept for backwards compatibility
+	// useLogView is a deprecated flag, but kept for backwards compatibility
 	useLogView = cli.BoolFlag{
 		Name: "use-log-view",
 		Usage: "Deprecated flag. This flag's value is not used anymore as the only way the node starts now is within " +
@@ -212,17 +217,17 @@ var (
 			" log level.",
 		Value: "*:" + logger.LogInfo.String(),
 	}
-	//logFile is used when the log output needs to be logged in a file
+	// logFile is used when the log output needs to be logged in a file
 	logSaveFile = cli.BoolFlag{
 		Name:  "log-save",
 		Usage: "Boolean option for enabling log saving. If set, it will automatically save all the logs into a file.",
 	}
-	//logWithCorrelation is used to enable log correlation elements
+	// logWithCorrelation is used to enable log correlation elements
 	logWithCorrelation = cli.BoolFlag{
 		Name:  "log-correlation",
 		Usage: "Boolean option for enabling log correlation elements.",
 	}
-	//logWithLoggerName is used to enable log correlation elements
+	// logWithLoggerName is used to enable log correlation elements
 	logWithLoggerName = cli.BoolFlag{
 		Name:  "log-logger-name",
 		Usage: "Boolean option for logger name in the logs.",
@@ -365,6 +370,7 @@ func getFlags() []cli.Flag {
 		redundancyLevel,
 		fullArchive,
 		memBallast,
+		memoryUsageToCreateProfiles,
 	}
 }
 
@@ -415,6 +421,11 @@ func applyFlags(ctx *cli.Context, cfgs *config.Configs, flagsConfig *config.Cont
 	if ctx.IsSet(fullArchive.Name) {
 		cfgs.PreferencesConfig.Preferences.FullArchive = ctx.GlobalBool(fullArchive.Name)
 	}
+	if ctx.IsSet(memoryUsageToCreateProfiles.Name) {
+		cfgs.GeneralConfig.Health.MemoryUsageToCreateProfiles = int(ctx.GlobalUint64(memoryUsageToCreateProfiles.Name))
+		log.Info("setting a new value for the memoryUsageToCreateProfiles option",
+			"new value", cfgs.GeneralConfig.Health.MemoryUsageToCreateProfiles)
+	}
 
 	importDbDirectoryValue := ctx.GlobalString(importDbDirectory.Name)
 	importDBConfigs := &config.ImportDbConfig{
@@ -499,14 +510,6 @@ func processConfigImportDBMode(log logger.Logger, configs *config.Configs) error
 	p2pConfigs.Node.ThresholdMinConnectedPeers = 0
 	p2pConfigs.KadDhtPeerDiscovery.Enabled = false
 
-	defaultSecondsToCheckHealth := 300                   // 5minutes
-	defaultDiagnoseMemoryLimit := 6 * 1024 * 1024 * 1024 // 6GB
-
-	generalConfigs.Health.IntervalDiagnoseComponentsDeeplyInSeconds = defaultSecondsToCheckHealth
-	generalConfigs.Health.IntervalDiagnoseComponentsInSeconds = defaultSecondsToCheckHealth
-	generalConfigs.Health.IntervalVerifyMemoryInSeconds = defaultSecondsToCheckHealth
-	generalConfigs.Health.MemoryUsageToCreateProfiles = defaultDiagnoseMemoryLimit
-
 	alterStorageConfigsForDBImport(generalConfigs)
 
 	log.Warn("the node is in import mode! Will auto-set some config values, including storage config values",
@@ -520,10 +523,6 @@ func processConfigImportDBMode(log logger.Logger, configs *config.Configs) error
 		"import DB start in epoch", importDbFlags.ImportDBStartInEpoch,
 		"import DB shard ID", importDbFlags.ImportDBTargetShardID,
 		"kad dht discoverer", "off",
-		"health interval diagnose components deeply in seconds", generalConfigs.Health.IntervalDiagnoseComponentsDeeplyInSeconds,
-		"health interval diagnose components in seconds", generalConfigs.Health.IntervalDiagnoseComponentsInSeconds,
-		"health interval verify memory in seconds", generalConfigs.Health.IntervalVerifyMemoryInSeconds,
-		"health memory usage threshold", core.ConvertBytes(uint64(generalConfigs.Health.MemoryUsageToCreateProfiles)),
 	)
 	return nil
 }

config/config.go

@@ -415,7 +415,8 @@ type QueryVirtualMachineConfig struct {
 
 // VirtualMachineGasConfig holds the configuration for the virtual machine(s) gas operations
 type VirtualMachineGasConfig struct {
-	MaxGasPerVmQuery uint64
+	ShardMaxGasPerVmQuery uint64
+	MetaMaxGasPerVmQuery  uint64
 }
 
 // BuiltInFunctionsConfig holds the configuration for the built in functions

config/tomlConfig_test.go

@@ -107,7 +107,8 @@ func TestTomlParser(t *testing.T) {
 				VirtualMachineConfig: VirtualMachineConfig{ArwenVersions: arwenVersions},
 			},
 			GasConfig: VirtualMachineGasConfig{
-				MaxGasPerVmQuery: 1_500_000_000,
+				ShardMaxGasPerVmQuery: 1_500_000_000,
+				MetaMaxGasPerVmQuery:  0,
 			},
 		},
 		Debug: DebugConfig{
@@ -196,7 +197,8 @@ func TestTomlParser(t *testing.T) {
         ]
 
 	[VirtualMachine.GasConfig]
-		MaxGasPerVmQuery = 1500000000
+		ShardMaxGasPerVmQuery = 1500000000
+		MetaMaxGasPerVmQuery = 0
 
 [Debug]
     [Debug.InterceptorResolver]

examples/construction_test.go

@@ -19,12 +19,14 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var addressEncoder, _ = pubkeyConverter.NewBech32PubkeyConverter(32, &mock.LoggerMock{})
-var signingMarshalizer = &marshal.JsonMarshalizer{}
-var signer = &singlesig.Ed25519Signer{}
-var signingCryptoSuite = ed25519.NewEd25519()
-var contentMarshalizer = &marshal.GogoProtoMarshalizer{}
-var contentHasher = blake2b.NewBlake2b()
+var (
+	addressEncoder, _  = pubkeyConverter.NewBech32PubkeyConverter(32, &mock.LoggerMock{})
+	signingMarshalizer = &marshal.JsonMarshalizer{}
+	signer             = &singlesig.Ed25519Signer{}
+	signingCryptoSuite = ed25519.NewEd25519()
+	contentMarshalizer = &marshal.GogoProtoMarshalizer{}
+	contentHasher      = blake2b.NewBlake2b()
+)
 
 const alicePrivateKeyHex = "413f42575f7f26fad3317a778771212fdb80245850981e48b58a4f25e344e8f9"
 

factory/apiResolverFactory.go

@@ -343,7 +343,10 @@ func createScQueryElement(
 		NilCompiledSCStore:    true,
 	}
 
+	maxGasForVmQueries := args.generalConfig.VirtualMachine.GasConfig.ShardMaxGasPerVmQuery
 	if args.processComponents.ShardCoordinator().SelfId() == core.MetachainShardId {
+		maxGasForVmQueries = args.generalConfig.VirtualMachine.GasConfig.MetaMaxGasPerVmQuery
+
 		blockChainHookImpl, errBlockChainHook := hooks.NewBlockChainHookImpl(argsHook)
 		if errBlockChainHook != nil {
 			return nil, errBlockChainHook
@@ -404,6 +407,8 @@ func createScQueryElement(
 		}
 	}
 
+	log.Debug("maximum gas per VM Query", "value", maxGasForVmQueries)
+
 	vmContainer, err := vmFactory.Create()
 	if err != nil {
 		return nil, err
@@ -422,7 +427,7 @@ func createScQueryElement(
 		ArwenChangeLocker:        args.coreComponents.ArwenChangeLocker(),
 		Bootstrapper:             args.bootstrapper,
 		AllowExternalQueriesChan: args.allowVMQueriesChan,
-		MaxGasLimitPerQuery:      args.generalConfig.VirtualMachine.GasConfig.MaxGasPerVmQuery,
+		MaxGasLimitPerQuery:      maxGasForVmQueries,
 	}
 
 	return smartContract.NewSCQueryService(argsNewSCQueryService)

factory/apiResolverFactory_test.go

@@ -12,6 +12,10 @@ import (
 )
 
 func TestCreateApiResolver(t *testing.T) {
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
+
 	shardCoordinator := mock.NewMultiShardsCoordinatorMock(1)
 	coreComponents := getCoreComponents()
 	coreComponents.StatusHandlerUtils().Metrics()

factory/blockProcessorCreator_test.go

@@ -26,6 +26,9 @@ import (
 
 func Test_newBlockProcessorCreatorForShard(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
 
 	shardCoordinator := mock.NewMultiShardsCoordinatorMock(2)
 	pcf, _ := factory.NewProcessComponentsFactory(getProcessComponentsArgs(shardCoordinator))
@@ -58,6 +61,9 @@ func Test_newBlockProcessorCreatorForShard(t *testing.T) {
 
 func Test_newBlockProcessorCreatorForMeta(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
 
 	coreComponents := getCoreComponents()
 	shardC := mock.NewMultiShardsCoordinatorMock(1)

factory/bootstrapComponentsHandler_test.go

@@ -12,6 +12,9 @@ import (
 // ------------ Test ManagedBootstrapComponents --------------------
 func TestNewManagedBootstrapComponents(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
 
 	args := getBootStrapArgs()
 	bcf, _ := factory.NewBootstrapComponentsFactory(args)
@@ -23,15 +26,21 @@ func TestNewManagedBootstrapComponents(t *testing.T) {
 
 func TestNewBootstrapComponentsFactory_NilFactory(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
 
 	mbc, err := factory.NewManagedBootstrapComponents(nil)
 
 	require.Nil(t, mbc)
 	require.Equal(t, errorsErd.ErrNilBootstrapComponentsFactory, err)
 }
 
-func TestManagedBootstrapComponents_CheckSubcomponents_NoCreate(t *testing.T) {
+func TestManagedBootstrapComponents_CheckSubcomponentsNoCreate(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
 
 	args := getBootStrapArgs()
 	bcf, _ := factory.NewBootstrapComponentsFactory(args)
@@ -43,6 +52,9 @@ func TestManagedBootstrapComponents_CheckSubcomponents_NoCreate(t *testing.T) {
 
 func TestManagedBootstrapComponents_Create(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
 
 	args := getBootStrapArgs()
 	bcf, _ := factory.NewBootstrapComponentsFactory(args)
@@ -55,8 +67,11 @@ func TestManagedBootstrapComponents_Create(t *testing.T) {
 	require.Nil(t, err)
 }
 
-func TestManagedBootstrapComponents_Create_NilInternalMarshalizer(t *testing.T) {
+func TestManagedBootstrapComponents_CreateNilInternalMarshalizer(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
 
 	args := getBootStrapArgs()
 	coreComponents := getDefaultCoreComponents()
@@ -71,6 +86,10 @@ func TestManagedBootstrapComponents_Create_NilInternalMarshalizer(t *testing.T)
 
 func TestManagedBootstrapComponents_Close(t *testing.T) {
 	t.Parallel()
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
+
 	args := getBootStrapArgs()
 
 	bcf, _ := factory.NewBootstrapComponentsFactory(args)