[DFI-594] vm mod: retry mechanism refactoring (#183)

* [DFI-594] vm mod: retry mechanism refactoring (option to disable request timeout) * [DFI-594] Linter fix * [DFI-594] Sleep added (useful for tests only)
dfinance · Aug 5, 2020 · 89b6c8b · 89b6c8b
1 parent a7ec62d
commit 89b6c8b
Show file tree

Hide file tree

Showing 16 changed files with 456 additions and 185 deletions.
diff --git a/app/app.go b/app/app.go
@@ -263,7 +263,7 @@ func NewDnServiceApp(logger log.Logger, db dbm.DB, config *config.VMConfig, invC
 	)
 
 	// Upgrade handler with name matching proposal name should be registered here.
-	//app.upgradeKeeper.SetUpgradeHandler("My_update", func(ctx sdk.Context, plan upgrade.Plan) { })
+	app.upgradeKeeper.SetUpgradeHandler("v0.6.1", func(ctx sdk.Context, plan upgrade.Plan) {})
 
 	// VMKeeper stores VM resources and interacts with DVM.
 	app.vmKeeper = vm.NewKeeper(

diff --git a/app/common_test.go b/app/common_test.go
@@ -236,12 +236,10 @@ func NewTestDnAppDVM(t *testing.T, logOpts ...log.Option) (*DnServiceApp, string
 
 	// create VM config
 	config := &vmConfig.VMConfig{
-		Address:           dvmAddr,
-		DataListen:        dsAddr,
-		MaxAttempts:       10,
-		InitialBackoff:    500,
-		MaxBackoff:        1500,
-		BackoffMultiplier: 0.1,
+		Address:        dvmAddr,
+		DataListen:     dsAddr,
+		MaxAttempts:    10,
+		ReqTimeoutInMs: 1000,
 	}
 
 	// create app

diff --git a/app/integ_gov_test.go b/app/integ_gov_test.go
@@ -63,7 +63,7 @@ func TestIntegGov_StdlibUpdate(t *testing.T) {
 		t,
 		true,
 		cliTester.DaemonLogLevelOption("x/vm/dsserver:info,x/vm:info,x/gov:info,main:info,state:info,*:error"),
-		cliTester.VMCommunicationOption(50, 1000, 100),
+		cliTester.VMCommunicationOption(5, 1000),
 		cliTester.VMCommunicationBaseAddressNetOption("tcp://127.0.0.1"),
 	)
 	defer ct.Close()

diff --git a/app/integ_linux_test.go b/app/integ_linux_test.go
@@ -35,7 +35,7 @@ func TestIntegVM_CommunicationUDSOverDocker(t *testing.T) {
 	ct := cliTester.New(
 		t,
 		false,
-		cliTester.VMCommunicationOption(50, 1000, 100),
+		cliTester.VMCommunicationOption(5, 1000),
 		cliTester.VMCommunicationBaseAddressUDSOption(dsSocket, dvmSocket),
 	)
 	defer ct.Close()

diff --git a/app/integ_test.go b/app/integ_test.go
@@ -3,10 +3,8 @@
 package app
 
 import (
-	"context"
 	"encoding/hex"
 	"io/ioutil"
-	"net"
 	"os"
 	"path"
 	"strings"
@@ -15,75 +13,17 @@ import (
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/server"
-	"github.com/dfinance/dvm-proto/go/vm_grpc"
 	"github.com/stretchr/testify/require"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	grpcStatus "google.golang.org/grpc/status"
 
 	"github.com/dfinance/dnode/helpers"
 	"github.com/dfinance/dnode/helpers/tests"
 	cliTester "github.com/dfinance/dnode/helpers/tests/clitester"
+	"github.com/dfinance/dnode/helpers/tests/mockdvm"
 	testUtils "github.com/dfinance/dnode/helpers/tests/utils"
 	"github.com/dfinance/dnode/x/vm"
 	"github.com/dfinance/dnode/x/vm/client/vm_client"
 )
 
-type MockDVM struct {
-	server        *grpc.Server
-	failExecution bool
-	failResponse  bool
-	execDelay     time.Duration
-}
-
-func (s *MockDVM) SetExecutionFail() { s.failExecution = true }
-func (s *MockDVM) SetExecutionOK()   { s.failExecution = false }
-func (s *MockDVM) SetResponseFail()  { s.failResponse = true }
-func (s *MockDVM) SetResponseOK()    { s.failResponse = false }
-func (s *MockDVM) SetExecutionDelay(dur time.Duration) {
-	s.execDelay = dur
-}
-func (s *MockDVM) Stop() {
-	if s.server != nil {
-		s.server.Stop()
-	}
-}
-
-func (s *MockDVM) PublishModule(ctx context.Context, in *vm_grpc.VMPublishModule) (*vm_grpc.VMExecuteResponse, error) {
-	if s.failExecution {
-		return nil, grpcStatus.Errorf(codes.Internal, "failing gRPC execution")
-	}
-
-	resp := &vm_grpc.VMExecuteResponse{}
-	if !s.failResponse {
-		resp = &vm_grpc.VMExecuteResponse{
-			WriteSet:     nil,
-			Events:       nil,
-			GasUsed:      1,
-			Status:       vm_grpc.ContractStatus_Discard,
-			StatusStruct: nil,
-		}
-	}
-
-	return resp, nil
-}
-
-func StartMockDVMService(listener net.Listener) *MockDVM {
-	s := &MockDVM{
-		execDelay: 100 * time.Millisecond,
-	}
-
-	server := grpc.NewServer()
-	vm_grpc.RegisterVMModulePublisherServer(server, s)
-
-	go func() {
-		server.Serve(listener)
-	}()
-	s.server = server
-
-	return s
-}
-
 // Test dnode crash on VM Tx failure
 func TestInteg_ConsensusFailure(t *testing.T) {
 	const script = `
@@ -167,7 +107,7 @@ func TestIntegVM_ExecuteScriptViaCLI(t *testing.T) {
 	ct := cliTester.New(
 		t,
 		false,
-		cliTester.VMCommunicationOption(50, 1000, 100),
+		cliTester.VMCommunicationOption(5, 1000),
 		cliTester.VMCommunicationBaseAddressNetOption("tcp://127.0.0.1"),
 	)
 	defer ct.Close()
@@ -254,7 +194,7 @@ func TestIntegVM_ExecuteScriptViaREST(t *testing.T) {
 	ct := cliTester.New(
 		t,
 		false,
-		cliTester.VMCommunicationOption(50, 1000, 100),
+		cliTester.VMCommunicationOption(5, 1000),
 		cliTester.VMCommunicationBaseAddressNetOption("tcp://127.0.0.1"),
 	)
 	defer ct.Close()
@@ -358,7 +298,7 @@ func TestIntegVM_DeployModuleViaCLI(t *testing.T) {
 	ct := cliTester.New(
 		t,
 		false,
-		cliTester.VMCommunicationOption(50, 1000, 100),
+		cliTester.VMCommunicationOption(5, 1000),
 		cliTester.VMCommunicationBaseAddressNetOption("tcp://127.0.0.1"),
 	)
 	defer ct.Close()
@@ -417,7 +357,7 @@ func TestIntegVM_DeployModuleViaREST(t *testing.T) {
 	ct := cliTester.New(
 		t,
 		false,
-		cliTester.VMCommunicationOption(50, 1000, 100),
+		cliTester.VMCommunicationOption(5, 1000),
 		cliTester.VMCommunicationBaseAddressNetOption("tcp://127.0.0.1"),
 	)
 	defer ct.Close()
@@ -486,7 +426,7 @@ func TestIntegVM_RequestRetry(t *testing.T) {
 	ct := cliTester.New(
 		t,
 		true,
-		cliTester.VMCommunicationOption(100, 500, 10),
+		cliTester.VMCommunicationOption(5, 100),
 		cliTester.VMCommunicationBaseAddressUDSOption(dsSocket, mockDVMSocket),
 	)
 	defer ct.Close()
@@ -496,7 +436,7 @@ func TestIntegVM_RequestRetry(t *testing.T) {
 	mockDVMListener, err := helpers.GetGRpcNetListener("unix://" + mockDVMSocketPath)
 	require.NoError(t, err, "creating MockDVM listener")
 
-	mockDvm := StartMockDVMService(mockDVMListener)
+	mockDvm := mockdvm.StartMockDVMService(mockDVMListener)
 	defer mockDvm.Stop()
 	require.NoError(t, testUtils.WaitForFileExists(mockDVMSocketPath, 1*time.Second), "MockDVM start failed")
 
@@ -580,7 +520,7 @@ func TestIntegVM_CommunicationUDS(t *testing.T) {
 	ct := cliTester.New(
 		t,
 		false,
-		cliTester.VMCommunicationOption(50, 1000, 100),
+		cliTester.VMCommunicationOption(5, 1000),
 		cliTester.VMCommunicationBaseAddressUDSOption(dsSocket, dvmSocket),
 	)
 	defer ct.Close()

diff --git a/cmd/config/config.go b/cmd/config/config.go
@@ -31,10 +31,8 @@ const (
 	DefaultDataListen = "tcp://127.0.0.1:50052" // Default data server address to listen for connections from VM.
 
 	// Default retry configs.
-	DefaultMaxAttempts       = 0    // Default VM retry attempts.
-	DefaultInitialBackoff    = 1000 // Default VM 100 milliseconds for retry attempts.
-	DefaultMaxBackoff        = 2000 // Default VM max backoff.
-	DefaultBackoffMultiplier = 0.1  // Default backoff multiplayer (10)
+	DefaultMaxAttempts = 0 // Default maximum attempts for retry.
+	DefaultReqTimeout  = 0 // Default request timeout per attempt [ms].
 
 	// Default governance params.
 	DefaultGovMinDepositAmount = "100000000000000000000" // 100 dfi
@@ -52,23 +50,18 @@ type VMConfig struct {
 	Address    string `mapstructure:"vm_address"`     // address of virtual machine.
 	DataListen string `mapstructure:"vm_data_listen"` // data listen.
 
-	// Retry policy.
-	// Example how backoff works - https://stackoverflow.com/questions/43224683/what-does-backoffmultiplier-mean-in-defaultretrypolicy.
-	MaxAttempts       int     `mapstructure:"vm_retry_max_attempts"`       // maximum attempts for retry, for infinity retry - use 0.
-	InitialBackoff    int     `mapstructure:"vm_retry_initial_backoff"`    // initial back off in ms.
-	MaxBackoff        int     `mapstructure:"vm_retry_max_backoff"`        // max backoff in ms.
-	BackoffMultiplier float64 `mapstructure:"vm_retry_backoff_multiplier"` // backoff multiplier.
+	// Retry policy
+	MaxAttempts    uint `mapstructure:"vm_retry_max_attempts"`   // maximum attempts for retry (0 - infinity)
+	ReqTimeoutInMs uint `mapstructure:"vm_retry_req_timeout_ms"` // request timeout per attempt (0 - infinity) [ms]
 }
 
 // Default VM configuration.
 func DefaultVMConfig() *VMConfig {
 	return &VMConfig{
-		Address:           DefaultVMAddress,
-		DataListen:        DefaultDataListen,
-		MaxAttempts:       DefaultMaxAttempts,
-		InitialBackoff:    DefaultInitialBackoff,
-		MaxBackoff:        DefaultMaxBackoff,
-		BackoffMultiplier: DefaultBackoffMultiplier,
+		Address:        DefaultVMAddress,
+		DataListen:     DefaultDataListen,
+		MaxAttempts:    DefaultMaxAttempts,
+		ReqTimeoutInMs: DefaultReqTimeout,
 	}
 }
 

diff --git a/cmd/config/vm_config_template.go b/cmd/config/vm_config_template.go
@@ -29,18 +29,10 @@ vm_data_listen = "{{ .DataListen }}"
 # VM retry settings.
 
 ## Retry max attempts.
-## Default is 0 - infinity attempts, -1 - to disable.
+## Default is 0 - infinity attempts.
 vm_retry_max_attempts = {{ .MaxAttempts }}
 
-## Initial backoff in ms.
-## Default is 100ms.
-vm_retry_initial_backoff = {{ .InitialBackoff }}
-
-## Max backoff in ms.
-## Default is 150ms.
-vm_retry_max_backoff = {{ .MaxBackoff }}
-
-## Backoff multiplier.
-## Default is 0.1 
-vm_retry_backoff_multiplier = {{ .BackoffMultiplier }}
+## Request timeout per attempt in ms.
+## Default is 0 - infinite (no timeout).
+vm_retry_req_timeout_ms = {{ .ReqTimeoutInMs }}
 `