Merge pull request #4032 from ElrondNetwork/merge-dev-into-liqiud-sta…

…king-28-04-22

Merge dev into liqiud staking 28 04 22

.github/workflows/code-coverage.yml

@@ -5,7 +5,8 @@ on:
     branches:
       - master
   pull_request:
-    branches: [ master, development ]
+    branches: [ master, development, feat/*, rc/* ]
+
   workflow_dispatch:
 
 jobs:

.github/workflows/golangci-lint.yml

@@ -4,7 +4,8 @@ on:
     branches:
       - master
   pull_request:
-    branches: [ master, development, feat/* ]
+    branches: [ master, development, feat/*, rc/* ]
+
 jobs:
   golangci:
     name: golangci linter

README.md

@@ -34,7 +34,6 @@ $ mkdir -p $GOPATH/src/github.com/ElrondNetwork
 $ cd $GOPATH/src/github.com/ElrondNetwork
 $ git clone https://github.com/ElrondNetwork/elrond-go
 $ cd elrond-go && git checkout master
-$ GO111MODULE=on go mod vendor
 $ cd cmd/node && go build
 ```
 The node depends on the Arwen Virtual Machine, which is automatically managed by the node.
@@ -170,6 +169,7 @@ docker run -d -v /absolute/path/to/config/:/data/ elrondnetwork/elrond-go-node:l
 - [x] Testing
   - [x] Automate tests with AWS
   - [x] Nodes Monitoring
+- [x] DEX integration
 
 ### In progress
 
@@ -181,7 +181,6 @@ docker run -d -v /absolute/path/to/config/:/data/ elrondnetwork/elrond-go-node:l
   - [ ] Merging 
   - [ ] Redundancy
 - [ ] Privacy
-- [ ] DEX integration
 - [ ] Interoperability
 - [ ] Optimizations
   - [ ] Smart Contract 

api/groups/transactionGroup.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ElrondNetwork/elrond-go/api/errors"
 	"github.com/ElrondNetwork/elrond-go/api/middleware"
 	"github.com/ElrondNetwork/elrond-go/api/shared"
+	"github.com/ElrondNetwork/elrond-go/common"
 	txSimData "github.com/ElrondNetwork/elrond-go/process/txsimulator/data"
 	"github.com/gin-gonic/gin"
 )
@@ -28,6 +29,7 @@ const (
 	costPath                         = "/cost"
 	sendMultiplePath                 = "/send-multiple"
 	getTransactionPath               = "/:txhash"
+	getTransactionsPool              = "/pool"
 
 	queryParamWithResults    = "withResults"
 	queryParamCheckSignature = "checkSignature"
@@ -42,6 +44,7 @@ type transactionFacadeHandler interface {
 	SendBulkTransactions([]*transaction.Transaction) (uint64, error)
 	SimulateTransactionExecution(tx *transaction.Transaction) (*txSimData.SimulationResults, error)
 	GetTransaction(hash string, withResults bool) (*transaction.ApiTransactionResult, error)
+	GetTransactionsPool() (*common.TransactionsPoolAPIResponse, error)
 	ComputeTransactionGasLimit(tx *transaction.Transaction) (*transaction.CostResponse, error)
 	EncodeAddressPubkey(pk []byte) (string, error)
 	GetThrottlerForEndpoint(endpoint string) (core.Throttler, bool)
@@ -93,6 +96,11 @@ func NewTransactionGroup(facade transactionFacadeHandler) (*transactionGroup, er
 			Method:  http.MethodPost,
 			Handler: tg.computeTransactionGasLimit,
 		},
+		{
+			Path:    getTransactionsPool,
+			Method:  http.MethodGet,
+			Handler: tg.getTransactionsPool,
+		},
 		{
 			Path:    sendMultiplePath,
 			Method:  http.MethodPost,
@@ -531,6 +539,31 @@ func (tg *transactionGroup) computeTransactionGasLimit(c *gin.Context) {
 	)
 }
 
+// getTransactionsPool returns the transactions hashes in the pool
+func (tg *transactionGroup) getTransactionsPool(c *gin.Context) {
+	txsHashes, err := tg.getFacade().GetTransactionsPool()
+	if err != nil {
+		c.JSON(
+			http.StatusInternalServerError,
+			shared.GenericAPIResponse{
+				Data:  nil,
+				Error: err.Error(),
+				Code:  shared.ReturnCodeInternalError,
+			},
+		)
+		return
+	}
+
+	c.JSON(
+		http.StatusOK,
+		shared.GenericAPIResponse{
+			Data:  gin.H{"txPool": txsHashes},
+			Error: "",
+			Code:  shared.ReturnCodeSuccess,
+		},
+	)
+}
+
 func getQueryParamWithResults(c *gin.Context) (bool, error) {
 	withResultsStr := c.Request.URL.Query().Get(queryParamWithResults)
 	if withResultsStr == "" {

api/groups/transactionGroup_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ElrondNetwork/elrond-go/api/groups"
 	"github.com/ElrondNetwork/elrond-go/api/mock"
 	"github.com/ElrondNetwork/elrond-go/api/shared"
+	"github.com/ElrondNetwork/elrond-go/common"
 	"github.com/ElrondNetwork/elrond-go/config"
 	txSimData "github.com/ElrondNetwork/elrond-go/process/txsimulator/data"
 	"github.com/stretchr/testify/assert"
@@ -87,6 +88,16 @@ type transactionCostResponse struct {
 	Code  string                      `json:"code"`
 }
 
+type txsPoolResponseData struct {
+	TxPool common.TransactionsPoolAPIResponse `json:"txPool"`
+}
+
+type txsPoolResponse struct {
+	Data  txsPoolResponseData `json:"data"`
+	Error string              `json:"error"`
+	Code  string              `json:"code"`
+}
+
 func TestGetTransaction_WithCorrectHashShouldReturnTransaction(t *testing.T) {
 	sender := "sender"
 	receiver := "receiver"
@@ -823,6 +834,63 @@ func TestSimulateTransaction(t *testing.T) {
 	assert.Equal(t, string(shared.ReturnCodeSuccess), simulateResponse.Code)
 }
 
+func TestGetTransactionsPoolShouldError(t *testing.T) {
+	t.Parallel()
+
+	expectedErr := errors.New("expected error")
+	facade := mock.FacadeStub{
+		GetTransactionsPoolCalled: func() (*common.TransactionsPoolAPIResponse, error) {
+			return nil, expectedErr
+		},
+	}
+
+	transactionGroup, err := groups.NewTransactionGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(transactionGroup, "transaction", getTransactionRoutesConfig())
+
+	req, _ := http.NewRequest("GET", "/transaction/pool", nil)
+
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	txsPoolResp := generalResponse{}
+	loadResponse(resp.Body, &txsPoolResp)
+
+	assert.Equal(t, http.StatusInternalServerError, resp.Code)
+	assert.True(t, strings.Contains(txsPoolResp.Error, expectedErr.Error()))
+}
+
+func TestGetTransactionsPoolShouldWork(t *testing.T) {
+	t.Parallel()
+
+	expectedTxPool := &common.TransactionsPoolAPIResponse{
+		RegularTransactions: []string{"tx", "tx2"},
+	}
+	facade := mock.FacadeStub{
+		GetTransactionsPoolCalled: func() (*common.TransactionsPoolAPIResponse, error) {
+			return expectedTxPool, nil
+		},
+	}
+
+	transactionGroup, err := groups.NewTransactionGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(transactionGroup, "transaction", getTransactionRoutesConfig())
+
+	req, _ := http.NewRequest("GET", "/transaction/pool", nil)
+
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	txsPoolResp := txsPoolResponse{}
+	loadResponse(resp.Body, &txsPoolResp)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+	assert.Empty(t, txsPoolResp.Error)
+	assert.Equal(t, *expectedTxPool, txsPoolResp.Data.TxPool)
+}
+
 func getTransactionRoutesConfig() config.ApiRoutesConfig {
 	return config.ApiRoutesConfig{
 		APIPackages: map[string]config.APIPackageConfig{
@@ -831,6 +899,7 @@ func getTransactionRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/send", Open: true},
 					{Name: "/send-multiple", Open: true},
 					{Name: "/cost", Open: true},
+					{Name: "/pool", Open: true},
 					{Name: "/:txhash", Open: true},
 					{Name: "/:txhash/status", Open: true},
 					{Name: "/simulate", Open: true},

api/mock/facadeStub.go

@@ -70,6 +70,7 @@ type FacadeStub struct {
 	VerifyProofCalled                       func(string, string, [][]byte) (bool, error)
 	GetTokenSupplyCalled                    func(token string) (*api.ESDTSupply, error)
 	GetGenesisNodesPubKeysCalled            func() (map[uint32][]string, map[uint32][]string, error)
+	GetTransactionsPoolCalled               func() (*common.TransactionsPoolAPIResponse, error)
 }
 
 // GetTokenSupply -
@@ -431,6 +432,15 @@ func (f *FacadeStub) GetGenesisNodesPubKeys() (map[uint32][]string, map[uint32][
 	return nil, nil, nil
 }
 
+// GetTransactionsPool -
+func (f *FacadeStub) GetTransactionsPool() (*common.TransactionsPoolAPIResponse, error) {
+	if f.GetTransactionsPoolCalled != nil {
+		return f.GetTransactionsPoolCalled()
+	}
+
+	return nil, nil
+}
+
 // Trigger -
 func (f *FacadeStub) Trigger(_ uint32, _ bool) error {
 	return nil

api/shared/interface.go

@@ -113,5 +113,6 @@ type FacadeHandler interface {
 	RestAPIServerDebugMode() bool
 	PprofEnabled() bool
 	GetGenesisNodesPubKeys() (map[uint32][]string, map[uint32][]string, error)
+	GetTransactionsPool() (*common.TransactionsPoolAPIResponse, error)
 	IsInterfaceNil() bool
 }

cmd/node/config/api.toml

@@ -163,6 +163,9 @@
         # /transaction/cost will receive a single transaction in JSON format and will return the estimated cost of it
         { Name = "/cost", Open = true },
 
+        # /transaction/pool will return the hashes of the transactions that are currently in the pool
+        { Name = "/pool", Open = true },
+
         # /transaction/:txhash will return the transaction in JSON format based on its hash
         { Name = "/:txhash", Open = true },
     ]

cmd/node/config/config.toml

@@ -55,6 +55,20 @@
     # Applicable only for observers
     ObserverCleanOldEpochsData = false
 
+    # If this flag is set to false, the node won't delete the AccountsTrie database for old epochs
+    # Applicable for both observers and validators
+    # WARNING! Setting this to false will increase each epoch's directory size with the trie snapshot,
+    # which might easily cause the node to run out of disk space.
+    AccountsTrieCleanOldEpochsData = true
+
+    # AccountsTrieSkipRemovalCustomPattern represents the custom pattern that determines when AccountsTrie database
+    # doesn't have to be cleaned
+    # Format: %x,%y - if an epoch is divisible by x or by y, then the AccountsTrie database won't be removed
+    # If empty, then all databases will be removed
+    # If invalid format, then an error will be returned
+    # Applicable only if AccountsTrieCleanOldEpochsData is set to true
+    AccountsTrieSkipRemovalCustomPattern = "%50"
+
     # NumEpochsToKeep - if the flag above is set to true, this will set the number of epochs to keep in the storage.
     # Epochs older that (current epoch - NumOfEpochsToKeep) will be removed
     NumEpochsToKeep = 4
@@ -559,10 +573,6 @@
         # clutter the network exactly in the same moment
         MaxDeviationTimeInMilliseconds = 25
 
-[Logger]
-    Path = "logs"
-    StackTraceDepth = 2
-
 [AddressPubkeyConverter]
     Length = 32
     Type = "bech32"
@@ -864,7 +874,8 @@
         MaxOpenFiles = 10
 
 [Logs]
-    LogFileLifeSpanInSec = 86400
+    LogFileLifeSpanInMB = 1024 # 1GB
+    LogFileLifeSpanInSec = 86400 # 1 day
 
 [TrieSync]
     NumConcurrentTrieSyncers  = 200

cmd/node/factory/interface.go

@@ -53,13 +53,13 @@ type P2PAntifloodHandler interface {
 
 // FileLoggingHandler will handle log file rotation
 type FileLoggingHandler interface {
-	ChangeFileLifeSpan(newDuration time.Duration) error
+	ChangeFileLifeSpan(newDuration time.Duration, newSizeInMB uint64) error
 	Close() error
 	IsInterfaceNil() bool
 }
 
-// TODO: find a better naming
 // StatusHandlersUtils provides some functionality for statusHandlers
+// TODO: find a better naming
 type StatusHandlersUtils interface {
 	StatusHandler() core.AppStatusHandler
 	Metrics() external.StatusMetricsHandler

cmd/node/main.go

@@ -97,7 +97,10 @@ func startNodeRunner(c *cli.Context, log logger.Logger, version string) error {
 	}
 
 	if !check.IfNil(fileLogging) {
-		err := fileLogging.ChangeFileLifeSpan(time.Second * time.Duration(cfgs.GeneralConfig.Logs.LogFileLifeSpanInSec))
+		timeLogLifeSpan := time.Second * time.Duration(cfgs.GeneralConfig.Logs.LogFileLifeSpanInSec)
+		sizeLogLifeSpanInMB := uint64(cfgs.GeneralConfig.Logs.LogFileLifeSpanInMB)
+
+		err := fileLogging.ChangeFileLifeSpan(timeLogLifeSpan, sizeLogLifeSpanInMB)
 		if err != nil {
 			return err
 		}
@@ -243,7 +246,12 @@ func attachFileLogger(log logger.Logger, flagsConfig *config.ContextFlagsConfig)
 	var fileLogging factory.FileLoggingHandler
 	var err error
 	if flagsConfig.SaveLogFile {
-		fileLogging, err = logging.NewFileLogging(flagsConfig.WorkingDir, defaultLogsPath, logFilePrefix)
+		args := logging.ArgsFileLogging{
+			WorkingDir:      flagsConfig.WorkingDir,
+			DefaultLogsPath: defaultLogsPath,
+			LogFilePrefix:   logFilePrefix,
+		}
+		fileLogging, err = logging.NewFileLogging(args)
 		if err != nil {
 			return nil, fmt.Errorf("%w creating a log file", err)
 		}

cmd/seednode/config/config.toml

@@ -4,8 +4,9 @@
 #     the reencoded version (in percents).
 #     0 disables the feature.
 [Marshalizer]
-   Type = "gogo protobuf"
-   SizeCheckDelta = 10
+    Type = "gogo protobuf"
+    SizeCheckDelta = 10
 
 [Logs]
-   LogFileLifeSpanInSec = 86400
+    LogFileLifeSpanInMB = 1024 # 1GB
+    LogFileLifeSpanInSec = 86400 # 1 day

cmd/seednode/main.go

@@ -78,7 +78,7 @@ VERSION:
 			" log level.",
 		Value: "*:" + logger.LogInfo.String(),
 	}
-	//logFile is used when the log output needs to be logged in a file
+	// logFile is used when the log output needs to be logged in a file
 	logSaveFile = cli.BoolFlag{
 		Name:  "log-save",
 		Usage: "Boolean option for enabling log saving. If set, it will automatically save all the logs into a file.",
@@ -151,12 +151,19 @@ func startNode(ctx *cli.Context) error {
 	var fileLogging factory.FileLoggingHandler
 	if withLogFile {
 		workingDir := getWorkingDir(log)
-		fileLogging, err = logging.NewFileLogging(workingDir, defaultLogsPath, logFilePrefix)
+		args := logging.ArgsFileLogging{
+			WorkingDir:      workingDir,
+			DefaultLogsPath: defaultLogsPath,
+			LogFilePrefix:   logFilePrefix,
+		}
+		fileLogging, err = logging.NewFileLogging(args)
 		if err != nil {
 			return fmt.Errorf("%w creating a log file", err)
 		}
 
-		err = fileLogging.ChangeFileLifeSpan(time.Second * time.Duration(generalConfig.Logs.LogFileLifeSpanInSec))
+		timeLogLifeSpan := time.Second * time.Duration(generalConfig.Logs.LogFileLifeSpanInSec)
+		sizeLogLifeSpanInMB := uint64(generalConfig.Logs.LogFileLifeSpanInMB)
+		err = fileLogging.ChangeFileLifeSpan(timeLogLifeSpan, sizeLogLifeSpanInMB)
 		if err != nil {
 			return err
 		}