Test purger, fix some nonces

- Test all the purger functions
- Fix nonces set by til (previously til started with nonce 1 for pool l2txs,
  but the correct implementation is to start with nonce 0)
- Rename L2DB.CheckNonces to L2DB.invalidateOldNoncesQuery
- Rename L2DB.checkNoncesQuery to L2DB.InvalidateOldNonces

Related #392 (Fix checkNoncesQuery)
Resolve #396

coordinator/coordinator.go

@@ -245,7 +245,7 @@ func (c *Coordinator) syncStats(ctx context.Context, stats *synchronizer.Stats)
 	if c.pipeline == nil {
 		// Mark invalid in Pool due to forged L2Txs
 		// for _, batch := range batches {
-		// 	if err := poolMarkInvalidOldNoncesFromL2Txs(c.l2DB,
+		// 	if err := c.l2DB.InvalidateOldNonces(
 		// 		idxsNonceFromL2Txs(batch.L2Txs), batch.Batch.BatchNum); err != nil {
 		// 		return err
 		// 	}
@@ -850,8 +850,7 @@ func (p *Pipeline) forgeBatch(ctx context.Context, batchNum common.BatchNum, sel
 	// the poolL2Txs selected.  Will mark as invalid the txs that have a
 	// (fromIdx, nonce) which already appears in the selected txs (includes
 	// all the nonces smaller than the current one)
-	err = poolMarkInvalidOldNoncesFromL2Txs(p.l2DB, idxsNonceFromPoolL2Txs(poolL2Txs),
-		batchInfo.BatchNum)
+	err = p.l2DB.InvalidateOldNonces(idxsNonceFromPoolL2Txs(poolL2Txs), batchInfo.BatchNum)
 	if err != nil {
 		return nil, tracerr.Wrap(err)
 	}

coordinator/coordinator_test.go

@@ -92,13 +92,11 @@ var nLevels uint32 = 32   //nolint:deadcode,unused
 var maxFeeTxs uint32 = 64 //nolint:deadcode,varcheck
 
 func newTestModules(t *testing.T) modules {
-	nLevels := 32
-
 	var err error
 	syncDBPath, err = ioutil.TempDir("", "tmpSyncDB")
 	require.NoError(t, err)
 	deleteme = append(deleteme, syncDBPath)
-	syncStateDB, err := statedb.NewStateDB(syncDBPath, statedb.TypeSynchronizer, nLevels)
+	syncStateDB, err := statedb.NewStateDB(syncDBPath, statedb.TypeSynchronizer, 48)
 	assert.NoError(t, err)
 
 	pass := os.Getenv("POSTGRES_PASS")

coordinator/purger.go

@@ -35,10 +35,10 @@ type Purger struct {
 // CanPurge returns true if it's a good time to purge according to the
 // configuration
 func (p *Purger) CanPurge(blockNum, batchNum int64) bool {
-	if blockNum > p.lastPurgeBlock+p.cfg.PurgeBlockDelay {
+	if blockNum >= p.lastPurgeBlock+p.cfg.PurgeBlockDelay {
 		return true
 	}
-	if batchNum > p.lastPurgeBatch+p.cfg.PurgeBatchDelay {
+	if batchNum >= p.lastPurgeBatch+p.cfg.PurgeBatchDelay {
 		return true
 	}
 	return false
@@ -47,10 +47,10 @@ func (p *Purger) CanPurge(blockNum, batchNum int64) bool {
 // CanInvalidate returns true if it's a good time to invalidate according to
 // the configuration
 func (p *Purger) CanInvalidate(blockNum, batchNum int64) bool {
-	if blockNum > p.lastInvalidateBlock+p.cfg.InvalidateBlockDelay {
+	if blockNum >= p.lastInvalidateBlock+p.cfg.InvalidateBlockDelay {
 		return true
 	}
-	if batchNum > p.lastInvalidateBatch+p.cfg.InvalidateBatchDelay {
+	if batchNum >= p.lastInvalidateBatch+p.cfg.InvalidateBatchDelay {
 		return true
 	}
 	return false
@@ -114,15 +114,6 @@ func idxsNonceFromPoolL2Txs(txs []common.PoolL2Tx) []common.IdxNonce {
 	return idxsNonce
 }
 
-// poolMarkInvalidOldNoncesFromL2Txs marks as invalid the txs in the pool that
-// contain nonces equal or older to the highest nonce used in a forged l2Tx for
-// the
-// corresponding sender account
-func poolMarkInvalidOldNoncesFromL2Txs(l2DB *l2db.L2DB,
-	idxsNonce []common.IdxNonce, batchNum common.BatchNum) error {
-	return l2DB.CheckNonces(idxsNonce, batchNum)
-}
-
 // poolMarkInvalidOldNonces marks as invalid txs in the pool that contain
 // nonces equal or older to the nonce of the corresponding sender account
 func poolMarkInvalidOldNonces(l2DB *l2db.L2DB, stateDB *statedb.LocalStateDB,
@@ -147,9 +138,8 @@ func poolMarkInvalidOldNonces(l2DB *l2db.L2DB, stateDB *statedb.LocalStateDB,
 				return tracerr.Wrap(fmt.Errorf("unexpected stateDB error with idx %v: %w", idx, err))
 			}
 		}
-		fmt.Printf("DBG acc: %#v\n", acc)
 		idxsNonce[i].Idx = idx
 		idxsNonce[i].Nonce = acc.Nonce
 	}
-	return l2DB.CheckNonces(idxsNonce, batchNum)
+	return l2DB.InvalidateOldNonces(idxsNonce, batchNum)
 }

coordinator/purger_test.go

@@ -1,3 +1,287 @@
 package coordinator
 
-// TODO: Test purger functions
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/hermeznetwork/hermez-node/common"
+	dbUtils "github.com/hermeznetwork/hermez-node/db"
+	"github.com/hermeznetwork/hermez-node/db/l2db"
+	"github.com/hermeznetwork/hermez-node/db/statedb"
+	"github.com/hermeznetwork/hermez-node/test"
+	"github.com/hermeznetwork/hermez-node/test/til"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newL2DB(t *testing.T) *l2db.L2DB {
+	pass := os.Getenv("POSTGRES_PASS")
+	db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
+	require.NoError(t, err)
+	test.WipeDB(db)
+	return l2db.NewL2DB(db, 10, 100, 24*time.Hour)
+}
+
+func newStateDB(t *testing.T) *statedb.LocalStateDB {
+	syncDBPath, err := ioutil.TempDir("", "tmpSyncDB")
+	require.NoError(t, err)
+	deleteme = append(deleteme, syncDBPath)
+	syncStateDB, err := statedb.NewStateDB(syncDBPath, statedb.TypeSynchronizer, 48)
+	assert.NoError(t, err)
+	stateDBPath, err := ioutil.TempDir("", "tmpStateDB")
+	require.NoError(t, err)
+	deleteme = append(deleteme, stateDBPath)
+	stateDB, err := statedb.NewLocalStateDB(stateDBPath, syncStateDB, statedb.TypeTxSelector, 0)
+	require.NoError(t, err)
+	return stateDB
+}
+
+func TestCanPurgeCanInvalidate(t *testing.T) {
+	cfg := PurgerCfg{
+		PurgeBatchDelay:      2,
+		PurgeBlockDelay:      6,
+		InvalidateBatchDelay: 4,
+		InvalidateBlockDelay: 8,
+	}
+	p := Purger{
+		cfg: cfg,
+	}
+	startBlockNum := int64(1000)
+	startBatchNum := int64(10)
+	blockNum := startBlockNum
+	batchNum := startBatchNum
+
+	assert.True(t, p.CanPurge(blockNum, batchNum))
+	p.lastPurgeBlock = startBlockNum
+	p.lastPurgeBatch = startBatchNum
+	assert.False(t, p.CanPurge(blockNum, batchNum))
+
+	blockNum = startBlockNum + cfg.PurgeBlockDelay - 1
+	batchNum = startBatchNum + cfg.PurgeBatchDelay - 1
+	assert.False(t, p.CanPurge(blockNum, batchNum))
+	blockNum = startBlockNum + cfg.PurgeBlockDelay - 1
+	batchNum = startBatchNum + cfg.PurgeBatchDelay
+	assert.True(t, p.CanPurge(blockNum, batchNum))
+	blockNum = startBlockNum + cfg.PurgeBlockDelay
+	batchNum = startBatchNum + cfg.PurgeBatchDelay - 1
+	assert.True(t, p.CanPurge(blockNum, batchNum))
+
+	assert.True(t, p.CanInvalidate(blockNum, batchNum))
+	p.lastInvalidateBlock = startBlockNum
+	p.lastInvalidateBatch = startBatchNum
+	assert.False(t, p.CanInvalidate(blockNum, batchNum))
+
+	blockNum = startBlockNum + cfg.InvalidateBlockDelay - 1
+	batchNum = startBatchNum + cfg.InvalidateBatchDelay - 1
+	assert.False(t, p.CanInvalidate(blockNum, batchNum))
+	blockNum = startBlockNum + cfg.InvalidateBlockDelay - 1
+	batchNum = startBatchNum + cfg.InvalidateBatchDelay
+	assert.True(t, p.CanInvalidate(blockNum, batchNum))
+	blockNum = startBlockNum + cfg.InvalidateBlockDelay
+	batchNum = startBatchNum + cfg.InvalidateBatchDelay - 1
+	assert.True(t, p.CanInvalidate(blockNum, batchNum))
+}
+
+func TestPurgeMaybeInvalidateMaybe(t *testing.T) {
+	cfg := PurgerCfg{
+		PurgeBatchDelay:      2,
+		PurgeBlockDelay:      6,
+		InvalidateBatchDelay: 4,
+		InvalidateBlockDelay: 8,
+	}
+	p := Purger{
+		cfg: cfg,
+	}
+	l2DB := newL2DB(t)
+	stateDB := newStateDB(t)
+
+	startBlockNum := int64(1000)
+	startBatchNum := int64(10)
+
+	p.lastPurgeBlock = startBlockNum
+	p.lastPurgeBatch = startBatchNum
+
+	blockNum := startBlockNum + cfg.PurgeBlockDelay - 1
+	batchNum := startBatchNum + cfg.PurgeBatchDelay - 1
+	ok, err := p.PurgeMaybe(l2DB, blockNum, batchNum)
+	require.NoError(t, err)
+	assert.False(t, ok)
+	// At this point the purger will purge.  The second time it doesn't
+	// because it the first time it has updates the last time it did.
+	blockNum = startBlockNum + cfg.PurgeBlockDelay - 1
+	batchNum = startBatchNum + cfg.PurgeBatchDelay
+	ok, err = p.PurgeMaybe(l2DB, blockNum, batchNum)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	ok, err = p.PurgeMaybe(l2DB, blockNum, batchNum)
+	require.NoError(t, err)
+	assert.False(t, ok)
+
+	p.lastInvalidateBlock = startBlockNum
+	p.lastInvalidateBatch = startBatchNum
+
+	blockNum = startBlockNum + cfg.InvalidateBlockDelay - 1
+	batchNum = startBatchNum + cfg.InvalidateBatchDelay - 1
+	ok, err = p.InvalidateMaybe(l2DB, stateDB, blockNum, batchNum)
+	require.NoError(t, err)
+	assert.False(t, ok)
+	// At this point the purger will invaidate.  The second time it doesn't
+	// because it the first time it has updates the last time it did.
+	blockNum = startBlockNum + cfg.InvalidateBlockDelay - 1
+	batchNum = startBatchNum + cfg.InvalidateBatchDelay
+	ok, err = p.InvalidateMaybe(l2DB, stateDB, blockNum, batchNum)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	ok, err = p.InvalidateMaybe(l2DB, stateDB, blockNum, batchNum)
+	require.NoError(t, err)
+	assert.False(t, ok)
+}
+
+func TestIdxsNonce(t *testing.T) {
+	inputIdxsNonce := []common.IdxNonce{
+		{Idx: 256, Nonce: 1},
+		{Idx: 256, Nonce: 2},
+		{Idx: 257, Nonce: 3},
+		{Idx: 258, Nonce: 5},
+		{Idx: 258, Nonce: 2},
+	}
+	expectedIdxsNonce := map[common.Idx]common.Nonce{
+		common.Idx(256): common.Nonce(2),
+		common.Idx(257): common.Nonce(3),
+		common.Idx(258): common.Nonce(5),
+	}
+
+	l2txs := make([]common.L2Tx, len(inputIdxsNonce))
+	for i, idxNonce := range inputIdxsNonce {
+		l2txs[i].FromIdx = idxNonce.Idx
+		l2txs[i].Nonce = idxNonce.Nonce
+	}
+	idxsNonce := idxsNonceFromL2Txs(l2txs)
+	assert.Equal(t, len(expectedIdxsNonce), len(idxsNonce))
+	for _, idxNonce := range idxsNonce {
+		nonce := expectedIdxsNonce[idxNonce.Idx]
+		assert.Equal(t, nonce, idxNonce.Nonce)
+	}
+
+	pooll2txs := make([]common.PoolL2Tx, len(inputIdxsNonce))
+	for i, idxNonce := range inputIdxsNonce {
+		pooll2txs[i].FromIdx = idxNonce.Idx
+		pooll2txs[i].Nonce = idxNonce.Nonce
+	}
+	idxsNonce = idxsNonceFromPoolL2Txs(pooll2txs)
+	assert.Equal(t, len(expectedIdxsNonce), len(idxsNonce))
+	for _, idxNonce := range idxsNonce {
+		nonce := expectedIdxsNonce[idxNonce.Idx]
+		assert.Equal(t, nonce, idxNonce.Nonce)
+	}
+}
+
+func TestPoolMarkInvalidOldNonces(t *testing.T) {
+	l2DB := newL2DB(t)
+	stateDB := newStateDB(t)
+
+	set0 := `
+		Type: Blockchain
+
+		CreateAccountDeposit(0) A: 1000 // Idx=256
+		CreateAccountDeposit(0) B: 1000 // Idx=257
+		CreateAccountDeposit(0) C: 1000 // Idx=258
+		CreateAccountDeposit(0) D: 1000 // Idx=259
+
+		> batchL1
+		> batchL1
+		> block
+	`
+	tc := til.NewContext(common.RollupConstMaxL1UserTx)
+	blocks, err := tc.GenerateBlocks(set0)
+	require.NoError(t, err)
+	tilCfgExtra := til.ConfigExtra{
+		CoordUser: "A",
+	}
+	// Call FillBlocksExtra to fill `Batch.CreatedAccounts`
+	err = tc.FillBlocksExtra(blocks, &tilCfgExtra)
+	require.NoError(t, err)
+	require.Equal(t, 4, len(blocks[0].Rollup.Batches[1].CreatedAccounts)) // sanity check
+
+	for _, acc := range blocks[0].Rollup.Batches[1].CreatedAccounts {
+		_, err := stateDB.CreateAccount(acc.Idx, &acc) //nolint:gosec
+		require.NoError(t, err)
+	}
+
+	setPool0 := `
+		Type: PoolL2
+		PoolTransfer(0) A-B: 10 (1)
+		PoolTransfer(0) A-C: 10 (1)
+		PoolTransfer(0) A-D: 10 (1)
+		PoolTransfer(0) B-A: 10 (1)
+		PoolTransfer(0) B-C: 10 (1)
+		PoolTransfer(0) C-A: 10 (1)
+	`
+	// We expect the following nonces
+	nonces0 := map[string]int64{"A": 3, "B": 2, "C": 1, "D": 0}
+	l2txs0, err := tc.GeneratePoolL2Txs(setPool0)
+	assert.Nil(t, err)
+	assert.Equal(t, 6, len(l2txs0))
+	for _, tx := range l2txs0 {
+		require.NoError(t, l2DB.AddTxTest(&tx)) //nolint:gosec
+	}
+
+	// Update the accounts in the StateDB, making the txs in the setPool0
+	// invalid
+	for name, user := range tc.Users {
+		for _, _acc := range user.Accounts {
+			require.Equal(t, common.Nonce(nonces0[name]), _acc.Nonce) // sanity check
+			acc, err := stateDB.GetAccount(_acc.Idx)
+			require.NoError(t, err)
+			require.Equal(t, common.Nonce(0), acc.Nonce) // sanity check
+			acc.Nonce = _acc.Nonce
+			_, err = stateDB.UpdateAccount(acc.Idx, acc)
+			require.NoError(t, err)
+		}
+	}
+
+	setPool1 := `
+		Type: PoolL2
+		PoolTransfer(0) A-B: 10 (1)
+		PoolTransfer(0) A-C: 10 (1)
+		PoolTransfer(0) A-D: 10 (1)
+		PoolTransfer(0) B-A: 10 (1)
+		PoolTransfer(0) B-C: 10 (1)
+		PoolTransfer(0) C-A: 10 (1)
+	`
+	// We expect the following nonces
+	nonces1 := map[string]int64{"A": 6, "B": 4, "C": 2, "D": 0}
+	l2txs1, err := tc.GeneratePoolL2Txs(setPool1)
+	require.NoError(t, err)
+	assert.Equal(t, 6, len(l2txs1))
+	for _, tx := range l2txs1 {
+		require.NoError(t, l2DB.AddTxTest(&tx)) //nolint:gosec
+	}
+
+	for name, user := range tc.Users {
+		for _, _acc := range user.Accounts {
+			require.Equal(t, common.Nonce(nonces1[name]), _acc.Nonce) // sanity check
+			acc, err := stateDB.GetAccount(_acc.Idx)
+			require.NoError(t, err)
+			require.Equal(t, common.Nonce(nonces0[name]), acc.Nonce) // sanity check
+		}
+	}
+
+	// Now we should have 12 txs in the pool, all marked as pending.  Since
+	// we updated the stateDB with the nonces after setPool0, the first 6
+	// txs will be marked as invalid
+
+	pendingTxs, err := l2DB.GetPendingTxs()
+	require.NoError(t, err)
+	assert.Equal(t, 12, len(pendingTxs))
+
+	batchNum := common.BatchNum(1)
+	err = poolMarkInvalidOldNonces(l2DB, stateDB, batchNum)
+	require.NoError(t, err)
+
+	pendingTxs, err = l2DB.GetPendingTxs()
+	require.NoError(t, err)
+	assert.Equal(t, 6, len(pendingTxs))
+}