[FAB-16921] Upgrade older ledger data w/ couchdb

This CR adds a backward compability test using couchdb as
statedb. It upgrades v13 ledger data with state couchdb
- update idStore, drop blockIndex, history, and state databases.
Then it starts the upgraded ledger and verifies ledger data.
- rename v11_test.go to v1x_test.go to cover tests using
v1.1 and v1.3 ledger data

Change-Id: I479d978f89a437d4a45fadd89775eb2be38f5e8c
Signed-off-by: Wenjian Qiao <wenjianq@gmail.com>

.gitattributes

@@ -14,4 +14,5 @@
 *.txt text eol=lf
 *.yaml text eol=lf
 *.yml text eol=lf
+*.zip binary
 LICENSE text eol=lf

common/ledger/testutil/test_util.go

@@ -8,11 +8,13 @@ package testutil
 
 import (
 	"archive/tar"
+	"archive/zip"
 	"bytes"
 	"crypto/rand"
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -108,3 +110,50 @@ func copyFile(srcpath, destpath string) error {
 	}
 	return nil
 }
+
+// Unzip will decompress the src zip file to the dest directory.
+// If createTopLevelDirInZip is true, it creates the top level dir when unzipped.
+// Otherwise, it trims off the top level dir when unzipped. For example, ledersData/historydb/abc will become historydb/abc.
+func Unzip(src string, dest string, createTopLevelDirInZip bool) error {
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	// iterate all the dirs and files in the zip file
+	for _, file := range r.File {
+		filePath := file.Name
+		if !createTopLevelDirInZip {
+			// trim off the top level dir - for example, trim ledgersData/historydb/abc to historydb/abc
+			index := strings.Index(filePath, string(filepath.Separator))
+			filePath = filePath[index+1:]
+		}
+
+		fullPath := filepath.Join(dest, filePath)
+		if file.FileInfo().IsDir() {
+			os.MkdirAll(fullPath, os.ModePerm)
+			continue
+		}
+		if err = os.MkdirAll(filepath.Dir(fullPath), os.ModePerm); err != nil {
+			return err
+		}
+		outFile, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+		if err != nil {
+			return err
+		}
+		rc, err := file.Open()
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(outFile, rc)
+
+		outFile.Close()
+		rc.Close()
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

core/ledger/kvledger/kv_ledger_provider_test.go

@@ -143,7 +143,7 @@ func TestNewProviderIdStoreFormatError(t *testing.T) {
 	conf, cleanup := testConfig(t)
 	defer cleanup()
 
-	testutil.CopyDir("tests/testdata/v11/sample_ledgers/ledgersData", conf.RootFSPath, true)
+	require.NoError(t, testutil.Unzip("tests/testdata/v11/sample_ledgers/ledgersData.zip", conf.RootFSPath, false))
 
 	// NewProvider fails because ledgerProvider (idStore) has old format
 	_, err := NewProvider(

core/ledger/kvledger/tests/env.go

@@ -70,7 +70,10 @@ func (e *env) cleanup() {
 	if e.ledgerMgr != nil {
 		e.ledgerMgr.Close()
 	}
-	e.assert.NoError(os.RemoveAll(e.initializer.Config.RootFSPath))
+	// Ignore RemoveAll error because when a test mounts a dir to a couchdb container,
+	// the mounted dir cannot be deleted in CI builds. This has no impact to CI because it gets a new VM for each build.
+	// When running the test locally (macOS and linux VM), the mounted dirs are deleted without any error.
+	os.RemoveAll(e.initializer.Config.RootFSPath)
 }
 
 func (e *env) closeAllLedgersAndDrop(flags rebuildable) {
@@ -221,7 +224,7 @@ func populateMissingsWithTestDefaults(t *testing.T, initializer *ledgermgmt.Init
 	}
 
 	if initializer.Config == nil {
-		rootPath, err := ioutil.TempDir("", "ledgersData")
+		rootPath, err := ioutil.TempDir("/tmp", "ledgersData")
 		if err != nil {
 			t.Fatalf("Failed to create root directory: %s", err)
 		}

core/ledger/kvledger/tests/testdata/v13_statecouchdb/couchdb_etc/local.d/cluster.ini

@@ -0,0 +1,5 @@
+# The local.d directory will be mounted to the couchdb container so that
+# this file will overwrite the default values for number of shards and nodes
+[cluster]
+q=1
+n=1

core/ledger/kvledger/tests/util.go

@@ -7,6 +7,9 @@ SPDX-License-Identifier: Apache-2.0
 package tests
 
 import (
+	"fmt"
+	"testing"
+
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric-protos-go/common"
 	"github.com/hyperledger/fabric-protos-go/ledger/rwset"
@@ -15,9 +18,13 @@ import (
 	configtxtest "github.com/hyperledger/fabric/common/configtx/test"
 	"github.com/hyperledger/fabric/common/crypto"
 	"github.com/hyperledger/fabric/common/flogging"
+	"github.com/hyperledger/fabric/common/metrics/disabled"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/tests/fakes"
 	lutils "github.com/hyperledger/fabric/core/ledger/util"
+	"github.com/hyperledger/fabric/core/ledger/util/couchdb"
+	"github.com/hyperledger/fabric/integration/runner"
 	"github.com/hyperledger/fabric/protoutil"
+	"github.com/stretchr/testify/require"
 )
 
 var logger = flogging.MustGetLogger("test2")
@@ -75,6 +82,9 @@ func convertToMemberOrgsPolicy(members []string) *common.CollectionPolicyConfig
 }
 
 func convertFromMemberOrgsPolicy(policy *common.CollectionPolicyConfig) []string {
+	if policy.GetSignaturePolicy() == nil {
+		return nil
+	}
 	ids := policy.GetSignaturePolicy().Identities
 	var members []string
 	for _, id := range ids {
@@ -211,3 +221,34 @@ func setBlockFlagsToValid(block *common.Block) {
 	block.Metadata.Metadata[common.BlockMetadataIndex_TRANSACTIONS_FILTER] =
 		lutils.NewTxValidationFlagsSetValue(len(block.Data.Data), protopeer.TxValidationCode_VALID)
 }
+
+func couchDBSetup(t *testing.T, couchdbMountDir string, localdHostDir string) (addr string, cleanup func()) {
+	couchDB := &runner.CouchDB{
+		Name: "ledger13_upgrade_test",
+		Binds: []string{
+			fmt.Sprintf("%s:%s", couchdbMountDir, "/opt/couchdb/data"),
+			fmt.Sprintf("%s:%s", localdHostDir, "/opt/couchdb/etc/local.d"),
+		},
+	}
+	err := couchDB.Start()
+	require.NoError(t, err)
+	return couchDB.Address(), func() {
+		couchDB.Stop()
+	}
+}
+
+func dropCouchDBs(t *testing.T, couchdbConfig *couchdb.Config) {
+	couchInstance, err := couchdb.CreateCouchInstance(couchdbConfig, &disabled.Provider{})
+	require.NoError(t, err)
+	dbNames, err := couchInstance.RetrieveApplicationDBNames()
+	require.NoError(t, err)
+	for _, dbName := range dbNames {
+		db := &couchdb.CouchDatabase{
+			CouchInstance: couchInstance,
+			DBName:        dbName,
+		}
+		response, err := db.DropDatabase()
+		require.NoError(t, err)
+		require.True(t, response.Ok)
+	}
+}