Merge pull request #705 from klaytn/release/v1.5.3

[Master] release/v1.5.3 QA Sign-off

.circleci/config.yml

@@ -27,8 +27,19 @@ executors:
     working_directory: /go/src/github.com/klaytn/klaytn
     docker:
       - image: klaytn/build_base:1.1-go1.14.6-solc0.4.24
-      - image: localstack/localstack:0.10.9
+      - image: localstack/localstack:0.11.5
       - image: circleci/mysql:5.7
+      - image: circleci/redis:6.0.8-alpine
+      - name: zookeeper
+        image: wurstmeister/zookeeper
+      - name: kafka
+        image: wurstmeister/kafka
+        environment:
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://kafka:9094
+          KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://kafka:9094
+          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+          KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
   rpm-executor:
     working_directory: /go/src/github.com/klaytn/klaytn
     docker:
@@ -145,7 +156,7 @@ commands:
   tag-verify:
     steps:
       - run:
-          name: "Verify tag and file verison match"
+          name: "Verify tag and file version match"
           command: |
             echo "tag version is " $CIRCLE_TAG
             KLAYTN_VERSION=$(go run build/rpm/main.go version)
@@ -159,7 +170,7 @@ commands:
   tagger-verify:
     steps:
       - run:
-          name: "Verify tag and file verison match"
+          name: "Verify tag and file version match"
           command: |
               TAGGER=$(git for-each-ref --format='%(tagger)' refs/tags/$CIRCLE_TAG | sed 's/ .*//')
               if [ $TAGGER == 'circleci-klaytn' ]; then
@@ -219,6 +230,56 @@ commands:
               else
                 echo "Need to delete branch manually"
               fi
+  wait-other-containers-ready:
+    steps:
+      - run:
+          name: "Waiting for MySQL to be ready"
+          command: |
+            for i in `seq 1 10`;
+            do
+              nc -z 127.0.0.1 3306 && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for MySQL && exit 1
+      - run:
+          name: "Waiting for Redis to be ready"
+          command: |
+            for i in `seq 1 10`;
+            do
+              nc -z 127.0.0.1 6379 && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for Redis && exit 1
+      - run:
+          name: "Waiting for zookeeper to be ready"
+          command: |
+            for i in `seq 1 10`;
+            do
+              nc -z zookeeper 2181 && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for zookeeper && exit 1
+      - run:
+          name: "Waiting for Kafka to be ready"
+          command: |
+            for i in `seq 1 10`;
+            do
+              nc -z kafka 9092 && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for Kafka && exit 1
+      - run:
+          name: "Wait until localstack container running"
+          command: |
+            timeout -t 60 sh -c \
+            'until nc -z localhost 4566; do
+              echo "Waiting for Localstack ..."
+              sleep 1
+            done'
   notify-success:
     steps:
       - run:
@@ -263,17 +324,7 @@ jobs:
     steps:
       - install-ssh-alpine
       - checkout
-      - run:
-        # Our primary container isn't MYSQL so run a sleep command until it's ready.
-            name: "Waiting for MySQL to be ready"
-            command: |
-              for i in `seq 1 10`;
-              do
-                nc -z 127.0.0.1 3306 && echo Success && exit 0
-                echo -n .
-                sleep 1
-              done
-              echo Failed waiting for MySQL && exit 1
+      - wait-other-containers-ready
       - run:
           name: "Run test datasync"
           command: make test-datasync
@@ -303,17 +354,11 @@ jobs:
     steps:
       - install-ssh-alpine
       - checkout
-      - run:
-          name: "Wait until localstack container running"
-          command: |
-            timeout -t 60 sh -c \
-            'until curl --silent --fail "http://localhost:4572"; do
-              echo "Waiting for Localstack S3 ..."
-              sleep 1
-            done'
+      - wait-other-containers-ready
       - run:
           name: "Run test others"
-          command: make test-others
+          command: |
+            make test-others
 
   pass-tests:
     executor: default
@@ -329,11 +374,12 @@ jobs:
       - tagger-verify
 
   coverage:
-    executor: test-executor
+    executor: test-others-executor
     resource_class: xlarge
     steps:
       - install-ssh-alpine
       - checkout
+      - wait-other-containers-ready
       - run:
           shell: /bin/bash
           name: "Run coverage tests"
@@ -586,3 +632,4 @@ workflows:
               only: dev
     jobs:
       - linters
+

accounts/keystore/key.go

@@ -236,26 +236,34 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (Key, accounts.Accoun
 	return key, a, err
 }
 
-func writeKeyFile(file string, content []byte) error {
+func writeTemporaryKeyFile(file string, content []byte) (string, error) {
 	// Create the keystore directory with appropriate permissions
 	// in case it is not present yet.
 	const dirPerm = 0700
 	if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil {
-		return err
+		return "", err
 	}
 	// Atomic write: create a temporary hidden file first
 	// then move it into place. TempFile assigns mode 0600.
 	f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
 	if err != nil {
-		return err
+		return "", err
 	}
 	if _, err := f.Write(content); err != nil {
 		f.Close()
 		os.Remove(f.Name())
-		return err
+		return "", err
 	}
 	f.Close()
-	return os.Rename(f.Name(), file)
+	return f.Name(), err
+}
+
+func writeKeyFile(file string, content []byte) error {
+	name, err := writeTemporaryKeyFile(file, content)
+	if err != nil {
+		return err
+	}
+	return os.Rename(name, file)
 }
 
 // keyFileName implements the naming convention for keyfiles:

accounts/keystore/keystore.go

@@ -78,7 +78,7 @@ type unlocked struct {
 // NewKeyStore creates a keystore for the given directory.
 func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
 	keydir, _ = filepath.Abs(keydir)
-	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
+	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP, false}}
 	ks.init(keydir)
 	return ks
 }

accounts/keystore/keystore_passphrase.go

@@ -37,6 +37,7 @@ import (
 	"golang.org/x/crypto/scrypt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 )
 
@@ -67,6 +68,10 @@ type keyStorePassphrase struct {
 	keysDirPath string
 	scryptN     int
 	scryptP     int
+	// skipKeyFileVerification disables the security-feature which does
+	// reads and decrypts any newly created keyfiles. This should be 'false' in all
+	// cases except tests -- setting this to 'true' is not recommended.
+	skipKeyFileVerification bool
 }
 
 func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (Key, error) {
@@ -88,7 +93,7 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 
 // StoreKey generates a key, encrypts with 'auth' and stores in the given directory
 func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
-	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, rand.Reader, auth)
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP, false}, rand.Reader, auth)
 	return a.Address, err
 }
 
@@ -97,7 +102,25 @@ func (ks keyStorePassphrase) StoreKey(filename string, key Key, auth string) err
 	if err != nil {
 		return err
 	}
-	return writeKeyFile(filename, keyjson)
+	// Write into temporary file
+	tmpName, err := writeTemporaryKeyFile(filename, keyjson)
+	if err != nil {
+		return err
+	}
+	if ks.skipKeyFileVerification == false { //do not skip file verification
+		// Verify that we can decrypt the file with the given password.
+		_, err = ks.GetKey(key.GetAddress(), tmpName, auth)
+		if err != nil {
+			msg := "An error was encountered when saving and verifying the keystore file. \n" +
+				"This indicates that the keystore is corrupted. \n" +
+				"The corrupted file is stored at \n%v\n" +
+				"Please file a ticket at:\n\n" +
+				"https://github.com/klaytn/klaytn/issues" +
+				"The error was : %w"
+			return fmt.Errorf(msg, tmpName, err)
+		}
+	}
+	return os.Rename(tmpName, filename)
 }
 
 func (ks keyStorePassphrase) JoinPath(filename string) string {

accounts/keystore/keystore_plain_test.go

@@ -41,7 +41,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 		t.Fatal(err)
 	}
 	if encrypted {
-		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP}
+		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
 	} else {
 		ks = &keyStorePlain{d}
 	}
@@ -174,7 +174,7 @@ func TestV1_1(t *testing.T) {
 
 func TestV1_2(t *testing.T) {
 	t.Parallel()
-	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP}
+	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP, true}
 	addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
 	file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e"
 	k, err := ks.GetKey(addr, file, "g")

api/api_public_blockchain.go

@@ -23,6 +23,9 @@ package api
 import (
 	"context"
 	"fmt"
+	"math/big"
+	"time"
+
 	"github.com/klaytn/klaytn/blockchain"
 	"github.com/klaytn/klaytn/blockchain/types"
 	"github.com/klaytn/klaytn/blockchain/types/account"
@@ -35,8 +38,6 @@ import (
 	"github.com/klaytn/klaytn/networks/rpc"
 	"github.com/klaytn/klaytn/params"
 	"github.com/klaytn/klaytn/ser/rlp"
-	"math/big"
-	"time"
 )
 
 const defaultGasPrice = 25 * params.Ston
@@ -208,12 +209,6 @@ func (s *PublicBlockChainAPI) GetAccountKey(ctx context.Context, address common.
 	return serAccKey, state.Error()
 }
 
-// WriteThroughCaching returns if write through caching is enabled or not.
-// If enabled, when data write happens, cache write happens at the same time.
-func (s *PublicBlockChainAPI) WriteThroughCaching() bool {
-	return common.WriteThroughCaching
-}
-
 // IsParallelDBWrite returns if parallel write is enabled or not.
 // If enabled, data written in WriteBlockWithState is being written in parallel manner.
 func (s *PublicBlockChainAPI) IsParallelDBWrite() bool {