From d50ab72252063fa56904d215f050aba2798155df Mon Sep 17 00:00:00 2001 From: SamYuan1990 Date: Fri, 7 Aug 2020 18:20:57 +0800 Subject: [PATCH] Enable mTLS Adding parameter for mock server add mtls for mock server according to https://github.com/grpc/grpc-go/issues/403 using mtls when create grpc connection Refactor using node as parameter Mock server test refactor with gexec Fixs #35 Signed-off-by: SamYuan1990 --- README.md | 63 ++++++++++++--- azure-pipelines.yml | 3 +- go.mod | 10 ++- go.sum | 24 ++++++ mock/fabric/main.go | 39 ++++++++- pkg/infra/client.go | 32 +++++--- pkg/infra/config.go | 33 +++++++- pkg/infra/config_test.go | 31 +++++-- pkg/infra/mock_test.go | 161 +++++++++++++++++++++++++++++++++++++ pkg/infra/mtlsprepare.sh | 15 ++++ pkg/infra/observer.go | 6 +- pkg/infra/proposer.go | 22 ++--- pkg/infra/proposer_test.go | 17 ++-- test/configmock.yaml | 4 +- test/configmockmtls.yaml | 23 ++++++ test/integration-test.sh | 16 +--- 16 files changed, 417 insertions(+), 82 deletions(-) create mode 100644 pkg/infra/mock_test.go create mode 100755 pkg/infra/mtlsprepare.sh create mode 100644 test/configmockmtls.yaml diff --git a/README.md b/README.md index afed2801..37f583da 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,10 @@ Clone this repo and run `go build ./cmd/stupid` at root dir. This is a go module ### Configure -Modify `config.yaml` according to your network. This is a sample: +Modify `config.yaml` according to your network. +
+This is a sample: + ```yaml endorsers: - addr: localhost:7051 @@ -62,12 +65,55 @@ sign_cert: ./organizations/peerOrganizations/org1.example.com/users/User1@org1.e num_of_conn: 10 client_per_conn: 10 ``` +
+ +For `endorsers`, `committer`, `orderer` we use a node structure defined as below: +``` +node: &node + addr: peer0.org1.example.com:7051 + tls_ca_cert: TLS server cert +``` +You may need to add peer name, i.e. `peer0.org1.example.com,peer0.org2.example.com` to your `/etc/hosts` + +
+Further samples related with TLS + +``` +node: &node + addr: peer0.org1.example.com:7051 + tls_ca_cert: TLS server cert or mTLS client cert + tls_ca_key: mTLS client key + tls_ca_root: mTLS ca root +``` + +For non TLS +``` +node: &node + addr: peer0.org1.example.com:7051 +``` + +For TLS +``` +node: &node + addr: peer0.org1.example.com:7051 + tls_ca_cert: peer0.org1.example.com-cert.pem +``` -`endorsers`: include the addr and tls ca cert of peers. Peer address is in IP:Port format. You may need to add peer name, i.e. `peer0.org1.example.com,peer0.org2.example.com` to your `/etc/hosts` +For MTLS +``` +node: &node + addr: peer0.org1.example.com:7051 + tls_ca_cert: User1@org1.example.com/tls/client.crt + tls_ca_key: User1@org1.example.com/tls/client.key + tls_ca_root: User1@org1.example.com/tls/ca.crt +``` +
+ +`endorsers`: the peer which will execute endorserment. `committer`: observe tx commitment from these peers. -`orderer`: include the addr and tls ca cert of orderer. Orderer address is in IP:Port format. It does not support sending traffic to multiple orderers, yet. You may need to add orderer name, i.e. `orderer.example.com` to your `/etc/hosts` +`orderer`: include the addr and tls ca cert of orderer. This tool sends traffic as a Fabric user, and requires following configs @@ -101,6 +147,11 @@ Execute `./stupid config.yaml 40000` to generate 40000 transactions to Fabric. *Set this to integer times of batchsize, so that last block is not cut due to timeout*. For example, if you have batch size of 500, set this to 500, 1000, 40000, 100000, etc. +### log + +We use logrus(https://github.com/sirupsen/logrus) for logging, pls set log level by envrionment as `export STUPID_LOGLEVEL=debug`. +Here are the values, by default is warn level. +`"panic", "fatal", "error", "warn", "warning", "info", "debug", "trace"` ## Development @@ -112,12 +163,6 @@ Stupid consists of several workers that run in goroutines, so that the pipeline ![stupid workflow](stupid.jpeg) -### log - -We use logrus(https://github.com/sirupsen/logrus) for logging, pls set log level by envrionment as `export STUPID_LOGLEVEL=debug`. -Here are the values, by default is warn level. -`"panic", "fatal", "error", "warn", "warning", "info", "debug", "trace"` - ## Tips - Put this generator closer to Fabric, on even on the same machine. This is to prevent network bandwidth from being the bottleneck. You can use tools like `iftop` to monitor network traffic. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cc638053..60ba3e4d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,6 +43,7 @@ jobs: command: 'test' arguments: '-v ./... -cover' workingDirectory: '$(System.DefaultWorkingDirectory)' + - job: displayName: integration-test strategy: @@ -56,8 +57,6 @@ jobs: FABRIC_14: FABRIC_VERSION: '14' INTERGATION_CASE: 'ANDLogic' - MOCK_FABRIC: - FABRIC_VERSION: 'mock' LATEST: FABRIC_VERSION: '' steps: diff --git a/go.mod b/go.mod index aaf46e3e..b6e1f437 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,11 @@ go 1.12 require ( github.com/Shopify/sarama v1.23.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.4.2 github.com/hashicorp/go-version v1.2.0 // indirect + github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c // indirect github.com/hyperledger/fabric v2.0.1+incompatible github.com/hyperledger/fabric-amcl v0.0.0-20200128223036-d1aa2665426a // indirect github.com/hyperledger/fabric-protos-go v0.0.0-20200424173316-dd554ba3746e @@ -18,11 +20,13 @@ require ( github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.6.0 github.com/spf13/viper v1.4.0 // indirect + github.com/square/certstrap v1.2.0 // indirect github.com/stretchr/objx v0.2.0 // indirect - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.5.1 // indirect github.com/sykesm/zap-logfmt v0.0.2 // indirect - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect - golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect + github.com/urfave/cli v1.22.4 // indirect + golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect + golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 // indirect google.golang.org/grpc v1.24.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index 3c995e69..88cbd3f3 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,10 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -86,6 +90,10 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c h1:kQWxfPIHVLbgLzphqk3QUflDy9QdksZR4ygR807bpy0= +github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= +github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c h1:aY2hhxLhjEAbfXOx2nRJxCXezC6CO2V/yN+OCr1srtk= +github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hyperledger/fabric v2.0.1+incompatible h1:7W+yG0gLKTC7NLcWPT3vfpnaseztPpH9wXGfAW7yvBs= @@ -163,7 +171,11 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -184,6 +196,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/square/certstrap v1.2.0 h1:ecgyABrbFLr8jSbOC6oTBmBek0t/HqtgrMUZCPuyfdw= +github.com/square/certstrap v1.2.0/go.mod h1:CUHqV+fxJW0Y5UQFnnbYwQ7bpKXO1AKbic9g73799yw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -196,6 +210,10 @@ github.com/sykesm/zap-logfmt v0.0.2 h1:czSzn+PIXCOAP/4NAIHTTziIKB8201PzoDkKTn+VR github.com/sykesm/zap-logfmt v0.0.2/go.mod h1:TerDJT124HaO8UTpZ2wJCipJRAKQ9XONM1mzUabIh6M= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v1.21.0 h1:wYSSj06510qPIzGSua9ZqsncMmWE3Zr55KBERygyrxE= +github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ= +github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -210,10 +228,13 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -241,6 +262,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -253,6 +275,8 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed h1:J22ig1FUekjjkmZUM7pTKixYm8DvrYsvrBZdunYeIuQ= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= diff --git a/mock/fabric/main.go b/mock/fabric/main.go index c38edfe5..942f6dae 100644 --- a/mock/fabric/main.go +++ b/mock/fabric/main.go @@ -2,14 +2,20 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "fmt" "io" + "io/ioutil" "net" + "os" + "strconv" "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric-protos-go/orderer" "github.com/hyperledger/fabric-protos-go/peer" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" ) type Peer struct { @@ -86,6 +92,15 @@ func main() { panic(err) } + mtls := false + + if len(os.Args) > 1 { + mtls, err = strconv.ParseBool(os.Args[1]) + if err != nil { + panic(err) + } + } + fmt.Println("Start listening on localhost...") blockC := make(chan struct{}, 1000) @@ -98,8 +113,30 @@ func main() { o := &Orderer{ TxC: blockC, } - grpcServer := grpc.NewServer() + + if mtls { + peerCert, err := tls.LoadX509KeyPair("./server.crt", + "./server.key") + if err != nil { + fmt.Println("load peer cert/key error:", err) + return + } + caCert, err := ioutil.ReadFile("./ExampleCA.crt") + if err != nil { + fmt.Println("read ca cert file error:", err) + return + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + ta := credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{peerCert}, + ClientCAs: caCertPool, + ClientAuth: tls.RequireAndVerifyClientCert, + }) + fmt.Println("Enable mtls") + grpcServer = grpc.NewServer(grpc.Creds(ta)) + } peer.RegisterEndorserServer(grpcServer, p) peer.RegisterDeliverServer(grpcServer, p) orderer.RegisterAtomicBroadcastServer(grpcServer, o) diff --git a/pkg/infra/client.go b/pkg/infra/client.go index ad6d99ff..3be0e91b 100644 --- a/pkg/infra/client.go +++ b/pkg/infra/client.go @@ -10,10 +10,10 @@ import ( "github.com/hyperledger/fabric/core/comm" ) -func CreateGRPCClient(cert []byte) (*comm.GRPCClient, error) { +func CreateGRPCClient(node Node) (*comm.GRPCClient, error) { var certs [][]byte - if cert != nil { - certs = append(certs, cert) + if node.TLSCACertByte != nil { + certs = append(certs, node.TLSCACertByte) } config := comm.ClientConfig{} config.Timeout = 5 * time.Second @@ -25,6 +25,14 @@ func CreateGRPCClient(cert []byte) (*comm.GRPCClient, error) { if len(certs) > 0 { config.SecOpts.UseTLS = true + if len(node.TLSCAKey) > 0 && len(node.TLSCARoot) > 0 { + config.SecOpts.RequireClientCert = true + config.SecOpts.Certificate = node.TLSCACertByte + config.SecOpts.Key = node.TLSCAKeyByte + if node.TLSCARootByte != nil { + config.SecOpts.ClientRootCAs = append(config.SecOpts.ClientRootCAs, node.TLSCARootByte) + } + } } grpcClient, err := comm.NewGRPCClient(config) @@ -35,13 +43,13 @@ func CreateGRPCClient(cert []byte) (*comm.GRPCClient, error) { return grpcClient, nil } -func CreateEndorserClient(addr string, tlscacert []byte) (peer.EndorserClient, error) { - gRPCClient, err := CreateGRPCClient(tlscacert) +func CreateEndorserClient(node Node) (peer.EndorserClient, error) { + gRPCClient, err := CreateGRPCClient(node) if err != nil { return nil, err } - conn, err := gRPCClient.NewConnection(addr, func(tlsConfig *tls.Config) { tlsConfig.InsecureSkipVerify = true }) + conn, err := gRPCClient.NewConnection(node.Addr, func(tlsConfig *tls.Config) { tlsConfig.InsecureSkipVerify = true }) if err != nil { return nil, err } @@ -49,13 +57,13 @@ func CreateEndorserClient(addr string, tlscacert []byte) (peer.EndorserClient, e return peer.NewEndorserClient(conn), nil } -func CreateBroadcastClient(addr string, tlscacert []byte) (orderer.AtomicBroadcast_BroadcastClient, error) { - gRPCClient, err := CreateGRPCClient(tlscacert) +func CreateBroadcastClient(node Node) (orderer.AtomicBroadcast_BroadcastClient, error) { + gRPCClient, err := CreateGRPCClient(node) if err != nil { return nil, err } - conn, err := gRPCClient.NewConnection(addr, func(tlsConfig *tls.Config) { tlsConfig.InsecureSkipVerify = true }) + conn, err := gRPCClient.NewConnection(node.Addr, func(tlsConfig *tls.Config) { tlsConfig.InsecureSkipVerify = true }) if err != nil { return nil, err } @@ -63,13 +71,13 @@ func CreateBroadcastClient(addr string, tlscacert []byte) (orderer.AtomicBroadca return orderer.NewAtomicBroadcastClient(conn).Broadcast(context.Background()) } -func CreateDeliverFilteredClient(addr string, tlscacert []byte) (peer.Deliver_DeliverFilteredClient, error) { - gRPCClient, err := CreateGRPCClient(tlscacert) +func CreateDeliverFilteredClient(node Node) (peer.Deliver_DeliverFilteredClient, error) { + gRPCClient, err := CreateGRPCClient(node) if err != nil { return nil, err } - conn, err := gRPCClient.NewConnection(addr, func(tlsConfig *tls.Config) { tlsConfig.InsecureSkipVerify = true }) + conn, err := gRPCClient.NewConnection(node.Addr, func(tlsConfig *tls.Config) { tlsConfig.InsecureSkipVerify = true }) if err != nil { return nil, err } diff --git a/pkg/infra/config.go b/pkg/infra/config.go index 0eeb0314..db0ab6bd 100644 --- a/pkg/infra/config.go +++ b/pkg/infra/config.go @@ -24,8 +24,13 @@ type Config struct { } type Node struct { - Addr string `yaml:"addr"` - TLSCACert string `yaml:"tls_ca_cert"` + Addr string `yaml:"addr"` + TLSCACert string `yaml:"tls_ca_cert"` + TLSCAKey string `yaml:"tls_ca_key"` + TLSCARoot string `yaml:"tls_ca_root"` + TLSCACertByte []byte + TLSCAKeyByte []byte + TLSCARootByte []byte } func LoadConfig(f string) Config { @@ -39,6 +44,12 @@ func LoadConfig(f string) Config { panic(err) } + for i, _ := range config.Endorsers { + config.Endorsers[i].loadNodeConfig() + } + config.Committer.loadNodeConfig() + config.Orderer.loadNodeConfig() + return config } @@ -93,3 +104,21 @@ func GetTLSCACerts(file string) ([]byte, error) { } return in, nil } + +func (n *Node) loadNodeConfig() { + TLSCACert, err := GetTLSCACerts(n.TLSCACert) + if err != nil { + panic(err) + } + certPEM, err := GetTLSCACerts(n.TLSCAKey) + if err != nil { + panic(err) + } + TLSCARoot, err := GetTLSCACerts(n.TLSCARoot) + if err != nil { + panic(err) + } + n.TLSCACertByte = TLSCACert + n.TLSCAKeyByte = certPEM + n.TLSCARootByte = TLSCARoot +} diff --git a/pkg/infra/config_test.go b/pkg/infra/config_test.go index e0b1896c..bf563014 100644 --- a/pkg/infra/config_test.go +++ b/pkg/infra/config_test.go @@ -9,20 +9,32 @@ import ( . "github.com/onsi/gomega" ) +const DUMMY_TLS = "./dummy.pem" +const PEM_BUF = "a" + var _ = Describe("Config", func() { + BeforeEach(func() { + f, err := os.OpenFile(DUMMY_TLS, os.O_CREATE|os.O_WRONLY, 0777) + Expect(err).ShouldNot(HaveOccurred()) + + _, err = f.Write([]byte(PEM_BUF)) + Expect(err).ShouldNot(HaveOccurred()) + + }) + Context("config", func() { It("successful load", func() { var configText = ` org1peer0: &org1peer0 addr: peer0.org1.example.com:7051 - tls_ca_cert: /path/to/org1peer0/tls/ca/cert + tls_ca_cert: ./dummy.pem org2peer0: &org2peer0 addr: peer0.org2.example.com:7051 - tls_ca_cert: /path/to/org2peer0/tls/ca/cert + tls_ca_cert: ./dummy.pem org0orderer0: &org0orderer0 addr: orderer.example.com:7050 - tls_ca_cert: /path/to/orderer/tls/ca/cert + tls_ca_cert: ./dummy.pem endorsers: - *org1peer0 @@ -52,11 +64,11 @@ client_per_conn: 40` Expect(c).To(Equal(infra.Config{ Endorsers: []infra.Node{ - {Addr: "peer0.org1.example.com:7051", TLSCACert: "/path/to/org1peer0/tls/ca/cert"}, - {Addr: "peer0.org2.example.com:7051", TLSCACert: "/path/to/org2peer0/tls/ca/cert"}, + {Addr: "peer0.org1.example.com:7051", TLSCACert: DUMMY_TLS, TLSCACertByte: []byte(PEM_BUF)}, + {Addr: "peer0.org2.example.com:7051", TLSCACert: DUMMY_TLS, TLSCACertByte: []byte(PEM_BUF)}, }, - Committer: infra.Node{Addr: "peer0.org2.example.com:7051", TLSCACert: "/path/to/org2peer0/tls/ca/cert"}, - Orderer: infra.Node{Addr: "orderer.example.com:7050", TLSCACert: "/path/to/orderer/tls/ca/cert"}, + Committer: infra.Node{Addr: "peer0.org2.example.com:7051", TLSCACert: DUMMY_TLS, TLSCACertByte: []byte(PEM_BUF)}, + Orderer: infra.Node{Addr: "orderer.example.com:7050", TLSCACert: DUMMY_TLS, TLSCACertByte: []byte(PEM_BUF)}, Channel: "mychannel", Chaincode: "mycc", Version: "", @@ -69,4 +81,9 @@ client_per_conn: 40` })) }) }) + + AfterEach(func() { + err := os.Remove(DUMMY_TLS) + Expect(err).ShouldNot(HaveOccurred()) + }) }) diff --git a/pkg/infra/mock_test.go b/pkg/infra/mock_test.go new file mode 100644 index 00000000..278ca72c --- /dev/null +++ b/pkg/infra/mock_test.go @@ -0,0 +1,161 @@ +package infra_test + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "os/exec" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gbytes" + + "github.com/onsi/gomega/gexec" +) + +// generate cert and key and populate them to files +func genCertKey(key, cert *os.File) error { + priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + if err != nil { + return err + } + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"STUPID"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return err + } + + err = pem.Encode(cert, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + if err != nil { + return err + } + + b, err := x509.MarshalECPrivateKey(priv) + if err != nil { + return nil + } + err = pem.Encode(key, &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}) + if err != nil { + return err + } + + return nil +} + +const PK = "./priv_sk" +const CERT = "./User1@org1.example.com-cert.pem" + +var _ = Describe("Mock test", func() { + + var certFile, keyFile *os.File + var err error + var compiledMock, compiledStupid string + var mock_session, stupid_session *gexec.Session + + BeforeSuite(func() { + + Prepare_command := exec.Command("./mtlsprepare.sh") + _, err = gexec.Start(Prepare_command, nil, nil) + Expect(err).ShouldNot(HaveOccurred()) + + certFile, err = os.OpenFile(CERT, os.O_CREATE|os.O_WRONLY, 0777) + Expect(err).ShouldNot(HaveOccurred()) + + keyFile, err = os.OpenFile(PK, os.O_CREATE|os.O_WRONLY, 0777) + Expect(err).ShouldNot(HaveOccurred()) + + err = genCertKey(keyFile, certFile) + Expect(err).ShouldNot(HaveOccurred()) + + compiledMock, err = gexec.Build("../../mock/fabric") + Expect(err).ShouldNot(HaveOccurred()) + + compiledStupid, err = gexec.Build("../../cmd/stupid") + Expect(err).ShouldNot(HaveOccurred()) + + fmt.Println(certFile.Name()) + + time.Sleep(time.Second * 10) + }) + + Context("Mock server test", func() { + + It("should running successfully when no tls", func() { + mock_command := exec.Command(compiledMock, "false") + mock_session, err = gexec.Start(mock_command, nil, nil) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(mock_session.Out).Should(Say("Start listening on localhost...")) + + stupid_command := exec.Command(compiledStupid, "../../test/configmock.yaml", "500") + stupid_session, err = gexec.Start(stupid_command, nil, nil) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(stupid_session.Out).Should(Say("Time.*Block.*Tx.*10.*")) + }) + + It("should running successfully when mutual tls", func() { + time.Sleep(time.Second * 10) + mock_command := exec.Command(compiledMock, "true") + mock_session, err = gexec.Start(mock_command, nil, nil) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(mock_session.Out).Should(Say("Start listening on localhost...")) + Eventually(mock_session.Out).Should(Say("Enable mtls")) + + stupid_command := exec.Command(compiledStupid, "../../test/configmockmtls.yaml", "500") + stupid_session, err = gexec.Start(stupid_command, nil, nil) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(stupid_session.Out).Should(Say("Time.*Block.*Tx.*10.*")) + //Say("Time.*Block.*Tx.*10.*") as regex matcher for string Time 0.06s Block 0 Tx 10 + }) + }) + + AfterEach(func() { + mock_session.Kill() + stupid_session.Kill() + }) + + AfterSuite(func() { + //cmd := exec.Command("ls", "/tmp") + //out, _ := cmd.Output() + + //fmt.Println(string(out)) + os.Remove(compiledStupid) + os.Remove(compiledMock) + os.Remove(certFile.Name()) + os.Remove(keyFile.Name()) + filelist := []string{ + "./client.crt", + "./client.csr", + "./client.key", + "./server.crt", + "./server.csr", + "./server.key", + "./ExampleCA.crl", + "./ExampleCA.crt", + "./ExampleCA.key", + } + for _, str := range filelist { + os.Remove(str) + } + }) + +}) diff --git a/pkg/infra/mtlsprepare.sh b/pkg/infra/mtlsprepare.sh new file mode 100755 index 00000000..c8a6a43f --- /dev/null +++ b/pkg/infra/mtlsprepare.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -ex + +#go get -u github.com/square/certstrap +#generate a root +docker run -v $PWD:/tmp squareup/certstrap --depot-path "/tmp" init --common-name "ExampleCA" --passphrase "" +sudo chmod 644 ExampleCA.* +#server key pair from root +docker run -v $PWD:/tmp squareup/certstrap --depot-path "/tmp" request-cert -cn server -ip 127.0.0.1 --passphrase "" +docker run -v $PWD:/tmp squareup/certstrap --depot-path "/tmp" sign server --CA ExampleCA +sudo chmod 644 server.* +#client key pair from root +docker run -v $PWD:/tmp squareup/certstrap --depot-path "/tmp" request-cert -cn client --passphrase "" +docker run -v $PWD:/tmp squareup/certstrap --depot-path "/tmp" sign client --CA ExampleCA +sudo chmod 644 client.* diff --git a/pkg/infra/observer.go b/pkg/infra/observer.go index 5909630c..e240ae64 100644 --- a/pkg/infra/observer.go +++ b/pkg/infra/observer.go @@ -15,11 +15,7 @@ type Observer struct { } func CreateObserver(channel string, node Node, crypto *Crypto, logger *log.Logger) *Observer { - TLSCACert, err := GetTLSCACerts(node.TLSCACert) - if err != nil { - panic(err) - } - deliverer, err := CreateDeliverFilteredClient(node.Addr, TLSCACert) + deliverer, err := CreateDeliverFilteredClient(node) if err != nil { panic(err) } diff --git a/pkg/infra/proposer.go b/pkg/infra/proposer.go index b55c92da..42660009 100644 --- a/pkg/infra/proposer.go +++ b/pkg/infra/proposer.go @@ -22,12 +22,8 @@ func CreateProposers(conn, client int, nodes []Node, logger *log.Logger) *Propos //one proposer per connection per peer for _, node := range nodes { row := make([]*Proposer, conn) - TLSCACert, err := GetTLSCACerts(node.TLSCACert) - if err != nil { - panic(err) - } for j := 0; j < conn; j++ { - row[j] = CreateProposer(node.Addr, TLSCACert, logger) + row[j] = CreateProposer(node, logger) } ps = append(ps, row) } @@ -53,13 +49,13 @@ type Proposer struct { logger *log.Logger } -func CreateProposer(addr string, TLSCACert []byte, logger *log.Logger) *Proposer { - endorser, err := CreateEndorserClient(addr, TLSCACert) +func CreateProposer(node Node, logger *log.Logger) *Proposer { + endorser, err := CreateEndorserClient(node) if err != nil { panic(err) } - return &Proposer{e: endorser, Addr: addr, logger: logger} + return &Proposer{e: endorser, Addr: node.Addr, logger: logger} } func (p *Proposer) Start(signed, processed chan *Elements, done <-chan struct{}, threshold int) { @@ -93,12 +89,8 @@ type Broadcasters []*Broadcaster func CreateBroadcasters(conn int, orderer Node, logger *log.Logger) Broadcasters { bs := make(Broadcasters, conn) - TLSCACert, err := GetTLSCACerts(orderer.TLSCACert) - if err != nil { - panic(err) - } for i := 0; i < conn; i++ { - bs[i] = CreateBroadcaster(orderer.Addr, TLSCACert, logger) + bs[i] = CreateBroadcaster(orderer, logger) } return bs @@ -116,8 +108,8 @@ type Broadcaster struct { logger *log.Logger } -func CreateBroadcaster(addr string, tlscacert []byte, logger *log.Logger) *Broadcaster { - client, err := CreateBroadcastClient(addr, tlscacert) +func CreateBroadcaster(node Node, logger *log.Logger) *Broadcaster { + client, err := CreateBroadcastClient(node) if err != nil { panic(err) } diff --git a/pkg/infra/proposer_test.go b/pkg/infra/proposer_test.go index 36fad5e9..5fb86572 100644 --- a/pkg/infra/proposer_test.go +++ b/pkg/infra/proposer_test.go @@ -18,20 +18,19 @@ var _ = Describe("Proposer", func() { Context("CreateProposer", func() { It("successfully creates a proposer", func() { - var dummy []byte - Proposer := infra.CreateProposer(addr, dummy, nil) - Expect(Proposer.Addr).To(Equal(addr)) - }) - - It("successfully creates a proposer even tls is null", func() { - Proposer := infra.CreateProposer(addr, nil, nil) + dummy := infra.Node{ + Addr: addr, + } + Proposer := infra.CreateProposer(dummy, nil) Expect(Proposer.Addr).To(Equal(addr)) }) It("error happen creates a proposer", func() { - var dummy []byte + dummy := infra.Node{ + Addr: "invalid_addr", + } Expect(func() { - infra.CreateProposer("invalid_addr", dummy, nil) + infra.CreateProposer(dummy, nil) }).To(Panic()) }) }) diff --git a/test/configmock.yaml b/test/configmock.yaml index 1315ad6b..f6c69df5 100644 --- a/test/configmock.yaml +++ b/test/configmock.yaml @@ -14,7 +14,7 @@ orderer: *node channel: test-channel chaincode: test-chaincode mspid: Org1MSP -private_key: ./organizations/peerOrganizations/org1.example.com/users/User1@org1.example.com/msp/keystore/priv_sk -sign_cert: ./organizations/peerOrganizations/org1.example.com/users/User1@org1.example.com/msp/signcerts/User1@org1.example.com-cert.pem +private_key: ./priv_sk +sign_cert: ./User1@org1.example.com-cert.pem num_of_conn: 10 client_per_conn: 10 diff --git a/test/configmockmtls.yaml b/test/configmockmtls.yaml new file mode 100644 index 00000000..002029e0 --- /dev/null +++ b/test/configmockmtls.yaml @@ -0,0 +1,23 @@ +# Definition of nodes +node: &node + addr: 127.0.0.1:10086 + tls_ca_cert: ./client.crt + tls_ca_key: ./client.key + tls_ca_root: ./ExampleCA.crt +# Nodes to interact with +endorsers: + - *node +# we might support multi-committer in the future for more complex test scenario, +# i.e. consider tx committed only if it's done on >50% of nodes. But for now, +# it seems sufficient to support single committer. +committer: *node +orderer: *node + +# Invocation configs +channel: test-channel +chaincode: test-chaincode +mspid: Org1MSP +private_key: ./priv_sk +sign_cert: ./User1@org1.example.com-cert.pem +num_of_conn: 10 +client_per_conn: 10 diff --git a/test/integration-test.sh b/test/integration-test.sh index 83e51ce5..974c0ee1 100755 --- a/test/integration-test.sh +++ b/test/integration-test.sh @@ -7,19 +7,6 @@ curl -vsS https://raw.githubusercontent.com/hyperledger/fabric/master/scripts/bo cd ./fabric-samples/ case $1 in - mock) - #This is a tiny, mocked Fabric that does **nothing**, so it runs really fast. - #We use this tool to test the limit of `stupid`, and making sure there's no performance regression. - #By Aug 2020, `stupid` could produce ~15k tps. - cd test-network - echo y | ./network.sh down - echo y | ./network.sh up - cp -r organizations "$DIR" - cd "$DIR" - CONFIG_FILE=./test/configmock.yaml - go build ./mock/fabric - nohup ./fabric 2>&1 & - ;; 14) git checkout release-1.4 cd first-network @@ -47,8 +34,7 @@ case $1 in echo y | ./network.sh deployCC "${ARGS[@]}" ;; *) - echo "Usage: $1 [mock|14|22]" - echo "For mock, use our mock version for fabric to test stupid itself" + echo "Usage: $1 [14|22]" echo "When given version, start byfn or test network basing on specific version of docker image" echo "For any value without mock, 14, 22 will show this hint" exit 0