Merge pull request #33 from bojand/bindata

Add support for binary data

README.md

@@ -58,6 +58,8 @@ Options:
   -d  The call data as stringified JSON.
       If the value is '@' then the request contents are read from stdin.
   -D  Path for call data JSON file. For example, /home/user/file.json or ./file.json.
+  -b  The call data comes as serialized binary message read from stdin.
+  -B  Path for the call data as serialized binary message.
   -m  Request metadata as stringified JSON.
   -M  Path for call metadata JSON file. For example, /home/user/metadata.json or ./metadata.json.
 
@@ -121,6 +123,18 @@ A simple unary call with metadata using template actions:
 ghz -proto ./greeter.proto -call helloworld.Greeter.SayHello -d '{"name":"Joe"}' -m '{"trace_id":"{{.RequestNumber}}","timestamp":"{{.TimestampUnix}}"}' 0.0.0.0:50051
 ```
 
+Using binary data file (see [writing a message](https://developers.google.com/protocol-buffers/docs/gotutorial#writing-a-message)):
+
+```sh
+ghz -proto ./greeter.proto -call helloworld.Greeter.SayHello -B ./hello_request_data.bin 0.0.0.0:50051
+```
+
+Or using binary from stdin:
+
+```sh
+ghz -proto ./greeter.proto -call helloworld.Greeter.SayHello -b 0.0.0.0:50051 < ./hello_request_data.bin
+```
+
 Custom number of requests and concurrency:
 
 ```sh

cmd/ghz/main.go

@@ -37,6 +37,8 @@ var (
 
 	data     = flag.String("d", "", "The call data as stringified JSON. If the value is '@' then the request contents are read from stdin.")
 	dataPath = flag.String("D", "", "Path for call data JSON file.")
+	binData  = flag.Bool("b", false, "The call data as serialized binary message read from stdin.")
+	binPath  = flag.String("B", "", "The call data as serialized binary message read from a file.")
 	md       = flag.String("m", "", "Request metadata as stringified JSON.")
 	mdPath   = flag.String("M", "", "Path for call metadata JSON file.")
 
@@ -80,6 +82,8 @@ Options:
   -d  The call data as stringified JSON.
       If the value is '@' then the request contents are read from stdin.
   -D  Path for call data JSON file. For example, /home/user/file.json or ./file.json.
+  -b  The call data comes as serialized binary message read from stdin.
+  -B  Path for the call data as serialized binary message.
   -m  Request metadata as stringified JSON.
   -M  Path for call metadata JSON file. For example, /home/user/metadata.json or ./metadata.json.
 
@@ -144,7 +148,9 @@ func main() {
 		}
 
 		cfg, err = config.New(*proto, *protoset, *call, *cert, *cname, *n, *c, *q, *z, *x, *t,
-			*data, *dataPath, *md, *mdPath, *output, *format, host, *ct, *kt, *cpus, iPaths, *insecure)
+			*data, *dataPath, *binData, *binPath, *md, *mdPath, *output, *format, host,
+			*ct, *kt, *cpus, iPaths, *insecure)
+
 		if err != nil {
 			errAndExit(err.Error())
 		}
@@ -202,6 +208,11 @@ func runTest(config *config.Config) (*ghz.Report, error) {
 		input = config.Protoset
 	}
 
+	binary := false
+	if len(config.BinData) > 0 {
+		binary = true
+	}
+
 	opts := &ghz.Options{
 		Proto:         input,
 		Call:          config.Call,
@@ -216,6 +227,8 @@ func runTest(config *config.Config) (*ghz.Report, error) {
 		DialTimtout:   config.DialTimeout,
 		KeepaliveTime: config.KeepaliveTime,
 		Data:          config.Data,
+		BinData:       config.BinData,
+		Binary:        binary,
 		Metadata:      config.Metadata,
 		Insecure:      config.Insecure,
 	}

config/config.go

@@ -29,6 +29,8 @@ type Config struct {
 	Timeout       int                `json:"t"`
 	Data          interface{}        `json:"d,omitempty"`
 	DataPath      string             `json:"D"`
+	BinData       []byte             `json:"-"`
+	BinDataPath   string             `json:"B"`
 	Metadata      *map[string]string `json:"m,omitempty"`
 	MetadataPath  string             `json:"M"`
 	Output        string             `json:"o"`
@@ -43,7 +45,7 @@ type Config struct {
 
 // New creates a new config
 func New(proto, protoset, call, cert, cName string, n, c, qps int, z time.Duration, x time.Duration,
-	timeout int, data, dataPath, metadata, mdPath, output, format, host string,
+	timeout int, data, dataPath string, binData bool, binPath, metadata, mdPath, output, format, host string,
 	dialTimout, keepaliveTime, cpus int, importPaths []string, insecure bool) (*Config, error) {
 
 	cfg := &Config{
@@ -59,6 +61,7 @@ func New(proto, protoset, call, cert, cName string, n, c, qps int, z time.Durati
 		X:             x,
 		Timeout:       timeout,
 		DataPath:      dataPath,
+		BinDataPath:   binPath,
 		MetadataPath:  mdPath,
 		Output:        output,
 		Format:        format,
@@ -82,6 +85,14 @@ func New(proto, protoset, call, cert, cName string, n, c, qps int, z time.Durati
 		return nil, err
 	}
 
+	if binData {
+		b, err := ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			return nil, err
+		}
+		cfg.BinData = b
+	}
+
 	err = cfg.setMetadata(metadata)
 	if err != nil {
 		return nil, err
@@ -174,7 +185,7 @@ func (c *Config) Validate() error {
 	}
 
 	if strings.TrimSpace(c.DataPath) == "" {
-		if c.Data == nil {
+		if strings.TrimSpace(c.BinDataPath) == "" && len(c.BinData) == 0 && c.Data == nil {
 			return errors.New("data: is required")
 		}
 	}
@@ -242,6 +253,15 @@ func (c *Config) initData() error {
 		}
 
 		return json.Unmarshal(d, &c.Data)
+	} else if c.BinData != nil {
+		return nil
+	} else if strings.TrimSpace(c.BinDataPath) != "" {
+		d, err := ioutil.ReadFile(c.BinDataPath)
+		if err != nil {
+			return err
+		}
+		c.BinData = d
+		return nil
 	}
 
 	return errors.New("No data specified")

config/config_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const expected = `{"proto":"asdf","protoset":"","call":"","cert":"","cName":"","n":0,"c":0,"q":0,"t":0,"D":"","M":"","o":"","O":"oval","host":"","T":0,"L":0,"cpus":0,"z":"4h30m0s","x":""}`
+const expected = `{"proto":"asdf","protoset":"","call":"","cert":"","cName":"","n":0,"c":0,"q":0,"t":0,"D":"","B":"","M":"","o":"","O":"oval","host":"","T":0,"L":0,"cpus":0,"z":"4h30m0s","x":""}`
 
 func TestConfig_MarshalJSON(t *testing.T) {
 	z, _ := time.ParseDuration("4h30m")
@@ -388,6 +388,12 @@ func TestConfig_Validate(t *testing.T) {
 		err := c.Validate()
 		assert.NoError(t, err)
 	})
+
+	t.Run("BinDataPath", func(t *testing.T) {
+		c := &Config{Proto: "asdf.proto", Call: "call", Cert: "cert", BinDataPath: "asdf"}
+		err := c.Validate()
+		assert.NoError(t, err)
+	})
 }
 
 func TestConfig_initData(t *testing.T) {
@@ -418,6 +424,24 @@ func TestConfig_initData(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, c.Data, data)
 	})
+
+	t.Run("with bin data", func(t *testing.T) {
+		in, err := ioutil.ReadFile("../testdata/hello_request_data.bin")
+		assert.NoError(t, err)
+		c := &Config{BinData: in}
+		err = c.initData()
+		assert.NoError(t, err)
+		assert.Equal(t, c.BinData, in)
+	})
+
+	t.Run("with bin data file", func(t *testing.T) {
+		in, err := ioutil.ReadFile("../testdata/hello_request_data.bin")
+		assert.NoError(t, err)
+		c := &Config{BinDataPath: "../testdata/hello_request_data.bin"}
+		err = c.initData()
+		assert.NoError(t, err)
+		assert.Equal(t, c.BinData, in)
+	})
 }
 
 func TestConfig_initDurations(t *testing.T) {

data.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
 	"github.com/jhump/protoreflect/desc"
 	"github.com/jhump/protoreflect/dynamic"
 )
@@ -94,3 +95,22 @@ func createPayloads(data interface{}, mtd *desc.MethodDescriptor) (*dynamic.Mess
 
 	return input, &streamInput, nil
 }
+
+func createPayloadsFromBin(binData []byte, mtd *desc.MethodDescriptor) (*dynamic.Message, *[]*dynamic.Message, error) {
+	md := mtd.GetInputType()
+	input := dynamic.NewMessage(md)
+	streamInput := make([]*dynamic.Message, 1)
+
+	err := proto.Unmarshal(binData, input)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if mtd.IsClientStreaming() && input != nil {
+		streamInput = make([]*dynamic.Message, 1)
+		streamInput[0] = input
+		input = nil
+	}
+
+	return input, &streamInput, nil
+}

requester.go

@@ -36,6 +36,8 @@ type Options struct {
 	DialTimtout   int                `json:"dialTimeout,omitempty"`
 	KeepaliveTime int                `json:"keepAlice,omitempty"`
 	Data          interface{}        `json:"data,omitempty"`
+	Binary        bool               `json:"binary"`
+	BinData       []byte             `json:"-"`
 	Metadata      *map[string]string `json:"metadata,omitempty"`
 	Insecure      bool               `json:"insecure,omitempty"`
 }
@@ -59,6 +61,7 @@ type Requester struct {
 
 	data     string
 	metadata string
+	binData  []byte
 
 	config  *Options
 	results chan *callResult
@@ -94,6 +97,7 @@ func New(mtd *desc.MethodDescriptor, c *Options) (*Requester, error) {
 		config:   c,
 		data:     string(dataJSON),
 		metadata: string(mdJSON),
+		binData:  c.BinData,
 		mtd:      mtd}
 
 	return reqr, nil
@@ -219,9 +223,24 @@ func (b *Requester) makeRequest() {
 
 	ctd := newCallTemplateData(b.mtd, reqNum)
 
-	dataMap, err := ctd.executeData(b.data)
-	if err != nil {
-		return
+	var input *dynamic.Message
+	var streamInput *[]*dynamic.Message
+
+	if len(b.data) > 0 && len(b.binData) == 0 {
+		dataMap, err := ctd.executeData(b.data)
+		if err != nil {
+			return
+		}
+		input, streamInput, err = createPayloads(dataMap, b.mtd)
+		if err != nil {
+			return
+		}
+	} else {
+		var err error
+		input, streamInput, err = createPayloadsFromBin(b.binData, b.mtd)
+		if err != nil {
+			return
+		}
 	}
 
 	mdMap, err := ctd.executeMetadata(b.metadata)
@@ -235,11 +254,6 @@ func (b *Requester) makeRequest() {
 		reqMD = &md
 	}
 
-	input, streamInput, err := createPayloads(dataMap, b.mtd)
-	if err != nil {
-		return
-	}
-
 	ctx := context.Background()
 
 	ctx, cancel := context.WithCancel(ctx)

requester_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/bojand/ghz/internal/helloworld"
@@ -141,6 +142,36 @@ func TestRequesterUnary(t *testing.T) {
 		}()
 		wg.Wait()
 	})
+
+	t.Run("test report binary", func(t *testing.T) {
+		gs.ResetCounters()
+
+		msg := &helloworld.HelloRequest{}
+		msg.Name = "bob"
+
+		binData, err := proto.Marshal(msg)
+
+		reqr, err := New(md, &Options{
+			Host:        localhost,
+			N:           5,
+			C:           1,
+			Timeout:     20,
+			DialTimtout: 20,
+			BinData:     binData,
+			Binary:      true,
+			Insecure:    true,
+		})
+		assert.NoError(t, err)
+
+		report, err := reqr.Run()
+		assert.NoError(t, err)
+		assert.NotNil(t, report)
+		assert.Equal(t, 5, int(report.Count))
+		assert.Len(t, report.ErrorDist, 0)
+
+		count := gs.GetCount(callType)
+		assert.Equal(t, 5, count)
+	})
 }
 
 func TestRequesterServerStreaming(t *testing.T) {
@@ -231,6 +262,50 @@ func TestRequesterClientStreaming(t *testing.T) {
 	assert.Equal(t, 16, count)
 }
 
+func TestRequesterClientStreamingBinary(t *testing.T) {
+	callType := helloworld.ClientStream
+
+	gs, s, err := startServer(false)
+
+	if err != nil {
+		assert.FailNow(t, err.Error())
+	}
+
+	defer s.Stop()
+
+	md, err := protodesc.GetMethodDescFromProto("helloworld.Greeter.SayHelloCS", "./testdata/greeter.proto", []string{})
+
+	msg := &helloworld.HelloRequest{}
+	msg.Name = "bob"
+
+	binData, err := proto.Marshal(msg)
+
+	gs.ResetCounters()
+
+	reqr, err := New(md, &Options{
+		Host:        localhost,
+		N:           24,
+		C:           4,
+		Timeout:     20,
+		DialTimtout: 20,
+		BinData:     binData,
+		Binary:      true,
+		Insecure:    true,
+	})
+	assert.NoError(t, err)
+
+	report, err := reqr.Run()
+	assert.NoError(t, err)
+	assert.NotNil(t, report)
+	assert.Equal(t, 24, int(report.Count))
+	assert.True(t, len(report.LatencyDistribution) > 1)
+	assert.True(t, len(report.Histogram) > 1)
+	assert.Len(t, report.ErrorDist, 0)
+
+	count := gs.GetCount(callType)
+	assert.Equal(t, 24, count)
+}
+
 func TestRequesterBidi(t *testing.T) {
 	callType := helloworld.Bidi