From 983b9da5222ddbeb4aa82ed92abc0c97078d3031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 10:23:03 +0200 Subject: [PATCH 01/36] Refactor (#8) Refactor the application to make it simplier. The point is to have a minimal core and multiple reusable plugins to add functionality. - No bidirectional references between structs. - Remove logic that can be built into a plugin (limiter) - Safe use of channels - Use interfaces to hide implementation - Loosely coupled modules - Reporter - Pipeline Closes #6 --- .travis.yml | 17 +- Makefile | 14 +- README.adoc | 78 ------ README.md | 52 ++++ backend.go | 251 ----------------- config.go | 8 + message.go | 47 ++++ messages.go | 47 ---- pipeline.go | 102 +++++++ rbforwarder.go | 146 +++------- rbforwarder_test.go | 438 +++++++++++++++++++++--------- report.go | 9 + reporter.go | 124 +++++++++ reporthandler.go | 157 ----------- senders/httpsender/helper.go | 76 ------ senders/httpsender/sender.go | 225 --------------- senders/httpsender/sender_test.go | 22 -- types/composer.go | 17 ++ types/messenger.go | 8 + 19 files changed, 731 insertions(+), 1107 deletions(-) delete mode 100644 README.adoc create mode 100644 README.md delete mode 100644 backend.go create mode 100644 config.go create mode 100644 message.go delete mode 100644 messages.go create mode 100644 pipeline.go create mode 100644 report.go create mode 100644 reporter.go delete mode 100644 reporthandler.go delete mode 100644 senders/httpsender/helper.go delete mode 100644 senders/httpsender/sender.go delete mode 100644 senders/httpsender/sender_test.go create mode 100644 types/composer.go create mode 100644 types/messenger.go diff --git a/.travis.yml b/.travis.yml index db32cda..88cd667 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,18 +8,21 @@ branches: - develop go: -- 1.5.2 + - 1.5.4 + - 1.6.3 script: - $HOME/gopath/bin/goveralls install: -- make get -- make get_dev + - go get github.com/mattn/goveralls + - go get github.com/go-playground/overalls + - go get -t ./... script: -#- make check -- make test + - go test -v -race -cover ./... -after_script: -- make coverage +after_success: + - overalls -covermode=set -project=github.com/redBorder/rbforwarder + - go tool cover -func overalls.coverprofile + - goveralls -coverprofile=overalls.coverprofile -service=travis-ci diff --git a/Makefile b/Makefile index 51506d8..95830e2 100644 --- a/Makefile +++ b/Makefile @@ -25,25 +25,15 @@ vet: test: @printf "$(MKL_YELLOW)Runing tests$(MKL_CLR_RESET)\n" - go test -cover ./... + go test -race ./... @printf "$(MKL_GREEN)Test passed$(MKL_CLR_RESET)\n" coverage: @printf "$(MKL_YELLOW)Computing coverage$(MKL_CLR_RESET)\n" @overalls -covermode=set -project=github.com/redBorder/rbforwarder @go tool cover -func overalls.coverprofile - @goveralls -coverprofile=overalls.coverprofile -service=travis-ci @rm -f overalls.coverprofile -get_dev: - @printf "$(MKL_YELLOW)Installing deps$(MKL_CLR_RESET)\n" - go get golang.org/x/tools/cmd/cover - go get github.com/kisielk/errcheck - go get github.com/stretchr/testify/assert - go get github.com/mattn/goveralls - go get github.com/axw/gocov/gocov - go get github.com/go-playground/overalls - get: @printf "$(MKL_YELLOW)Installing deps$(MKL_CLR_RESET)\n" - go get -t ./... + go get -v ./... diff --git a/README.adoc b/README.adoc deleted file mode 100644 index b0ebd55..0000000 --- a/README.adoc +++ /dev/null @@ -1,78 +0,0 @@ -image:https://img.shields.io/packagist/l/doctrine/orm.svg?maxAge=2592000[] -image:https://travis-ci.org/redBorder/rbforwarder.svg?branch=master["Build Status", link="https://travis-ci.org/redBorder/rbforwarder"] -image:https://goreportcard.com/badge/github.com/redBorder/rbforwarder["Go Report", link=https://goreportcard.com/report/github.com/redBorder/rbforwarder] -image:https://coveralls.io/repos/github/redBorder/rbforwarder/badge.svg?branch=master["Coverage",link=https://coveralls.io/github/redBorder/rbforwarder?branch=master] -image:https://godoc.org/github.com/redBorder/rbforwarder?status.svg[link=https://godoc.org/github.com/redBorder/rbforwarder] - -= rbforwarder - -*rbforwarder* is a multi-protocol data forwarder. You can use it to forward data -between different protocols, for instance, creating a Kafka consumer to get -data from Kafka and use rbforwarder to send it to an HTTP endpoint. - -The application also support decoding, encoding and processing data, for example -it can decode a JSON and performs modifications on it before send it encoded as -JSON again. - -The application uses a pipeline with multiple workers to process data. It also -have a reporter to know if messages has been delivered successfully and can -retry messages on fail. - -== Components - -*Senders* - -* MQTT -* HTTP -* Kafka - -*Decoders / Encoders* - -* JSON - -*Processors* - -- Template - -== Road Map - -|=== -| Milestone | Feature | Status - -| 0.1 -| Backend -| _Beta_ - -| 0.2 -| Report Handler -| _Beta_ - -| 0.4 -| HTTP Sender -| _Beta_ - -| 0.5 -| JSON encoder/decoder -| _Pending_ - -| 0.6 -| JSON Template Processor -| _Pending_ - -| 0.7 -| Kafka Producer -| _Pending_ - -| 0.8 -| MQTT Publisher -| _Pending_ - -| 0.9 -| InfluxDB metrics -| _Pending_ - -| 1.0 -| Benchmarks -| _Pending_ - -|=== diff --git a/README.md b/README.md new file mode 100644 index 0000000..4345255 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +![](https://img.shields.io/packagist/l/doctrine/orm.svg?maxAge=2592000) +[![](https://travis-ci.org/redBorder/rbforwarder.svg?branch=master)](https://travis-ci.org/redBorder/rbforwarder) +[![](https://goreportcard.com/badge/github.com/redBorder/rbforwarder)](https://goreportcard.com/report/github.com/redBorder/rbforwarder) +[![](https://coveralls.io/repos/github/redBorder/rbforwarder/badge.svg?branch=master)](https://coveralls.io/github/redBorder/rbforwarder?branch=master) +[![](https://godoc.org/github.com/redBorder/rbforwarder?status.svg)](https://godoc.org/github.com/redBorder/rbforwarder) + +# rbforwarder + +**rbforwarder** is an extensible, protocol agnostic and easy to use tool for +process data. It allows you to create a custom pipeline in a modular way. + +For example, you can read data from a Kafka broker and use **rbforwarder** to +build a **pipeline** that decodes the JSON, filter or add some fields, encodes +the data again to JSON and send it using using multiple protocols HTTP, MQTT, +AMQP, etc. It's easy to write a **component** for the pipeline. + +## Features + +- Support for multiple workers for every **component**. +- Support **buffer pooling**, for fine-grained memory control. +- Asynchronous report system. Different gorutine for send and receive responses. +- Built-in retry. The **rbforwarder** can retry messages on fail. + +## Components + +- Send data to an endpoint + - MQTT + - HTTP + - Fast HTTP + - Kafka +- Decoders / Encoders + - JSON +- Utility + - Limiter + - Metrics + +## Road Map + +_The application is being on hard developing, breaking changes expected._ + +|Milestone | Feature | Status | +|----------|---------------------|-----------| +| 0.1 | Pipeline | _Testing_ | +| 0.2 | Reporter | _Testing_ | +| 0.3 | Limiter | _Testing_ | +| 0.4 | HTTP component | _Testing_ | +| 0.5 | MQTT component | _Pending_ | +| 0.6 | JSON component | _Pending_ | +| 0.7 | Kafka component | _Pending_ | +| 0.8 | Metrics component | _Pending_ | +| 0.9 | Benchmarks | _Pending_ | +| 1.0 | Stable | _Pending_ | diff --git a/backend.go b/backend.go deleted file mode 100644 index 57d3903..0000000 --- a/backend.go +++ /dev/null @@ -1,251 +0,0 @@ -package rbforwarder - -import ( - "bytes" - "time" -) - -// Decoder is the component that parses a raw buffer to a structure -type Decoder interface { - Init(int) error - Decode(*Message) error -} - -// Processor performs operations on a data structure -type Processor interface { - Init(int) error - Process(message *Message) (bool, error) -} - -// Encoder serializes a data structure to a output buffer -type Encoder interface { - Init(int) error - Encode(*Message) error -} - -// Sender takes a raw buffer and sent it using a network protocol -type Sender interface { - Init(int) error - Send(*Message) error -} - -// SenderHelper is used to create Senders instances -type SenderHelper interface { - CreateSender() Sender -} - -type backend struct { - decoder Decoder - processor Processor - encoder Encoder - senderHelper SenderHelper - - // Pool of workers - decoderPool chan chan *Message - processorPool chan chan *Message - encoderPool chan chan *Message - senderPool chan chan *Message - - currentProducedID uint64 - - input chan *Message - messages chan *Message - reports chan *Message - messagePool chan *Message - - workers int - queue int - maxMessages int - maxBytes int - - active bool - - currentMessages uint64 - currentBytes uint64 - keepSending chan struct{} -} - -func (b *backend) Init() { - b.decoderPool = make(chan chan *Message, b.workers) - b.processorPool = make(chan chan *Message, b.workers) - b.encoderPool = make(chan chan *Message, b.workers) - b.senderPool = make(chan chan *Message, b.workers) - - b.messages = make(chan *Message) - b.input = make(chan *Message) - b.reports = make(chan *Message) - b.messagePool = make(chan *Message, b.queue) - - b.keepSending = make(chan struct{}) - - for i := 0; i < b.queue; i++ { - b.messagePool <- &Message{ - Metadata: make(map[string]interface{}), - InputBuffer: new(bytes.Buffer), - OutputBuffer: new(bytes.Buffer), - - backend: b, - } - } - - for i := 0; i < b.workers; i++ { - b.startDecoder(i) - b.startProcessor(i) - b.startEncoder(i) - b.startSender(i) - } - - // Limit the messages/bytes per second - go func() { - for { - timer := time.NewTimer(1 * time.Second) - <-timer.C - b.keepSending <- struct{}{} - } - }() - - // Get messages from produces - done := make(chan struct{}) - go func() { - done <- struct{}{} - for m := range b.input { - - // Wait if the limit has ben reached - if b.maxMessages > 0 && b.currentMessages >= uint64(b.maxMessages) { - <-b.keepSending - b.currentMessages = 0 - } else if b.maxBytes > 0 && b.currentBytes >= uint64(b.maxBytes) { - <-b.keepSending - b.currentBytes = 0 - } - - // Send to workers - select { - case messageChannel := <-b.decoderPool: - select { - case messageChannel <- m: - b.currentMessages++ - b.currentBytes += uint64(m.InputBuffer.Len()) - case <-time.After(1 * time.Second): - Logger.Warn("Error on produce: Full queue") - } - case <-time.After(1 * time.Second): - m.Report(-1, "Error on produce: No workers available") - } - } - }() - <-done - - b.active = true - Logger.Debug("Backend ready") -} - -// Worker that decodes the received message -func (b *backend) startDecoder(i int) { - if b.decoder != nil { - b.decoder.Init(i) - } - workerChannel := make(chan *Message) - - go func() { - for { - // The worker is ready, put himself on the worker pool - b.decoderPool <- workerChannel - - // Wait for a new message - message := <-workerChannel - - // Perform work on the message - if b.decoder != nil { - b.decoder.Decode(message) - } - - // Get a worker for the next element on the pipe - messageChannel := <-b.processorPool - - // Send the message to the next worker - messageChannel <- message - } - }() -} - -// Worker that performs modifications on a decoded message -func (b *backend) startProcessor(i int) { - if b.processor != nil { - b.processor.Init(i) - } - workerChannel := make(chan *Message) - - // The worker is ready, put himself on the worker pool - go func() { - for { - // The worker is ready, put himself on the worker pool - b.processorPool <- workerChannel - - // Wait for a new message - message := <-workerChannel - - // Perform work on the message - if b.processor != nil { - b.processor.Process(message) - } - - // Get a worker for the next element on the pipe - messageChannel := <-b.encoderPool - - // Send the message to the next worker - messageChannel <- message - } - }() -} - -// Worker that encodes a modified message -func (b *backend) startEncoder(i int) { - if b.encoder != nil { - b.encoder.Init(i) - } - workerChannel := make(chan *Message) - - go func() { - for { - // The worker is ready, put himself on the worker pool - b.encoderPool <- workerChannel - - // Wait for a new message - message := <-workerChannel - - // Perform work on the message - if b.encoder != nil { - b.encoder.Encode(message) - } else { - message.OutputBuffer = message.InputBuffer - } - - // Get a worker for the next element on the pipe - messageChannel := <-b.senderPool - - // Send the message to the next worker - messageChannel <- message - } - }() -} - -// Worker that sends the message -func (b *backend) startSender(i int) { - if b.senderHelper == nil { - Logger.Fatal("No sender provided") - } - - sender := b.senderHelper.CreateSender() - sender.Init(i) - - workerChannel := make(chan *Message) - - go func() { - for { - b.senderPool <- workerChannel - message := <-workerChannel - sender.Send(message) - } - }() -} diff --git a/config.go b/config.go new file mode 100644 index 0000000..560aa07 --- /dev/null +++ b/config.go @@ -0,0 +1,8 @@ +package rbforwarder + +// Config stores the configuration for a forwarder +type Config struct { + Retries int + Backoff int + QueueSize int +} diff --git a/message.go b/message.go new file mode 100644 index 0000000..d2d79ba --- /dev/null +++ b/message.go @@ -0,0 +1,47 @@ +package rbforwarder + +import "errors" + +// message is used to send data through the pipeline +type message struct { + bufferStack [][]byte + + seq uint64 // Unique ID for the report, used to maintain sequence + status string // Result of the sending + code int // Result of the sending + retries int + opts map[string]interface{} + channel chan *message +} + +// PushData store data on an LIFO queue so the nexts handlers can use it +func (m *message) PushData(v []byte) { + m.bufferStack = append(m.bufferStack, v) +} + +// PopData get the data stored by the previous handler +func (m *message) PopData() (ret []byte, err error) { + if len(m.bufferStack) < 1 { + err = errors.New("No data on the stack") + return + } + + ret = m.bufferStack[len(m.bufferStack)-1] + m.bufferStack = m.bufferStack[0 : len(m.bufferStack)-1] + + return +} + +// GetOpt returns an option +func (m message) GetOpt(name string) interface{} { + return m.opts[name] +} + +func (m message) GetReport() Report { + return Report{ + code: m.code, + status: m.status, + retries: m.retries, + opts: m.opts, + } +} diff --git a/messages.go b/messages.go deleted file mode 100644 index 49fd884..0000000 --- a/messages.go +++ /dev/null @@ -1,47 +0,0 @@ -package rbforwarder - -import ( - "bytes" - "errors" - "sync/atomic" -) - -const ( - statusOk = 0 -) - -// Message is used to send data to the backend -type Message struct { - InputBuffer *bytes.Buffer // The original data from the source - Data interface{} // Can be used to store the data once it has been parsed - OutputBuffer *bytes.Buffer // The data that will be sent by the sender - Metadata map[string]interface{} // Opaque - - report Report - backend *backend // Use to send the message to the backend -} - -// Produce is used by the source to send messages to the backend -func (m *Message) Produce() error { - backend := m.backend - - if !backend.active { - return errors.New("Backend closed") - } - - m.report = Report{ - ID: atomic.AddUint64(&m.backend.currentProducedID, 1) - 1, - Metadata: m.Metadata, - } - - backend.input <- m - - return nil -} - -// Report is used by the sender to inform that a message has not been sent -func (m *Message) Report(statusCode int, status string) { - m.report.StatusCode = statusCode - m.report.Status = status - m.backend.reports <- m -} diff --git a/pipeline.go b/pipeline.go new file mode 100644 index 0000000..6dd3d0d --- /dev/null +++ b/pipeline.go @@ -0,0 +1,102 @@ +package rbforwarder + +import ( + "sync" + + "github.com/redBorder/rbforwarder/types" +) + +type component struct { + pool chan chan *message + workers int +} + +// pipeline contains the components +type pipeline struct { + components []component + input chan *message + retry chan *message + output chan *message +} + +// newPipeline creates a new Backend +func newPipeline(input, retry, output chan *message) *pipeline { + var wg sync.WaitGroup + p := &pipeline{ + input: input, + retry: retry, + output: output, + } + + wg.Add(1) + go func() { + wg.Done() + + for { + select { + case m, ok := <-p.input: + // If input channel has been closed, close output channel + if !ok { + for _, component := range p.components { + for i := 0; i < component.workers; i++ { + worker := <-component.pool + close(worker) + } + } + close(p.output) + } else { + worker := <-p.components[0].pool + worker <- m + } + + case m := <-p.retry: + worker := <-p.components[0].pool + worker <- m + } + } + }() + + wg.Wait() + return p +} + +// PushComponent adds a new component to the pipeline +func (p *pipeline) PushComponent(composser types.Composer, w int) { + var wg sync.WaitGroup + c := component{ + workers: w, + pool: make(chan chan *message, w), + } + + index := len(p.components) + p.components = append(p.components, c) + + for i := 0; i < w; i++ { + composser.Init(i) + + worker := make(chan *message) + p.components[index].pool <- worker + + wg.Add(1) + go func(i int) { + wg.Done() + for m := range worker { + composser.OnMessage(m, func(m types.Messenger) { + if len(p.components) >= index { + nextWorker := <-p.components[index+1].pool + nextWorker <- m.(*message) + } + }, func(m types.Messenger, code int, status string) { + rbmessage := m.(*message) + rbmessage.code = code + rbmessage.status = status + p.output <- rbmessage + }) + + p.components[index].pool <- worker + } + }(i) + } + + wg.Wait() +} diff --git a/rbforwarder.go b/rbforwarder.go index 99346d2..4407156 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -3,9 +3,9 @@ package rbforwarder import ( "errors" "sync/atomic" - "time" "github.com/Sirupsen/logrus" + "github.com/redBorder/rbforwarder/types" ) // Version is the current tag @@ -16,142 +16,86 @@ var log = logrus.New() // Logger for the package var Logger = logrus.NewEntry(log) -//------------------------------------------------------------------------------ -// RBForwarder -//------------------------------------------------------------------------------ - -// Config stores the configuration for a forwarder -type Config struct { - Retries int - Backoff int - Workers int - QueueSize int - MaxMessages int - MaxBytes int - ShowCounter int -} - // RBForwarder is the main objecto of the package. It has the main methods for // send messages and get reports. It has a backend for routing messages between // workers type RBForwarder struct { - backend *backend - reportHandler *reportHandler - reports chan Report - counter uint64 + p *pipeline + r *reporter - config Config + currentProducedID uint64 + working uint32 } // NewRBForwarder creates a new Forwarder object func NewRBForwarder(config Config) *RBForwarder { - backend := &backend{ - workers: config.Workers, - queue: config.QueueSize, - maxMessages: config.MaxMessages, - maxBytes: config.MaxBytes, - } - - forwarder := &RBForwarder{ - backend: backend, - reportHandler: newReportHandler(config.Retries, config.Backoff, config.QueueSize), - reports: make(chan Report, config.QueueSize), - config: config, + produces := make(chan *message, config.QueueSize) + retries := make(chan *message, config.QueueSize) + reports := make(chan *message, config.QueueSize) + + f := &RBForwarder{ + working: 1, + p: newPipeline(produces, retries, reports), + r: newReporter( + config.Retries, + config.Backoff, + reports, + retries, + ), } fields := logrus.Fields{ - "workers": config.Workers, "retries": config.Retries, "backoff_time": config.Backoff, "queue_size": config.QueueSize, - "max_messages": config.MaxMessages, - "max_bytes": config.MaxBytes, } Logger.WithFields(fields).Debug("Initialized rB Forwarder") - return forwarder + return f } -// Start spawning workers -func (f *RBForwarder) Start() { - - // Start the backend - f.backend.Init() - - // Start the report handler - f.reportHandler.Init() - - if f.config.ShowCounter > 0 { - go func() { - for { - timer := time.NewTimer( - time.Duration(f.config.ShowCounter) * time.Second, - ) - <-timer.C - if f.counter > 0 { - Logger.Infof( - "Messages per second %d", - f.counter/uint64(f.config.ShowCounter), - ) - f.counter = 0 - } - } - }() - } - - // Get reports from the backend and send them to the reportHandler - done := make(chan struct{}) - go func() { - done <- struct{}{} - for message := range f.backend.reports { - if message.report.StatusCode == 0 { - atomic.AddUint64(&f.counter, 1) - } - f.reportHandler.in <- message - } - }() - <-done - - // Listen for reutilizable messages and send them back to the pool - go func() { - done <- struct{}{} - for message := range f.reportHandler.freedMessages { - f.backend.messagePool <- message - } - }() - <-done -} - -// Close stop pending actions +// Close stops pending actions func (f *RBForwarder) Close() { - f.backend.active = false - f.reportHandler.close <- struct{}{} + atomic.StoreUint32(&f.working, 0) + close(f.p.input) } -// SetSenderHelper set a sender on the backend -func (f *RBForwarder) SetSenderHelper(SenderHelper SenderHelper) { - f.backend.senderHelper = SenderHelper +// PushComponents adds a new component to the pipeline +func (f *RBForwarder) PushComponents(components []types.Composer, w []int) { + for i, component := range components { + f.p.PushComponent(component, w[i]) + } } // GetReports is used by the source to get a report for a sent message. // Reports are delivered on the same order that was sent func (f *RBForwarder) GetReports() <-chan Report { - return f.reportHandler.GetReports() + return f.r.GetReports() } // GetOrderedReports is the same as GetReports() but the reports are delivered // in order func (f *RBForwarder) GetOrderedReports() <-chan Report { - return f.reportHandler.GetOrderedReports() + return f.r.GetOrderedReports() } -// TakeMessage returns a message from the message pool -func (f *RBForwarder) TakeMessage() (message *Message, err error) { - message, ok := <-f.backend.messagePool - if !ok { - err = errors.New("Pool closed") +// Produce is used by the source to send messages to the backend +func (f *RBForwarder) Produce(buf []byte, options map[string]interface{}) error { + if atomic.LoadUint32(&f.working) == 0 { + return errors.New("Forwarder has been closed") + } + + seq := f.currentProducedID + f.currentProducedID++ + + message := &message{ + seq: seq, + opts: options, } - return + message.PushData(buf) + f.p.input <- message + + return nil } diff --git a/rbforwarder_test.go b/rbforwarder_test.go index 623c060..26ad30a 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -1,192 +1,368 @@ package rbforwarder import ( - "fmt" - "math/rand" "testing" - "time" + "github.com/redBorder/rbforwarder/types" . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/mock" ) -type TestSender struct { - channel chan string +type MockMiddleComponent struct { + mock.Mock } -type TestSenderHelper struct { - channel chan string + +func (c *MockMiddleComponent) Init(id int) error { + args := c.Called() + return args.Error(0) } -func (tsender *TestSender) Init(id int) error { - return nil +func (c *MockMiddleComponent) OnMessage( + m types.Messenger, + next types.Next, + done types.Done, +) { + c.Called(m) + data, _ := m.PopData() + processedData := "-> [" + string(data) + "] <-" + m.PushData([]byte(processedData)) + next(m) + } -func (tsender *TestSender) Send(m *Message) error { - time.Sleep((time.Millisecond * 10) * time.Duration(rand.Int31n(50))) +type MockComponent struct { + mock.Mock + + channel chan string + + status string + statusCode int +} - select { - case tsender.channel <- string(m.OutputBuffer.Bytes()): - m.Report(0, "OK") - } - return nil +func (c *MockComponent) Init(id int) error { + args := c.Called() + return args.Error(0) } -func (tsh *TestSenderHelper) CreateSender() Sender { - return &TestSender{ - channel: tsh.channel, - } +func (c *MockComponent) OnMessage( + m types.Messenger, + next types.Next, + done types.Done, +) { + c.Called(m) + + data, _ := m.PopData() + c.channel <- string(data) + done(m, c.statusCode, c.status) } -func TestBackend(t *testing.T) { - Convey("Given a working backend", t, func() { - senderChannel := make(chan string, 100) - reportChannel := make(chan Report, 100) +func TestRBForwarder(t *testing.T) { + Convey("Given a single component working pipeline", t, func() { + numMessages := 10000 + numWorkers := 10 + numRetries := 3 - senderHelper := new(TestSenderHelper) - senderHelper.channel = senderChannel + component := &MockComponent{ + channel: make(chan string, 10000), + } rbforwarder := NewRBForwarder(Config{ - Retries: 0, - Workers: 10, - QueueSize: 10000, + Retries: numRetries, + QueueSize: numMessages, }) - go func() { - for report := range rbforwarder.GetOrderedReports() { - select { - case reportChannel <- report: - default: - Printf("Can't send report to report channel") + component.On("Init").Return(nil).Times(numWorkers) + + var components []types.Composer + var instances []int + components = append(components, component) + instances = append(instances, numWorkers) + + rbforwarder.PushComponents(components, instances) + + //////////////////////////////////////////////////////////////////////////// + + Convey("When a \"Hello World\" message is produced", func() { + component.status = "OK" + component.statusCode = 0 + closed := false + + component.On("OnMessage", mock.MatchedBy(func(m *message) bool { + opt := m.GetOpt("message_id") + + if !closed { + rbforwarder.Close() + closed = true } - } - }() - rbforwarder.SetSenderHelper(senderHelper) - rbforwarder.Start() + return opt.(string) == "test123" + })).Times(1) - Convey("When a \"Hello World\" message is received", func() { + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + ) - message, err := rbforwarder.TakeMessage() - if err != nil { - Printf(err.Error()) - } - message.InputBuffer.WriteString("Hola mundo") - if err := message.Produce(); err != nil { - Printf(err.Error()) - } + Convey("\"Hello World\" message should be get by the worker", func() { + var lastReport Report + var reports int + for report := range rbforwarder.GetReports() { + reports++ + lastReport = report + } + + So(lastReport, ShouldNotBeNil) + So(reports, ShouldEqual, 1) + So(lastReport.opts["message_id"], ShouldEqual, "test123") + So(lastReport.code, ShouldEqual, 0) + So(lastReport.status, ShouldEqual, "OK") + + So(err, ShouldBeNil) - Convey("A \"Hello World\" message should be sent", func() { - message := <-senderChannel - So(message, ShouldEqual, "Hola mundo") + component.AssertExpectations(t) }) }) - }) -} -func TestBackend2(t *testing.T) { - Convey("Given a working backend", t, func() { - reportChannel := make(chan Report, 100) - senderChannel := make(chan string, 100) + // //////////////////////////////////////////////////////////////////////////// - senderHelper := new(TestSenderHelper) - senderHelper.channel = senderChannel + Convey("When a message is produced after close forwarder", func() { + rbforwarder.Close() - rbforwarder := NewRBForwarder(Config{ - Retries: 0, - Workers: 10, - QueueSize: 10000, + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + ) + + Convey("Should error", func() { + So(err.Error(), ShouldEqual, "Forwarder has been closed") + }) }) - rbforwarder.SetSenderHelper(senderHelper) - rbforwarder.Start() + //////////////////////////////////////////////////////////////////////////// - Convey("When a \"Hello World\" message is received", func() { - go func() { - for report := range rbforwarder.GetOrderedReports() { - select { - case reportChannel <- report: - default: - Printf("Can't send report to report channel") - } + Convey("When calling OnMessage() with options", func() { + component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) + + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"option": "example_option"}, + ) + + rbforwarder.Close() + + Convey("Should be possible to read an option", func() { + for report := range rbforwarder.GetReports() { + So(report.opts, ShouldNotBeNil) + So(report.opts["option"], ShouldEqual, "example_option") } - }() - message, err := rbforwarder.TakeMessage() - if err != nil { - Printf(err.Error()) - } - message.InputBuffer.WriteString("Hola mundo") - if err := message.Produce(); err != nil { - Printf(err.Error()) - } + So(err, ShouldBeNil) + component.AssertExpectations(t) + }) - Convey("A report of the sent message should be received", func() { - report := <-reportChannel - So(report.StatusCode, ShouldEqual, 0) - So(report.Status, ShouldEqual, "OK") - So(report.ID, ShouldEqual, 0) + Convey("Should not be possible to read an nonexistent option", func() { + for report := range rbforwarder.GetReports() { + So(report.opts, ShouldNotBeNil) + So(report.opts["invalid"], ShouldBeEmpty) + } + + So(err, ShouldBeNil) + component.AssertExpectations(t) }) }) - }) -} -func TestBackend3(t *testing.T) { - Convey("Given a working backend", t, func() { - senderChannel := make(chan string, 100) - reportChannel := make(chan Report, 100) + // //////////////////////////////////////////////////////////////////////////// - senderHelper := new(TestSenderHelper) - senderHelper.channel = senderChannel + Convey("When calling OnMessage() without options", func() { + component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) - rbforwarder := NewRBForwarder(Config{ - Retries: 0, - Workers: 10, - QueueSize: 10000, + err := rbforwarder.Produce( + []byte("Hello World"), + nil, + ) + + rbforwarder.Close() + + Convey("Should not be possible to read the option", func() { + for report := range rbforwarder.GetReports() { + So(err, ShouldBeNil) + So(report.opts, ShouldBeNil) + } + + So(err, ShouldBeNil) + component.AssertExpectations(t) + }) }) - rbforwarder.SetSenderHelper(senderHelper) - rbforwarder.Start() + // //////////////////////////////////////////////////////////////////////////// - Convey("When multiple messages are received", func() { + Convey("When a message fails to send", func() { + component.status = "Fake Error" + component.statusCode = 99 + closed := false - fmt.Println("") - go func() { - for report := range rbforwarder.GetOrderedReports() { - select { - case reportChannel <- report: - default: - Printf("Can't send report to report channel") - } - } - }() + component.On("OnMessage", mock.MatchedBy(func(m *message) bool { + opt := m.GetOpt("message_id") - for i := 0; i < 100; i++ { - message, err := rbforwarder.TakeMessage() - if err != nil { - Printf(err.Error()) + if m.retries >= 3 && !closed { + closed = true + rbforwarder.Close() } - message.InputBuffer.WriteString("Message") - if err := message.Produce(); err != nil { - Printf(err.Error()) + + return opt.(string) == "test123" + })).Times(4) + + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + ) + + Convey("The message should be retried\n", func() { + So(err, ShouldBeNil) + + var reports int + var lastReport Report + for report := range rbforwarder.GetReports() { + reports++ + lastReport = report } - go func() { - <-senderChannel - }() + So(lastReport, ShouldNotBeNil) + So(reports, ShouldEqual, 1) + So(lastReport.opts["message_id"], ShouldEqual, "test123") + So(lastReport.status, ShouldEqual, "Fake Error") + So(lastReport.code, ShouldEqual, 99) + So(lastReport.retries, ShouldEqual, numRetries) + + component.AssertExpectations(t) + }) + }) + + //////////////////////////////////////////////////////////////////////////// + + Convey("When 10000 messages are produced", func() { + var numErr int + + component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")). + Return(nil). + Times(numMessages) + + for i := 0; i < numMessages; i++ { + if err := rbforwarder.Produce([]byte("Hello World"), + map[string]interface{}{"message_id": i}, + ); err != nil { + numErr++ + } } - Convey("Reports should be delivered ordered", func() { - var currentID uint64 + Convey("10000 reports should be received", func() { + var reports int + for range rbforwarder.GetReports() { + reports++ + if reports >= numMessages { + rbforwarder.Close() + } + } + + So(numErr, ShouldBeZeroValue) + So(reports, ShouldEqual, numMessages) + + component.AssertExpectations(t) + }) - forLoop: - for currentID = 0; currentID < 100; currentID++ { - report := <-reportChannel - if report.ID != currentID { - Printf("Missmatched report. Expected %d, Got %d", currentID, report.ID) - break forLoop + Convey("10000 reports should be received in order", func() { + ordered := true + var reports int + + for report := range rbforwarder.GetOrderedReports() { + if report.opts["message_id"] != reports { + ordered = false + } + reports++ + if reports >= numMessages { + rbforwarder.Close() } } - So(currentID, ShouldEqual, 100) + So(numErr, ShouldBeZeroValue) + So(ordered, ShouldBeTrue) + So(reports, ShouldEqual, numMessages) + + component.AssertExpectations(t) + }) + }) + }) + + Convey("Given a multi-component working pipeline", t, func() { + numMessages := 100 + numWorkers := 3 + numRetries := 3 + + component1 := &MockMiddleComponent{} + component2 := &MockComponent{ + channel: make(chan string, 10000), + } + + rbforwarder := NewRBForwarder(Config{ + Retries: numRetries, + QueueSize: numMessages, + }) + + for i := 0; i < numWorkers; i++ { + component1.On("Init").Return(nil) + component2.On("Init").Return(nil) + } + + var components []types.Composer + var instances []int + + components = append(components, component1) + components = append(components, component2) + + instances = append(instances, numWorkers) + instances = append(instances, numWorkers) + + rbforwarder.PushComponents(components, instances) + + Convey("When a \"Hello World\" message is produced", func() { + component2.status = "OK" + component2.statusCode = 0 + + component1.On("OnMessage", mock.MatchedBy(func(m *message) bool { + opt := m.GetOpt("message_id") + return opt.(string) == "test123" + })) + + component2.On("OnMessage", mock.MatchedBy(func(m *message) bool { + opt := m.GetOpt("message_id") + return opt.(string) == "test123" + })) + + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + ) + + rbforwarder.Close() + + Convey("\"Hello World\" message should be processed by the pipeline", func() { + reports := 0 + for report := range rbforwarder.GetReports() { + reports++ + + So(report.opts["message_id"], ShouldEqual, "test123") + So(report.code, ShouldEqual, 0) + So(report.status, ShouldEqual, "OK") + } + + m := <-component2.channel + + So(err, ShouldBeNil) + So(reports, ShouldEqual, 1) + So(m, ShouldEqual, "-> [Hello World] <-") + + component1.AssertExpectations(t) + component2.AssertExpectations(t) }) }) }) diff --git a/report.go b/report.go new file mode 100644 index 0000000..783a0ff --- /dev/null +++ b/report.go @@ -0,0 +1,9 @@ +package rbforwarder + +// Report contains information about a produced message +type Report struct { + code int + status string + retries int + opts map[string]interface{} +} diff --git a/reporter.go b/reporter.go new file mode 100644 index 0000000..df940fe --- /dev/null +++ b/reporter.go @@ -0,0 +1,124 @@ +package rbforwarder + +import ( + "sync" + "time" +) + +// reporter is used to handle the reports produced by the last element +// of the pipeline. The first element of the pipeline can know the status +// of the produced message using GetReports() or GetOrderedReports() +type reporter struct { + input chan *message // Receive messages from pipeline + retries chan *message // Send messages back to the pipeline + out chan *message // Send reports to the user + + queued map[uint64]Report // Store pending reports + currentReport uint64 // Last delivered report + + maxRetries int + backoff int + + wg sync.WaitGroup +} + +// newReportHandler creates a new instance of reportHandler +func newReporter( + maxRetries, backoff int, + input, retries chan *message, +) *reporter { + + r := &reporter{ + input: input, + retries: retries, + out: make(chan *message, 100), // NOTE Temp channel size + + queued: make(map[uint64]Report), + + maxRetries: maxRetries, + backoff: backoff, + } + + go func() { + // Get reports from the handler channel + for m := range r.input { + // If the message has status code 0 (success) send the report to the user + if m.code == 0 || r.maxRetries == 0 { + r.out <- m + continue + } + + // If the message has status code != 0 (fail) but has been retried the + // maximum number or retries also send it to the user + if r.maxRetries > 0 && m.retries >= r.maxRetries { + r.out <- m + continue + } + + // In othe case retry the message sending it again to the pipeline + r.wg.Add(1) + go func(m *message) { + defer r.wg.Done() + m.retries++ + <-time.After(time.Duration(r.backoff) * time.Second) + r.retries <- m + }(m) + } + + r.wg.Wait() + close(r.retries) + close(r.out) + }() + + Logger.Debug("Message Handler ready") + + return r +} + +func (r *reporter) GetReports() chan Report { + reports := make(chan Report) + + go func() { + for message := range r.out { + reports <- message.GetReport() + } + + close(reports) + }() + + return reports +} + +func (r *reporter) GetOrderedReports() chan Report { + reports := make(chan Report) + + go func() { + for message := range r.out { + report := message.GetReport() + + if message.seq == r.currentReport { + // The message is the expected. Send it. + reports <- report + r.currentReport++ + } else { + // This message is not the expected. Store it. + r.queued[message.seq] = report + } + + // Check if there are stored messages and send them. + for { + if currentReport, ok := r.queued[r.currentReport]; ok { + reports <- currentReport + delete(r.queued, r.currentReport) + r.currentReport++ + } else { + break + } + } + } + + close(reports) + }() + + return reports +} diff --git a/reporthandler.go b/reporthandler.go deleted file mode 100644 index f63a4c2..0000000 --- a/reporthandler.go +++ /dev/null @@ -1,157 +0,0 @@ -package rbforwarder - -import "time" - -// Report is used by the source to obtain the status of a sent message -type Report struct { - ID uint64 // Unique ID for the report, used to maintain sequence - Status string // Result of the sending - StatusCode int // Result of the sending - Retries int - Metadata map[string]interface{} -} - -// reportHandlerConfig is used to store the configuration for the reportHandler -type reportHandlerConfig struct { - maxRetries int - backoff int - queue int -} - -// reportHandler is used to handle the reports produced by the last element -// of the pipeline. The first element of the pipeline can know the status -// of the produced message using GetReports() or GetOrderedReports() -type reportHandler struct { - in chan *Message // Used to receive messages - freedMessages chan *Message // Used to send messages messages after its report has been delivered - unordered chan Report // Used to send reports out of order - out chan Report // Used to send reports in order - close chan struct{} // Used to stop sending reports - queued map[uint64]Report // Used to store pending reports - currentReport uint64 // Last delivered report - - config reportHandlerConfig -} - -// newReportHandler creates a new instance of reportHandler -func newReportHandler(maxRetries, backoff, queue int) *reportHandler { - return &reportHandler{ - in: make(chan *Message, queue), - freedMessages: make(chan *Message, queue), - unordered: make(chan Report, queue), - out: make(chan Report, queue), - close: make(chan struct{}), - queued: make(map[uint64]Report), - config: reportHandlerConfig{ - maxRetries: maxRetries, - backoff: backoff, - }, - } -} - -// Init initializes the processing of reports -func (r *reportHandler) Init() { - go func() { - // Get reports from the input channel - - forOuterLoop: - for { - select { - case <-r.close: - break forOuterLoop - case message := <-r.in: - - // Report when: - // - Message has been received successfully - // - Retrying has been disabled - // - The max number of retries has been reached - // Retry in the other case - if message.report.StatusCode == 0 || - r.config.maxRetries == 0 || - (r.config.maxRetries > 0 && - message.report.Retries >= r.config.maxRetries) { - - // Create a copy of the report - report := message.report - report.Metadata = message.Metadata - - // Reset message data - message.OutputBuffer.Reset() - message.Data = nil - message.Metadata = make(map[string]interface{}) - message.report = Report{} - - // Send back the message to the pool - r.freedMessages <- message - - // Send the report to the client - r.unordered <- report - } else { - go func() { - message.report.Retries++ - Logger. - WithField("ID", message.report.ID). - WithField("Retry", message.report.Retries). - WithField("Status", message.report.Status). - WithField("Code", message.report.StatusCode). - Warnf("Retrying message") - - <-time.After(time.Duration(r.config.backoff) * time.Second) - message.backend.input <- message - }() - } - } - } - close(r.unordered) - }() - - Logger.Debug("Report Handler ready") -} - -func (r *reportHandler) GetReports() chan Report { - done := make(chan struct{}) - - go func() { - done <- struct{}{} - - for report := range r.unordered { - r.out <- report - } - close(r.out) - }() - - <-done - return r.out -} - -func (r *reportHandler) GetOrderedReports() chan Report { - done := make(chan struct{}) - go func() { - done <- struct{}{} - for report := range r.unordered { - if report.ID == r.currentReport { - // The message is the expected. Send it. - r.out <- report - r.currentReport++ - } else { - // This message is not the expected. Store it. - r.queued[report.ID] = report - } - - // Check if there are stored messages and send them. - for { - if currentReport, ok := r.queued[r.currentReport]; ok { - r.out <- currentReport - delete(r.queued, r.currentReport) - r.currentReport++ - } else { - break - } - } - } - close(r.out) - }() - - <-done - return r.out -} diff --git a/senders/httpsender/helper.go b/senders/httpsender/helper.go deleted file mode 100644 index 9d00195..0000000 --- a/senders/httpsender/helper.go +++ /dev/null @@ -1,76 +0,0 @@ -package httpsender - -import ( - "time" - - "github.com/Sirupsen/logrus" - "github.com/redBorder/rbforwarder" -) - -var log = logrus.New() - -// Logger for the package -var Logger = logrus.NewEntry(log) - -// Helper is used to create instances of HTTP senders -type Helper struct { - config config -} - -// NewHelper creates a new sender helper -func NewHelper(rawConfig map[string]interface{}) Helper { - parsedConfig, _ := parseConfig(rawConfig) - - return Helper{ - config: parsedConfig, - } -} - -// CreateSender returns an instance of HTTP Sender -func (s Helper) CreateSender() rbforwarder.Sender { - httpSender := new(Sender) - httpSender.config = s.config - httpSender.logger = Logger.WithField("prefix", "sender") - - return httpSender -} - -// Parse the config from YAML file -func parseConfig(raw map[string]interface{}) (parsed config, err error) { - if raw["url"] != nil { - parsed.URL = raw["url"].(string) - } else { - Logger.Fatal("No url provided") - } - - if raw["endpoint"] != nil { - parsed.Endpoint = raw["endpoint"].(string) - } - - if raw["insecure"] != nil { - parsed.IgnoreCert = raw["insecure"].(bool) - if parsed.IgnoreCert { - Logger.Warn("Ignoring SSL certificates") - } - } - - if raw["batchsize"] != nil { - parsed.BatchSize = int64(raw["batchsize"].(int)) - } else { - parsed.BatchSize = 1 - } - - if raw["batchtimeout"] != nil { - parsed.BatchTimeout = time.Duration(raw["batchtimeout"].(int)) * time.Millisecond - } - - if raw["deflate"] != nil { - parsed.Deflate = raw["deflate"].(bool) - } - - if raw["showcounter"] != nil { - parsed.ShowCounter = raw["showcounter"].(int) - } - - return -} diff --git a/senders/httpsender/sender.go b/senders/httpsender/sender.go deleted file mode 100644 index 3159626..0000000 --- a/senders/httpsender/sender.go +++ /dev/null @@ -1,225 +0,0 @@ -package httpsender - -import ( - "bufio" - "bytes" - "compress/zlib" - "crypto/tls" - "io" - "net/http" - "sync" - "time" - - "github.com/Sirupsen/logrus" - "github.com/redBorder/rbforwarder" -) - -const ( - errRequest = 101 - errStatus = 102 - errHTTP = 103 -) - -// Sender receives data from pipe and send it via HTTP to an endpoint -type Sender struct { - id int - client *http.Client - batchBuffer map[string]*batchBuffer - - // Statistics - counter int64 - timer *time.Timer - - // Configuration - logger *logrus.Entry - config config -} - -type batchBuffer struct { - buff *bytes.Buffer - writer io.Writer - timer *time.Timer - mutex *sync.Mutex - messageCount int64 - messages []*rbforwarder.Message -} - -type config struct { - URL string - Endpoint string - IgnoreCert bool - Deflate bool - ShowCounter int - BatchSize int64 - BatchTimeout time.Duration -} - -// Init initializes an HTTP sender -func (s *Sender) Init(id int) error { - s.id = id - - // Create the client object. Useful for skipping SSL verify - tr := &http.Transport{} - if s.config.IgnoreCert { - tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - s.client = &http.Client{Transport: tr} - - // A map to store buffers for each endpoint - s.batchBuffer = make(map[string]*batchBuffer) - - // Show the messages per second every "showCounter" seconds - if s.config.ShowCounter > 0 { - go func() { - for { - timer := time.NewTimer(time.Duration(s.config.ShowCounter) * time.Second) - <-timer.C - s.logger.WithField("worker", s.id).Infof("Messages per second %d", s.counter/int64(s.config.ShowCounter)) - s.counter = 0 - } - }() - } - - return nil -} - -// Send stores a message received from the pipeline into a buffer to perform -// batching. -func (s *Sender) Send(message *rbforwarder.Message) error { - - // logger.Printf("[%d] Sending message ID: [%d]", s.id, message) - - // We can send batch only for messages with the same path - var path string - - if message.Metadata[s.config.Endpoint] != nil { - path = message.Metadata[s.config.Endpoint].(string) - } - - // Initialize buffer for path - if _, exists := s.batchBuffer[path]; !exists { - s.batchBuffer[path] = &batchBuffer{ - mutex: &sync.Mutex{}, - messageCount: 0, - buff: new(bytes.Buffer), - timer: time.NewTimer(s.config.BatchTimeout), - } - - if s.config.Deflate { - s.batchBuffer[path].writer = zlib.NewWriter(s.batchBuffer[path].buff) - } else { - s.batchBuffer[path].writer = bufio.NewWriter(s.batchBuffer[path].buff) - } - - // A go rutine for send all the messages stored on the buffer when a timeout - // occurred - if s.config.BatchTimeout != 0 { - go func() { - for { - <-s.batchBuffer[path].timer.C - s.batchBuffer[path].mutex.Lock() - if s.batchBuffer[path].messageCount > 0 { - s.batchSend(s.batchBuffer[path], path) - } - s.batchBuffer[path].mutex.Unlock() - s.batchBuffer[path].timer.Reset(s.config.BatchTimeout) - } - }() - } - } - - // Once the buffer is created, it's necessary to lock so a new message can't be - // writed to buffer meanwhile the timeout go rutine is sending a request - batchBuffer := s.batchBuffer[path] - batchBuffer.mutex.Lock() - - // Write the new message to the buffer and increase the number of messages in - // the buffer - if _, err := batchBuffer.writer.Write(message.OutputBuffer.Bytes()); err != nil { - s.logger.Error(err) - } - batchBuffer.messages = append(batchBuffer.messages, message) - batchBuffer.messageCount++ - - // Flush writers - if s.config.Deflate { - batchBuffer.writer.(*zlib.Writer).Flush() - } else { - batchBuffer.writer.(*bufio.Writer).Flush() - } - - // If there are enough messages on buffer it's time to send the POST - if batchBuffer.messageCount >= s.config.BatchSize { - s.batchSend(batchBuffer, path) - } - batchBuffer.mutex.Unlock() - - return nil -} - -func (s *Sender) batchSend(batchBuffer *batchBuffer, path string) { - - // Reset buffer and clear message counter - defer func() { - batchBuffer.messageCount = 0 - batchBuffer.buff = new(bytes.Buffer) - - // Reset writers - if s.config.Deflate { - batchBuffer.writer = zlib.NewWriter(batchBuffer.buff) - } else { - batchBuffer.writer = bufio.NewWriter(batchBuffer.buff) - } - - // Reset timeout timer - batchBuffer.timer.Reset(s.config.BatchTimeout) - batchBuffer.messages = nil - }() - - // Stop the timeout timer - batchBuffer.timer.Stop() - - // Make sure the writer is closed - if s.config.Deflate { - batchBuffer.writer.(*zlib.Writer).Close() - } - - // Create the HTTP POST request - req, err := http.NewRequest("POST", s.config.URL+"/"+path, batchBuffer.buff) - if err != nil { - s.logger.Errorf("Error creating request: %s", err.Error()) - for _, message := range batchBuffer.messages { - message.Report(errRequest, err.Error()) - } - return - } - - // Use proper header for sending deflate - if s.config.Deflate { - req.Header.Add("Content-Encoding", "deflate") - } - - // Send the HTTP POST request - res, err := s.client.Do(req) - if err != nil { - for _, message := range batchBuffer.messages { - message.Report(errHTTP, err.Error()) - } - return - } - defer res.Body.Close() - - // Send the reports - if res.StatusCode >= 400 { - for _, message := range batchBuffer.messages { - message.Report(errStatus, res.Status) - } - } else { - for _, message := range batchBuffer.messages { - message.Report(0, res.Status) - } - } - - // Statistics - s.counter += batchBuffer.messageCount -} diff --git a/senders/httpsender/sender_test.go b/senders/httpsender/sender_test.go deleted file mode 100644 index adbedc5..0000000 --- a/senders/httpsender/sender_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package httpsender - -import ( - "net/http" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestInitSuccess(t *testing.T) { - httpSender := new(Sender) - httpSender.config.ShowCounter = 1 - httpSender.config.IgnoreCert = true - - err := httpSender.Init(0) - - assert.NoError(t, err) - assert.Equal(t, 0, httpSender.id) - assert.NotNil(t, httpSender.batchBuffer) - assert.True(t, httpSender.client.Transport.(*http.Transport). - TLSClientConfig.InsecureSkipVerify) -} diff --git a/types/composer.go b/types/composer.go new file mode 100644 index 0000000..e0bd721 --- /dev/null +++ b/types/composer.go @@ -0,0 +1,17 @@ +package types + +// Next should be called by a component in order to pass the message to the next +// component in the pipeline. +type Next func(Messenger) + +// Done should be called by a component in order to return the message to the +// message handler. Can be used by the last component to inform that the +// message processing is done o by a middle component to inform an error. +type Done func(Messenger, int, string) + +// Composer represents a component in the pipeline that performs a work on +// a message +type Composer interface { + Init(int) error + OnMessage(Messenger, Next, Done) +} diff --git a/types/messenger.go b/types/messenger.go new file mode 100644 index 0000000..5646523 --- /dev/null +++ b/types/messenger.go @@ -0,0 +1,8 @@ +package types + +// Messenger is used by modules to handle messages +type Messenger interface { + PopData() ([]byte, error) + PushData(data []byte) + GetOpt(name string) interface{} +} From 809055a26ba9c1151989c310a7909e0e69dde863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 2 Aug 2016 11:07:23 +0200 Subject: [PATCH 02/36] Update README.md --- README.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4345255..d0450cd 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ ![](https://img.shields.io/packagist/l/doctrine/orm.svg?maxAge=2592000) -[![](https://travis-ci.org/redBorder/rbforwarder.svg?branch=master)](https://travis-ci.org/redBorder/rbforwarder) +[![](https://travis-ci.org/redBorder/rbforwarder.svg?branch=develop)](https://travis-ci.org/redBorder/rbforwarder) [![](https://goreportcard.com/badge/github.com/redBorder/rbforwarder)](https://goreportcard.com/report/github.com/redBorder/rbforwarder) -[![](https://coveralls.io/repos/github/redBorder/rbforwarder/badge.svg?branch=master)](https://coveralls.io/github/redBorder/rbforwarder?branch=master) +[![](https://coveralls.io/repos/github/redBorder/rbforwarder/badge.svg?branch=develop)](https://coveralls.io/github/redBorder/rbforwarder?branch=develop) [![](https://godoc.org/github.com/redBorder/rbforwarder?status.svg)](https://godoc.org/github.com/redBorder/rbforwarder) # rbforwarder **rbforwarder** is an extensible, protocol agnostic and easy to use tool for -process data. It allows you to create a custom pipeline in a modular way. +processing data asynchronously. It allows you to create a custom pipeline in +a modular fashion. For example, you can read data from a Kafka broker and use **rbforwarder** to -build a **pipeline** that decodes the JSON, filter or add some fields, encodes -the data again to JSON and send it using using multiple protocols HTTP, MQTT, -AMQP, etc. It's easy to write a **component** for the pipeline. +build a **pipeline** to decodes the JSON, filter or add fields, encode +the data again to JSON and send it using using multiple protocols `HTTP`, +`MQTT`, `AMQP`, etc. It's easy to write a pipeline **component**. -## Features +## Built-in features -- Support for multiple workers for every **component**. -- Support **buffer pooling**, for fine-grained memory control. -- Asynchronous report system. Different gorutine for send and receive responses. -- Built-in retry. The **rbforwarder** can retry messages on fail. +- Support multiple **workers** for each components. +- Support **buffer pooling** for memory recycling. +- Asynchronous report system. Get responses on a separate gorutine. +- Built-in message retrying. The **rbforwarder** can retry messages on fail. +- Instrumentation to have an idea of how it is performing the application. ## Components - Send data to an endpoint - MQTT - HTTP - - Fast HTTP - Kafka - Decoders / Encoders - JSON - Utility - Limiter - - Metrics ## Road Map -_The application is being on hard developing, breaking changes expected._ +_The application is on hard development, breaking changes expected until 1.0._ |Milestone | Feature | Status | |----------|---------------------|-----------| -| 0.1 | Pipeline | _Testing_ | -| 0.2 | Reporter | _Testing_ | -| 0.3 | Limiter | _Testing_ | -| 0.4 | HTTP component | _Testing_ | -| 0.5 | MQTT component | _Pending_ | -| 0.6 | JSON component | _Pending_ | -| 0.7 | Kafka component | _Pending_ | -| 0.8 | Metrics component | _Pending_ | -| 0.9 | Benchmarks | _Pending_ | -| 1.0 | Stable | _Pending_ | +| 0.1 | Pipeline builder | Done | +| 0.2 | Reporter | Done | +| 0.3 | Buffer pool | _Pending_ | +| 0.4 | Batcher component | _Pending_ | +| 0.5 | Limiter component | _Pending_ | +| 0.6 | Instrumentation | _Pending_ | +| 0.7 | HTTP component | _Pending_ | +| 0.8 | JSON component | _Pending_ | +| 0.9 | MQTT component | _Pending_ | +| 1.0 | Kafka component | _Pending_ | From 32af15d0efcc25712e5dc6e6acd6281001a0994b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 13:58:26 +0200 Subject: [PATCH 03/36] :sparkles: Implemented multiple reports per message (#15) Closes #11 --- message.go | 47 +++++++++++++++++++++++++-------------------- rbforwarder.go | 5 ++++- rbforwarder_test.go | 40 +++++++------------------------------- reporter.go | 38 +++++++++++++++++++----------------- types/messenger.go | 1 - 5 files changed, 57 insertions(+), 74 deletions(-) diff --git a/message.go b/message.go index d2d79ba..8fd428d 100644 --- a/message.go +++ b/message.go @@ -1,47 +1,52 @@ package rbforwarder -import "errors" +import ( + "errors" + + "github.com/oleiade/lane" +) // message is used to send data through the pipeline type message struct { - bufferStack [][]byte - + payload *lane.Stack + opts *lane.Stack seq uint64 // Unique ID for the report, used to maintain sequence status string // Result of the sending code int // Result of the sending retries int - opts map[string]interface{} - channel chan *message } // PushData store data on an LIFO queue so the nexts handlers can use it func (m *message) PushData(v []byte) { - m.bufferStack = append(m.bufferStack, v) + if m.payload == nil { + m.payload = lane.NewStack() + } + + m.payload.Push(v) } // PopData get the data stored by the previous handler func (m *message) PopData() (ret []byte, err error) { - if len(m.bufferStack) < 1 { - err = errors.New("No data on the stack") + if m.payload.Empty() { + err = errors.New("Empty stack") return } - - ret = m.bufferStack[len(m.bufferStack)-1] - m.bufferStack = m.bufferStack[0 : len(m.bufferStack)-1] + ret = m.payload.Pop().([]byte) return } -// GetOpt returns an option -func (m message) GetOpt(name string) interface{} { - return m.opts[name] -} +func (m message) GetReports() []Report { + var reports []Report -func (m message) GetReport() Report { - return Report{ - code: m.code, - status: m.status, - retries: m.retries, - opts: m.opts, + for !m.opts.Empty() { + reports = append(reports, Report{ + code: m.code, + status: m.status, + retries: m.retries, + opts: m.opts.Pop().(map[string]interface{}), + }) } + + return reports } diff --git a/rbforwarder.go b/rbforwarder.go index 4407156..f0ca30b 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -5,6 +5,7 @@ import ( "sync/atomic" "github.com/Sirupsen/logrus" + "github.com/oleiade/lane" "github.com/redBorder/rbforwarder/types" ) @@ -91,10 +92,12 @@ func (f *RBForwarder) Produce(buf []byte, options map[string]interface{}) error message := &message{ seq: seq, - opts: options, + opts: lane.NewStack(), } message.PushData(buf) + message.opts.Push(options) + f.p.input <- message return nil diff --git a/rbforwarder_test.go b/rbforwarder_test.go index 26ad30a..cff3473 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -85,18 +85,8 @@ func TestRBForwarder(t *testing.T) { Convey("When a \"Hello World\" message is produced", func() { component.status = "OK" component.statusCode = 0 - closed := false - component.On("OnMessage", mock.MatchedBy(func(m *message) bool { - opt := m.GetOpt("message_id") - - if !closed { - rbforwarder.Close() - closed = true - } - - return opt.(string) == "test123" - })).Times(1) + component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")).Times(1) err := rbforwarder.Produce( []byte("Hello World"), @@ -109,6 +99,7 @@ func TestRBForwarder(t *testing.T) { for report := range rbforwarder.GetReports() { reports++ lastReport = report + rbforwarder.Close() } So(lastReport, ShouldNotBeNil) @@ -116,7 +107,6 @@ func TestRBForwarder(t *testing.T) { So(lastReport.opts["message_id"], ShouldEqual, "test123") So(lastReport.code, ShouldEqual, 0) So(lastReport.status, ShouldEqual, "OK") - So(err, ShouldBeNil) component.AssertExpectations(t) @@ -199,25 +189,15 @@ func TestRBForwarder(t *testing.T) { Convey("When a message fails to send", func() { component.status = "Fake Error" component.statusCode = 99 - closed := false - component.On("OnMessage", mock.MatchedBy(func(m *message) bool { - opt := m.GetOpt("message_id") - - if m.retries >= 3 && !closed { - closed = true - rbforwarder.Close() - } - - return opt.(string) == "test123" - })).Times(4) + component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")).Times(4) err := rbforwarder.Produce( []byte("Hello World"), map[string]interface{}{"message_id": "test123"}, ) - Convey("The message should be retried\n", func() { + Convey("The message should be retried", func() { So(err, ShouldBeNil) var reports int @@ -225,6 +205,7 @@ func TestRBForwarder(t *testing.T) { for report := range rbforwarder.GetReports() { reports++ lastReport = report + rbforwarder.Close() } So(lastReport, ShouldNotBeNil) @@ -328,15 +309,8 @@ func TestRBForwarder(t *testing.T) { component2.status = "OK" component2.statusCode = 0 - component1.On("OnMessage", mock.MatchedBy(func(m *message) bool { - opt := m.GetOpt("message_id") - return opt.(string) == "test123" - })) - - component2.On("OnMessage", mock.MatchedBy(func(m *message) bool { - opt := m.GetOpt("message_id") - return opt.(string) == "test123" - })) + component1.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) + component2.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) err := rbforwarder.Produce( []byte("Hello World"), diff --git a/reporter.go b/reporter.go index df940fe..64d3d83 100644 --- a/reporter.go +++ b/reporter.go @@ -80,7 +80,9 @@ func (r *reporter) GetReports() chan Report { go func() { for message := range r.out { - reports <- message.GetReport() + for _, report := range message.GetReports() { + reports <- report + } } close(reports) @@ -94,25 +96,25 @@ func (r *reporter) GetOrderedReports() chan Report { go func() { for message := range r.out { - report := message.GetReport() - - if message.seq == r.currentReport { - // The message is the expected. Send it. - reports <- report - r.currentReport++ - } else { - // This message is not the expected. Store it. - r.queued[message.seq] = report - } - - // Check if there are stored messages and send them. - for { - if currentReport, ok := r.queued[r.currentReport]; ok { - reports <- currentReport - delete(r.queued, r.currentReport) + for _, report := range message.GetReports() { + if message.seq == r.currentReport { + // The message is the expected. Send it. + reports <- report r.currentReport++ } else { - break + // This message is not the expected. Store it. + r.queued[message.seq] = report + } + + // Check if there are stored messages and send them. + for { + if currentReport, ok := r.queued[r.currentReport]; ok { + reports <- currentReport + delete(r.queued, r.currentReport) + r.currentReport++ + } else { + break + } } } } diff --git a/types/messenger.go b/types/messenger.go index 5646523..c8a6d2e 100644 --- a/types/messenger.go +++ b/types/messenger.go @@ -4,5 +4,4 @@ package types type Messenger interface { PopData() ([]byte, error) PushData(data []byte) - GetOpt(name string) interface{} } From d86acde413d8573ade841c776fb1fc3f18938af4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez?= Date: Wed, 3 Aug 2016 14:55:43 +0200 Subject: [PATCH 04/36] :sparkles: Remove PushData() and add PopOpts() methods (#17) Closes #16 --- message.go | 11 +++++++++++ rbforwarder_test.go | 3 ++- types/messenger.go | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/message.go b/message.go index 8fd428d..2d65471 100644 --- a/message.go +++ b/message.go @@ -36,6 +36,17 @@ func (m *message) PopData() (ret []byte, err error) { return } +// PopData get the data stored by the previous handler +func (m *message) PopOpts() (ret map[string]interface{}, err error) { + if m.opts.Empty() { + err = errors.New("Empty stack") + return + } + ret = m.opts.Pop().(map[string]interface{}) + + return +} + func (m message) GetReports() []Report { var reports []Report diff --git a/rbforwarder_test.go b/rbforwarder_test.go index cff3473..1e68dab 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -25,7 +25,8 @@ func (c *MockMiddleComponent) OnMessage( c.Called(m) data, _ := m.PopData() processedData := "-> [" + string(data) + "] <-" - m.PushData([]byte(processedData)) + message := m.(*message) + message.PushData([]byte(processedData)) next(m) } diff --git a/types/messenger.go b/types/messenger.go index c8a6d2e..59c4e4c 100644 --- a/types/messenger.go +++ b/types/messenger.go @@ -3,5 +3,5 @@ package types // Messenger is used by modules to handle messages type Messenger interface { PopData() ([]byte, error) - PushData(data []byte) + PopOpts() (map[string]interface{}, error) } From cbbb3ffc0ce489c5fec7a9bf9c1b353b60d0c3ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Thu, 4 Aug 2016 12:00:27 +0200 Subject: [PATCH 05/36] :sparkles: Make rbforwarder.Report an interface --- message.go | 21 ++++++++++++++++----- rbforwarder.go | 6 +++--- rbforwarder_test.go | 31 ++++++++++++++++--------------- report.go | 11 +++++++++-- reporter.go => reportHandler.go | 28 +++++++++++++++------------- types/messenger.go | 1 + types/reporter.go | 9 +++++++++ 7 files changed, 69 insertions(+), 38 deletions(-) rename reporter.go => reportHandler.go (77%) create mode 100644 types/reporter.go diff --git a/message.go b/message.go index 2d65471..b7a4f1f 100644 --- a/message.go +++ b/message.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/oleiade/lane" + "github.com/redBorder/rbforwarder/types" ) // message is used to send data through the pipeline @@ -27,31 +28,41 @@ func (m *message) PushData(v []byte) { // PopData get the data stored by the previous handler func (m *message) PopData() (ret []byte, err error) { + if m.payload == nil { + err = errors.New("Uninitialized payload") + return + } + if m.payload.Empty() { err = errors.New("Empty stack") return } - ret = m.payload.Pop().([]byte) + ret = m.payload.Pop().([]byte) return } // PopData get the data stored by the previous handler func (m *message) PopOpts() (ret map[string]interface{}, err error) { + if m.opts == nil { + err = errors.New("Uninitialized options") + return + } + if m.opts.Empty() { err = errors.New("Empty stack") return } - ret = m.opts.Pop().(map[string]interface{}) + ret = m.opts.Pop().(map[string]interface{}) return } -func (m message) GetReports() []Report { - var reports []Report +func (m message) Reports() []types.Reporter { + var reports []types.Reporter for !m.opts.Empty() { - reports = append(reports, Report{ + reports = append(reports, report{ code: m.code, status: m.status, retries: m.retries, diff --git a/rbforwarder.go b/rbforwarder.go index f0ca30b..eaf2422 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -22,7 +22,7 @@ var Logger = logrus.NewEntry(log) // workers type RBForwarder struct { p *pipeline - r *reporter + r *reportHandler currentProducedID uint64 working uint32 @@ -71,13 +71,13 @@ func (f *RBForwarder) PushComponents(components []types.Composer, w []int) { // GetReports is used by the source to get a report for a sent message. // Reports are delivered on the same order that was sent -func (f *RBForwarder) GetReports() <-chan Report { +func (f *RBForwarder) GetReports() <-chan types.Reporter { return f.r.GetReports() } // GetOrderedReports is the same as GetReports() but the reports are delivered // in order -func (f *RBForwarder) GetOrderedReports() <-chan Report { +func (f *RBForwarder) GetOrderedReports() <-chan types.Reporter { return f.r.GetOrderedReports() } diff --git a/rbforwarder_test.go b/rbforwarder_test.go index 1e68dab..927e3b7 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -95,11 +95,11 @@ func TestRBForwarder(t *testing.T) { ) Convey("\"Hello World\" message should be get by the worker", func() { - var lastReport Report + var lastReport report var reports int - for report := range rbforwarder.GetReports() { + for r := range rbforwarder.GetReports() { reports++ - lastReport = report + lastReport = r.(report) rbforwarder.Close() } @@ -143,8 +143,8 @@ func TestRBForwarder(t *testing.T) { Convey("Should be possible to read an option", func() { for report := range rbforwarder.GetReports() { - So(report.opts, ShouldNotBeNil) - So(report.opts["option"], ShouldEqual, "example_option") + So(report.GetOpts(), ShouldNotBeNil) + So(report.GetOpts()["option"], ShouldEqual, "example_option") } So(err, ShouldBeNil) @@ -153,8 +153,8 @@ func TestRBForwarder(t *testing.T) { Convey("Should not be possible to read an nonexistent option", func() { for report := range rbforwarder.GetReports() { - So(report.opts, ShouldNotBeNil) - So(report.opts["invalid"], ShouldBeEmpty) + So(report.GetOpts(), ShouldNotBeNil) + So(report.GetOpts()["invalid"], ShouldBeEmpty) } So(err, ShouldBeNil) @@ -177,7 +177,7 @@ func TestRBForwarder(t *testing.T) { Convey("Should not be possible to read the option", func() { for report := range rbforwarder.GetReports() { So(err, ShouldBeNil) - So(report.opts, ShouldBeNil) + So(report.GetOpts(), ShouldBeNil) } So(err, ShouldBeNil) @@ -202,10 +202,10 @@ func TestRBForwarder(t *testing.T) { So(err, ShouldBeNil) var reports int - var lastReport Report - for report := range rbforwarder.GetReports() { + var lastReport report + for r := range rbforwarder.GetReports() { reports++ - lastReport = report + lastReport = r.(report) rbforwarder.Close() } @@ -257,7 +257,7 @@ func TestRBForwarder(t *testing.T) { var reports int for report := range rbforwarder.GetOrderedReports() { - if report.opts["message_id"] != reports { + if report.GetOpts()["message_id"] != reports { ordered = false } reports++ @@ -325,9 +325,10 @@ func TestRBForwarder(t *testing.T) { for report := range rbforwarder.GetReports() { reports++ - So(report.opts["message_id"], ShouldEqual, "test123") - So(report.code, ShouldEqual, 0) - So(report.status, ShouldEqual, "OK") + So(report.GetOpts()["message_id"], ShouldEqual, "test123") + code, status, _ := report.Status() + So(code, ShouldEqual, 0) + So(status, ShouldEqual, "OK") } m := <-component2.channel diff --git a/report.go b/report.go index 783a0ff..8df6dc3 100644 --- a/report.go +++ b/report.go @@ -1,9 +1,16 @@ package rbforwarder -// Report contains information about a produced message -type Report struct { +type report struct { code int status string retries int opts map[string]interface{} } + +func (r report) Status() (code int, status string, retries int) { + return r.code, r.status, r.retries +} + +func (r report) GetOpts() map[string]interface{} { + return r.opts +} diff --git a/reporter.go b/reportHandler.go similarity index 77% rename from reporter.go rename to reportHandler.go index 64d3d83..b105b58 100644 --- a/reporter.go +++ b/reportHandler.go @@ -3,18 +3,20 @@ package rbforwarder import ( "sync" "time" + + "github.com/redBorder/rbforwarder/types" ) -// reporter is used to handle the reports produced by the last element +// reportHandler is used to handle the reports produced by the last element // of the pipeline. The first element of the pipeline can know the status // of the produced message using GetReports() or GetOrderedReports() -type reporter struct { +type reportHandler struct { input chan *message // Receive messages from pipeline retries chan *message // Send messages back to the pipeline out chan *message // Send reports to the user - queued map[uint64]Report // Store pending reports - currentReport uint64 // Last delivered report + queued map[uint64]types.Reporter // Store pending reports + currentReport uint64 // Last delivered report maxRetries int backoff int @@ -26,14 +28,14 @@ type reporter struct { func newReporter( maxRetries, backoff int, input, retries chan *message, -) *reporter { +) *reportHandler { - r := &reporter{ + r := &reportHandler{ input: input, retries: retries, out: make(chan *message, 100), // NOTE Temp channel size - queued: make(map[uint64]Report), + queued: make(map[uint64]types.Reporter), maxRetries: maxRetries, backoff: backoff, @@ -75,12 +77,12 @@ func newReporter( return r } -func (r *reporter) GetReports() chan Report { - reports := make(chan Report) +func (r *reportHandler) GetReports() chan types.Reporter { + reports := make(chan types.Reporter) go func() { for message := range r.out { - for _, report := range message.GetReports() { + for _, report := range message.Reports() { reports <- report } } @@ -91,12 +93,12 @@ func (r *reporter) GetReports() chan Report { return reports } -func (r *reporter) GetOrderedReports() chan Report { - reports := make(chan Report) +func (r *reportHandler) GetOrderedReports() chan types.Reporter { + reports := make(chan types.Reporter) go func() { for message := range r.out { - for _, report := range message.GetReports() { + for _, report := range message.Reports() { if message.seq == r.currentReport { // The message is the expected. Send it. reports <- report diff --git a/types/messenger.go b/types/messenger.go index 59c4e4c..7b01b55 100644 --- a/types/messenger.go +++ b/types/messenger.go @@ -4,4 +4,5 @@ package types type Messenger interface { PopData() ([]byte, error) PopOpts() (map[string]interface{}, error) + Reports() []Reporter } diff --git a/types/reporter.go b/types/reporter.go new file mode 100644 index 0000000..a670f5d --- /dev/null +++ b/types/reporter.go @@ -0,0 +1,9 @@ +package types + +// Reporter returns information about a processed message. The GetOpts method +// should return, at least, the same info that was pushed to the original +// message from the user. +type Reporter interface { + Status() (code int, status string, retries int) + GetOpts() map[string]interface{} +} From fdca418a4700be5cceda9cdd73a30659051101d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 3 Aug 2016 15:06:26 +0200 Subject: [PATCH 06/36] :sparkles: Add batcher component Closes #10 --- components/batcher/batch.go | 75 +++++++++++ components/batcher/batcher.go | 100 +++++++++++++++ components/batcher/batcher_test.go | 196 +++++++++++++++++++++++++++++ components/batcher/config.go | 8 ++ 4 files changed, 379 insertions(+) create mode 100644 components/batcher/batch.go create mode 100644 components/batcher/batcher.go create mode 100644 components/batcher/batcher_test.go create mode 100644 components/batcher/config.go diff --git a/components/batcher/batch.go b/components/batcher/batch.go new file mode 100644 index 0000000..ef50eb8 --- /dev/null +++ b/components/batcher/batch.go @@ -0,0 +1,75 @@ +package batcher + +import ( + "bytes" + "errors" + "time" + + "github.com/benbjohnson/clock" + "github.com/oleiade/lane" + "github.com/redBorder/rbforwarder/types" +) + +// BatchMessage contains multiple messages +type BatchMessage struct { + Group string // Name used to group messages + Buff *bytes.Buffer // Buffer for merge multiple messages + MessageCount uint // Current number of messages in the buffer + BytesCount uint // Current number of bytes in the buffer + Next types.Next // Call to pass the message to the next handler + Opts *lane.Stack // Original messages options +} + +// StartTimeout initializes a timeout used to send the messages when expires. No +// matters how many messages are in the buffer. +func (b *BatchMessage) StartTimeout(clk clock.Clock, timeoutMillis uint, ready chan *BatchMessage) *BatchMessage { + if clk == nil { + clk = clock.New() + } + + if timeoutMillis != 0 { + timer := clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) + + go func() { + <-timer.C + if b.MessageCount > 0 { + ready <- b + } + }() + } + return b +} + +// Send the batch of messages to the next handler in the pipeline +func (b *BatchMessage) Send(cb func()) { + cb() + b.Next(b) +} + +// Write merges a new message in the buffer +func (b *BatchMessage) Write(data []byte) { + b.Buff.Write(data) + b.MessageCount++ +} + +// PopData returns the buffer with all the messages merged +func (b *BatchMessage) PopData() (ret []byte, err error) { + return b.Buff.Bytes(), nil +} + +// PopOpts get the data stored by the previous handler +func (b *BatchMessage) PopOpts() (ret map[string]interface{}, err error) { + if b.Opts.Empty() { + err = errors.New("Empty stack") + return + } + + ret = b.Opts.Pop().(map[string]interface{}) + + return +} + +// Reports do nothing +func (b *BatchMessage) Reports() []types.Reporter { + return nil +} diff --git a/components/batcher/batcher.go b/components/batcher/batcher.go new file mode 100644 index 0000000..1303d41 --- /dev/null +++ b/components/batcher/batcher.go @@ -0,0 +1,100 @@ +package batcher + +import ( + "bytes" + "sync" + + "github.com/benbjohnson/clock" + "github.com/oleiade/lane" + "github.com/redBorder/rbforwarder/types" +) + +// Batcher allows to merge multiple messages in a single one +type Batcher struct { + id int // Worker ID + batches map[string]*BatchMessage // Collection of batches pending + newBatches chan *BatchMessage // Send messages to sender gorutine + pendingBatches chan *BatchMessage // Send messages to sender gorutine + wg sync.WaitGroup + clk clock.Clock + + config Config // Batcher configuration +} + +// Init starts a gorutine that can receive: +// - New messages that will be added to a existing or new batch of messages +// - A batch of messages that is ready to send (i.e. batch timeout has expired) +func (b *Batcher) Init(id int) { + b.batches = make(map[string]*BatchMessage) + b.newBatches = make(chan *BatchMessage) + b.pendingBatches = make(chan *BatchMessage) + + readyBatches := make(chan *BatchMessage) + + if b.clk == nil { + b.clk = clock.New() + } + + go func() { + for { + select { + case batchMessage := <-readyBatches: + batchMessage.Send(func() { + delete(b.batches, batchMessage.Group) + }) + + case batchMessage := <-b.newBatches: + batchMessage.StartTimeout(b.clk, b.config.TimeoutMillis, readyBatches) + b.batches[batchMessage.Group] = batchMessage + b.wg.Done() + + case batchMessage := <-b.pendingBatches: + opts, err := batchMessage.PopOpts() + if err != nil { + break + } + + b.batches[batchMessage.Group].Opts.Push(opts) + b.batches[batchMessage.Group].Write(batchMessage.Buff.Bytes()) + + if b.batches[batchMessage.Group].MessageCount >= b.config.Limit { + b.batches[batchMessage.Group].Send(func() { + delete(b.batches, batchMessage.Group) + }) + } + + b.wg.Done() + } + } + }() +} + +// OnMessage is called when a new message is receive. Add the new message to +// a batch +func (b *Batcher) OnMessage(m types.Messenger, next types.Next, done types.Done) { + opts, _ := m.PopOpts() + group, ok := opts["batch_group"].(string) + if !ok { + next(m) + } + + data, _ := m.PopData() + batchMessage := &BatchMessage{ + Buff: bytes.NewBuffer(data), + Opts: lane.NewStack(), + MessageCount: 1, + Next: next, + Group: group, + } + + batchMessage.Opts.Push(opts) + + b.wg.Add(1) + if _, exists := b.batches[group]; exists { + b.pendingBatches <- batchMessage + } else { + b.newBatches <- batchMessage + } + + b.wg.Wait() +} diff --git a/components/batcher/batcher_test.go b/components/batcher/batcher_test.go new file mode 100644 index 0000000..a98ed89 --- /dev/null +++ b/components/batcher/batcher_test.go @@ -0,0 +1,196 @@ +package batcher + +import ( + "testing" + "time" + + "github.com/benbjohnson/clock" + "github.com/redBorder/rbforwarder/types" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/mock" +) + +type NexterDoner struct { + mock.Mock + nextCalled chan struct{} +} + +func (nd *NexterDoner) Next(m types.Messenger) { + nd.nextCalled <- struct{}{} + nd.Called(m) +} + +func (nd *NexterDoner) Done(m types.Messenger, code int, status string) { + nd.Called(m, code, status) +} + +type TestMessage struct { + mock.Mock +} + +func (m *TestMessage) PopData() (data []byte, err error) { + args := m.Called() + + return args.Get(0).([]byte), args.Error(1) +} + +func (m *TestMessage) PopOpts() (opts map[string]interface{}, err error) { + args := m.Called() + + return args.Get(0).(map[string]interface{}), args.Error(1) +} + +func (m *TestMessage) Reports() []types.Reporter { + return nil +} + +func TestRBForwarder(t *testing.T) { + Convey("Given a batcher", t, func() { + batcher := &Batcher{config: Config{ + TimeoutMillis: 1000, + Limit: 10, + MaxPendingBatches: 10, + }, + clk: clock.NewMock(), + } + + batcher.Init(0) + + Convey("When a message is received, but not yet sent", func() { + m := new(TestMessage) + m.On("PopData").Return([]byte("Hello World"), nil) + m.On("PopOpts").Return(map[string]interface{}{ + "batch_group": "group1", + }, nil) + + batcher.OnMessage(m, nil, nil) + + Convey("Message should be present on the batch", func() { + var data *BatchMessage + var exists bool + data, exists = batcher.batches["group1"] + + So(exists, ShouldBeTrue) + So(string(data.Buff.Bytes()), ShouldEqual, "Hello World") + So(len(batcher.batches), ShouldEqual, 1) + + m.AssertExpectations(t) + }) + }) + + Convey("When the max number of messages is reached", func() { + m := new(TestMessage) + m.On("PopData").Return([]byte("ABC"), nil) + m.On("PopOpts").Return(map[string]interface{}{ + "batch_group": "group1", + }, nil) + + nd := new(NexterDoner) + nd.nextCalled = make(chan struct{}, 1) + nd.On("Next", mock.MatchedBy(func(m *BatchMessage) bool { + data, _ := m.PopData() + return string(data) == "ABCABCABCABCABCABCABCABCABCABC" + })).Times(1) + + for i := 0; i < int(batcher.config.Limit); i++ { + batcher.OnMessage(m, nd.Next, nil) + } + + Convey("The batch should be sent", func() { + nd.AssertExpectations(t) + <-nd.nextCalled + So(batcher.batches["group1"], ShouldBeNil) + So(len(batcher.batches), ShouldEqual, 0) + }) + }) + + Convey("When the timeout expires", func() { + m := new(TestMessage) + m.On("PopData").Return([]byte("ABC"), nil) + m.On("PopOpts").Return(map[string]interface{}{ + "batch_group": "group1", + }, nil) + + nd := new(NexterDoner) + nd.nextCalled = make(chan struct{}, 1) + nd.On("Next", mock.MatchedBy(func(m *BatchMessage) bool { + data, _ := m.PopData() + return string(data) == "ABCABCABCABCABC" + })).Times(1) + + for i := 0; i < 5; i++ { + batcher.OnMessage(m, nd.Next, nil) + } + + clk := batcher.clk.(*clock.Mock) + + Convey("The batch should be sent", func() { + clk.Add(500 * time.Millisecond) + So(batcher.batches["group1"], ShouldNotBeNil) + clk.Add(500 * time.Millisecond) + <-nd.nextCalled + So(batcher.batches["group1"], ShouldBeNil) + So(len(batcher.batches), ShouldEqual, 0) + nd.AssertExpectations(t) + }) + }) + + Convey("When multiple messages are received with differente groups", func() { + m1 := new(TestMessage) + m1.On("PopData").Return([]byte("MESSAGE 1"), nil) + m1.On("PopOpts").Return(map[string]interface{}{ + "batch_group": "group1", + }, nil) + + m2 := new(TestMessage) + m2.On("PopData").Return([]byte("MESSAGE 2"), nil) + m2.On("PopOpts").Return(map[string]interface{}{ + "batch_group": "group2", + }, nil) + + m3 := new(TestMessage) + m3.On("PopData").Return([]byte("MESSAGE 3"), nil) + m3.On("PopOpts").Return(map[string]interface{}{ + "batch_group": "group2", + }, nil) + + nd := new(NexterDoner) + nd.nextCalled = make(chan struct{}, 2) + nd.On("Next", mock.AnythingOfType("*batcher.BatchMessage")).Times(2) + + for i := 0; i < 3; i++ { + batcher.OnMessage(m1, nd.Next, nil) + } + for i := 0; i < 3; i++ { + batcher.OnMessage(m2, nd.Next, nil) + } + batcher.OnMessage(m3, nd.Next, nil) + + Convey("Each message should be in its group", func() { + var err error + group1, err := batcher.batches["group1"].PopData() + So(err, ShouldBeNil) + group2, err := batcher.batches["group2"].PopData() + So(err, ShouldBeNil) + So(string(group1), ShouldEqual, "MESSAGE 1MESSAGE 1MESSAGE 1") + So(string(group2), ShouldEqual, "MESSAGE 2MESSAGE 2MESSAGE 2MESSAGE 3") + So(len(batcher.batches), ShouldEqual, 2) + m1.AssertExpectations(t) + m2.AssertExpectations(t) + m3.AssertExpectations(t) + }) + + Convey("After a timeout the messages should be sent", func() { + clk := batcher.clk.(*clock.Mock) + So(len(batcher.batches), ShouldEqual, 2) + clk.Add(1000 * time.Second) + <-nd.nextCalled + <-nd.nextCalled + So(batcher.batches["group1"], ShouldBeNil) + So(batcher.batches["group2"], ShouldBeNil) + So(len(batcher.batches), ShouldEqual, 0) + nd.AssertExpectations(t) + }) + }) + }) +} diff --git a/components/batcher/config.go b/components/batcher/config.go new file mode 100644 index 0000000..f992080 --- /dev/null +++ b/components/batcher/config.go @@ -0,0 +1,8 @@ +package batcher + +// Config stores the config for a Batcher +type Config struct { + TimeoutMillis uint + Limit uint + MaxPendingBatches uint +} From 98c582e422c7103544b546010ebadcab17c502de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Thu, 4 Aug 2016 13:53:21 +0200 Subject: [PATCH 07/36] :white_check_mark: Add message.go testing --- message.go | 4 +- message_test.go | 113 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 message_test.go diff --git a/message.go b/message.go index b7a4f1f..b99f8aa 100644 --- a/message.go +++ b/message.go @@ -34,7 +34,7 @@ func (m *message) PopData() (ret []byte, err error) { } if m.payload.Empty() { - err = errors.New("Empty stack") + err = errors.New("No data") return } @@ -50,7 +50,7 @@ func (m *message) PopOpts() (ret map[string]interface{}, err error) { } if m.opts.Empty() { - err = errors.New("Empty stack") + err = errors.New("No options") return } diff --git a/message_test.go b/message_test.go new file mode 100644 index 0000000..4a19c4b --- /dev/null +++ b/message_test.go @@ -0,0 +1,113 @@ +package rbforwarder + +import ( + "testing" + + "github.com/oleiade/lane" + . "github.com/smartystreets/goconvey/convey" +) + +func TestMessage(t *testing.T) { + Convey("Given a payload and some metadata", t, func() { + payload := "This is the payload" + metadata := map[string]interface{}{ + "key1": "value1", + "key2": "value2", + } + + Convey("When the data is stored in a message", func() { + m := new(message) + m.payload = lane.NewStack() + m.opts = lane.NewStack() + + m.payload.Push([]byte(payload)) + m.opts.Push(metadata) + + Convey("Then the data can be recovered through messenger methods", func() { + data, err := m.PopData() + So(err, ShouldBeNil) + + opts, err := m.PopOpts() + So(err, ShouldBeNil) + + So(string(data), ShouldEqual, payload) + So(opts, ShouldNotBeNil) + + So(opts["key1"], ShouldEqual, "value1") + So(opts["key2"], ShouldEqual, "value2") + }) + }) + }) + + Convey("Given a message with nil data and opts", t, func() { + m := new(message) + + Convey("When trying to get message data", func() { + data, err1 := m.PopData() + opts, err2 := m.PopOpts() + + Convey("Then should error", func() { + So(err1, ShouldNotBeNil) + So(err2, ShouldNotBeNil) + So(err1.Error(), ShouldEqual, "Uninitialized payload") + So(err2.Error(), ShouldEqual, "Uninitialized options") + So(data, ShouldBeNil) + So(opts, ShouldBeNil) + }) + }) + }) + + Convey("Given a message with no options or no data", t, func() { + m := &message{ + payload: lane.NewStack(), + opts: lane.NewStack(), + } + + Convey("When trying to get message data", func() { + data, err1 := m.PopData() + opts, err2 := m.PopOpts() + + Convey("Then should error", func() { + So(err1, ShouldNotBeNil) + So(err2, ShouldNotBeNil) + So(err1.Error(), ShouldEqual, "No data") + So(err2.Error(), ShouldEqual, "No options") + So(data, ShouldBeNil) + So(opts, ShouldBeNil) + }) + }) + }) + + Convey("Given a delivered message", t, func() { + m := &message{ + payload: lane.NewStack(), + opts: lane.NewStack(), + code: 0, + status: "testing", + retries: 10, + } + + m.payload.Push([]byte("This message is delivered")) + m.opts.Push(map[string]interface{}{ + "key1": "value1", + "key2": "value2", + }) + + Convey("When the user needs a report for the message", func() { + r := m.Reports() + + Convey("Then a report can be obtained for the message", func() { + So(len(r), ShouldEqual, 1) + + code, status, retries := r[0].Status() + So(code, ShouldEqual, 0) + So(status, ShouldEqual, "testing") + So(retries, ShouldEqual, 10) + + opts := r[0].GetOpts() + So(opts["key1"], ShouldEqual, "value1") + So(opts["key2"], ShouldEqual, "value2") + }) + }) + }) +} From a90f28bcc87d54a512b6c9179de716597421b6cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Thu, 4 Aug 2016 13:59:21 +0200 Subject: [PATCH 08/36] :lipstick: Fix wrong test name --- components/batcher/batcher_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/batcher/batcher_test.go b/components/batcher/batcher_test.go index a98ed89..c8ccf60 100644 --- a/components/batcher/batcher_test.go +++ b/components/batcher/batcher_test.go @@ -44,7 +44,7 @@ func (m *TestMessage) Reports() []types.Reporter { return nil } -func TestRBForwarder(t *testing.T) { +func TestBatcher(t *testing.T) { Convey("Given a batcher", t, func() { batcher := &Batcher{config: Config{ TimeoutMillis: 1000, From 3d36c89c109c51388fad103fb8dcceb5f94a2588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 9 Aug 2016 21:40:04 +0200 Subject: [PATCH 09/36] :sparkles: Replaced message interface for a struct This decrease the risk of not pass the original info to the next component on the pipeline --- components/batch/batch.go | 68 +++++++ components/batch/batcher.go | 59 +++++++ components/batch/batcher_test.go | 215 +++++++++++++++++++++++ components/{batcher => batch}/config.go | 0 components/batcher/batch.go | 75 -------- components/batcher/batcher.go | 100 ----------- components/batcher/batcher_test.go | 196 --------------------- message.go | 74 -------- pipeline.go | 35 ++-- rbforwarder.go | 24 +-- rbforwarder_test.go | 115 +++++------- report.go | 6 +- reportHandler.go | 51 +++--- types/composer.go | 6 +- types/message.go | 64 +++++++ message_test.go => types/message_test.go | 60 +++---- types/messenger.go | 8 - types/reporter.go | 9 - 18 files changed, 544 insertions(+), 621 deletions(-) create mode 100644 components/batch/batch.go create mode 100644 components/batch/batcher.go create mode 100644 components/batch/batcher_test.go rename components/{batcher => batch}/config.go (100%) delete mode 100644 components/batcher/batch.go delete mode 100644 components/batcher/batcher.go delete mode 100644 components/batcher/batcher_test.go delete mode 100644 message.go create mode 100644 types/message.go rename message_test.go => types/message_test.go (65%) delete mode 100644 types/messenger.go delete mode 100644 types/reporter.go diff --git a/components/batch/batch.go b/components/batch/batch.go new file mode 100644 index 0000000..ffe7291 --- /dev/null +++ b/components/batch/batch.go @@ -0,0 +1,68 @@ +package batcher + +import ( + "bytes" + "time" + + "github.com/benbjohnson/clock" + "github.com/redBorder/rbforwarder/types" +) + +// Batch groups multiple messages +type Batch struct { + Group string + Message *types.Message + MessageCount uint // Current number of messages in the buffer + Next types.Next // Call to pass the message to the next handler +} + +// NewBatch creates a new instance of Batch +func NewBatch(m *types.Message, group string, next types.Next, clk clock.Clock, + timeoutMillis uint, ready chan *Batch) *Batch { + b := &Batch{ + Group: group, + Next: next, + Message: m, + MessageCount: 1, + } + + if clk == nil { + clk = clock.New() + } + + if timeoutMillis != 0 { + timer := clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) + + go func() { + <-timer.C + if b.MessageCount > 0 { + ready <- b + } + }() + } + + return b +} + +// Send the batch of messages to the next handler in the pipeline +func (b *Batch) Send(cb func()) { + cb() + b.Next(b.Message) +} + +// Add merges a new message in the buffer +func (b *Batch) Add(m *types.Message) { + newPayload := m.Payload.Pop().([]byte) + newOptions := m.Opts.Pop().(map[string]interface{}) + newReport := m.Reports.Pop() + + currentPayload := b.Message.Payload.Pop().([]byte) + buff := bytes.NewBuffer(currentPayload) + buff.Write(newPayload) + + b.Message.PushData(buff.Bytes()) + b.Message.Opts.Push(newOptions) + b.Message.Reports.Push(newReport) + + b.MessageCount++ +} diff --git a/components/batch/batcher.go b/components/batch/batcher.go new file mode 100644 index 0000000..99e49b5 --- /dev/null +++ b/components/batch/batcher.go @@ -0,0 +1,59 @@ +package batcher + +import ( + "sync" + + "github.com/benbjohnson/clock" + "github.com/redBorder/rbforwarder/types" +) + +// Batcher allows to merge multiple messages in a single one +type Batcher struct { + id int // Worker ID + batches map[string]*Batch // Collection of batches pending + readyBatches chan *Batch + wg sync.WaitGroup + clk clock.Clock + + config Config // Batcher configuration +} + +// Init starts a gorutine that can receive: +// - New messages that will be added to a existing or new batch of messages +// - A batch of messages that is ready to send (i.e. batch timeout has expired) +func (b *Batcher) Init(id int) { + b.batches = make(map[string]*Batch) + b.readyBatches = make(chan *Batch) + + if b.clk == nil { + b.clk = clock.New() + } + + go func() { + for batch := range b.readyBatches { + batch.Send(func() { + delete(b.batches, batch.Group) + }) + } + }() +} + +// OnMessage is called when a new message is receive. Add the new message to +// a batch +func (b *Batcher) OnMessage(m *types.Message, next types.Next, done types.Done) { + opts := m.Opts.Head().(map[string]interface{}) + group, exists := opts["batch_group"].(string) + if !exists { + next(m) + return + } + + if batch, exists := b.batches[group]; exists { + batch.Add(m) + if batch.MessageCount >= b.config.Limit { + b.readyBatches <- batch + } + } else { + b.batches[group] = NewBatch(m, group, next, b.clk, b.config.TimeoutMillis, b.readyBatches) + } +} diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go new file mode 100644 index 0000000..e94728d --- /dev/null +++ b/components/batch/batcher_test.go @@ -0,0 +1,215 @@ +package batcher + +import ( + "testing" + "time" + + "github.com/benbjohnson/clock" + "github.com/oleiade/lane" + "github.com/redBorder/rbforwarder/types" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/mock" +) + +type NexterDoner struct { + mock.Mock + nextCalled chan *types.Message +} + +func (nd *NexterDoner) Next(m *types.Message) { + nd.Called(m) + nd.nextCalled <- m +} + +func TestBatcher(t *testing.T) { + Convey("Given a batcher", t, func() { + batcher := &Batcher{config: Config{ + TimeoutMillis: 1000, + Limit: 10, + MaxPendingBatches: 10, + }, + clk: clock.NewMock(), + } + + batcher.Init(0) + + Convey("When a message is received, but not yet sent", func() { + m := &types.Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), + } + m.Payload.Push([]byte("Hello World")) + m.Opts.Push(map[string]interface{}{ + "batch_group": "group1", + }) + m.Reports.Push("Report") + + batcher.OnMessage(m, nil, nil) + + Convey("Message should be present on the batch", func() { + var batch *Batch + var exists bool + batch, exists = batcher.batches["group1"] + + So(exists, ShouldBeTrue) + data := batch.Message.Payload.Pop().([]byte) + opts := batch.Message.Opts.Pop().(map[string]interface{}) + report := batch.Message.Reports.Pop().(string) + So(string(data), ShouldEqual, "Hello World") + So(opts["batch_group"], ShouldEqual, "group1") + So(report, ShouldEqual, "Report") + So(len(batcher.batches), ShouldEqual, 1) + }) + }) + + Convey("When the max number of messages is reached", func() { + var messages []*types.Message + + for i := 0; i < int(batcher.config.Limit); i++ { + m := &types.Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), + } + m.Payload.Push([]byte("ABC")) + m.Opts.Push(map[string]interface{}{ + "batch_group": "group1", + }) + m.Reports.Push("Report") + + messages = append(messages, m) + } + + nd := new(NexterDoner) + nd.nextCalled = make(chan *types.Message) + nd.On("Next", mock.AnythingOfType("*types.Message")).Times(1) + + for i := 0; i < int(batcher.config.Limit); i++ { + batcher.OnMessage(messages[i], nd.Next, nil) + } + + Convey("The batch should be sent", func() { + m := <-nd.nextCalled + nd.AssertExpectations(t) + data := m.Payload.Pop().([]byte) + optsSize := m.Opts.Size() + reportsSize := m.Reports.Size() + So(string(data), ShouldEqual, "ABCABCABCABCABCABCABCABCABCABC") + So(m.Payload.Empty(), ShouldBeTrue) + So(optsSize, ShouldEqual, batcher.config.Limit) + So(reportsSize, ShouldEqual, batcher.config.Limit) + So(batcher.batches["group1"], ShouldBeNil) + So(len(batcher.batches), ShouldEqual, 0) + }) + }) + + Convey("When the timeout expires", func() { + var messages []*types.Message + + for i := 0; i < 5; i++ { + m := &types.Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), + } + m.Payload.Push([]byte("ABC")) + m.Opts.Push(map[string]interface{}{ + "batch_group": "group1", + }) + m.Reports.Push("Report") + + messages = append(messages, m) + } + + nd := new(NexterDoner) + nd.nextCalled = make(chan *types.Message, 1) + nd.On("Next", mock.AnythingOfType("*types.Message")).Times(1) + + for i := 0; i < 5; i++ { + batcher.OnMessage(messages[i], nd.Next, nil) + } + + clk := batcher.clk.(*clock.Mock) + + Convey("The batch should be sent", func() { + clk.Add(500 * time.Millisecond) + So(batcher.batches["group1"], ShouldNotBeNil) + clk.Add(500 * time.Millisecond) + <-nd.nextCalled + So(batcher.batches["group1"], ShouldBeNil) + So(len(batcher.batches), ShouldEqual, 0) + nd.AssertExpectations(t) + }) + }) + + Convey("When multiple messages are received with differente groups", func() { + m1 := &types.Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), + } + m1.Payload.Push([]byte("MESSAGE 1")) + m1.Opts.Push(map[string]interface{}{ + "batch_group": "group1", + }) + + m2 := &types.Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), + } + m2.Payload.Push([]byte("MESSAGE 2")) + m2.Opts.Push(map[string]interface{}{ + "batch_group": "group2", + }) + + m3 := &types.Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), + } + m3.Payload.Push([]byte("MESSAGE 3")) + m3.Opts.Push(map[string]interface{}{ + "batch_group": "group2", + }) + + nd := new(NexterDoner) + nd.nextCalled = make(chan *types.Message, 2) + nd.On("Next", mock.AnythingOfType("*types.Message")).Times(2) + + batcher.OnMessage(m1, nd.Next, nil) + batcher.OnMessage(m2, nd.Next, nil) + batcher.OnMessage(m3, nd.Next, nil) + + Convey("Each message should be in its group", func() { + var err error + group1 := batcher.batches["group1"].Message.Payload.Pop().([]byte) + So(err, ShouldBeNil) + + group2 := batcher.batches["group2"].Message.Payload.Pop().([]byte) + So(err, ShouldBeNil) + + So(string(group1), ShouldEqual, "MESSAGE 1") + So(string(group2), ShouldEqual, "MESSAGE 2MESSAGE 3") + So(len(batcher.batches), ShouldEqual, 2) + }) + + Convey("After a timeout the messages should be sent", func() { + clk := batcher.clk.(*clock.Mock) + So(len(batcher.batches), ShouldEqual, 2) + clk.Add(time.Duration(batcher.config.TimeoutMillis) * time.Millisecond) + group1 := <-nd.nextCalled + group1Data := group1.Payload.Pop().([]byte) + So(string(group1Data), ShouldEqual, "MESSAGE 1") + group2 := <-nd.nextCalled + group2Data := group2.Payload.Pop().([]byte) + So(string(group2Data), ShouldEqual, "MESSAGE 2MESSAGE 3") + So(batcher.batches["group1"], ShouldBeNil) + So(batcher.batches["group2"], ShouldBeNil) + So(len(batcher.batches), ShouldEqual, 0) + nd.AssertExpectations(t) + }) + }) + }) +} diff --git a/components/batcher/config.go b/components/batch/config.go similarity index 100% rename from components/batcher/config.go rename to components/batch/config.go diff --git a/components/batcher/batch.go b/components/batcher/batch.go deleted file mode 100644 index ef50eb8..0000000 --- a/components/batcher/batch.go +++ /dev/null @@ -1,75 +0,0 @@ -package batcher - -import ( - "bytes" - "errors" - "time" - - "github.com/benbjohnson/clock" - "github.com/oleiade/lane" - "github.com/redBorder/rbforwarder/types" -) - -// BatchMessage contains multiple messages -type BatchMessage struct { - Group string // Name used to group messages - Buff *bytes.Buffer // Buffer for merge multiple messages - MessageCount uint // Current number of messages in the buffer - BytesCount uint // Current number of bytes in the buffer - Next types.Next // Call to pass the message to the next handler - Opts *lane.Stack // Original messages options -} - -// StartTimeout initializes a timeout used to send the messages when expires. No -// matters how many messages are in the buffer. -func (b *BatchMessage) StartTimeout(clk clock.Clock, timeoutMillis uint, ready chan *BatchMessage) *BatchMessage { - if clk == nil { - clk = clock.New() - } - - if timeoutMillis != 0 { - timer := clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) - - go func() { - <-timer.C - if b.MessageCount > 0 { - ready <- b - } - }() - } - return b -} - -// Send the batch of messages to the next handler in the pipeline -func (b *BatchMessage) Send(cb func()) { - cb() - b.Next(b) -} - -// Write merges a new message in the buffer -func (b *BatchMessage) Write(data []byte) { - b.Buff.Write(data) - b.MessageCount++ -} - -// PopData returns the buffer with all the messages merged -func (b *BatchMessage) PopData() (ret []byte, err error) { - return b.Buff.Bytes(), nil -} - -// PopOpts get the data stored by the previous handler -func (b *BatchMessage) PopOpts() (ret map[string]interface{}, err error) { - if b.Opts.Empty() { - err = errors.New("Empty stack") - return - } - - ret = b.Opts.Pop().(map[string]interface{}) - - return -} - -// Reports do nothing -func (b *BatchMessage) Reports() []types.Reporter { - return nil -} diff --git a/components/batcher/batcher.go b/components/batcher/batcher.go deleted file mode 100644 index 1303d41..0000000 --- a/components/batcher/batcher.go +++ /dev/null @@ -1,100 +0,0 @@ -package batcher - -import ( - "bytes" - "sync" - - "github.com/benbjohnson/clock" - "github.com/oleiade/lane" - "github.com/redBorder/rbforwarder/types" -) - -// Batcher allows to merge multiple messages in a single one -type Batcher struct { - id int // Worker ID - batches map[string]*BatchMessage // Collection of batches pending - newBatches chan *BatchMessage // Send messages to sender gorutine - pendingBatches chan *BatchMessage // Send messages to sender gorutine - wg sync.WaitGroup - clk clock.Clock - - config Config // Batcher configuration -} - -// Init starts a gorutine that can receive: -// - New messages that will be added to a existing or new batch of messages -// - A batch of messages that is ready to send (i.e. batch timeout has expired) -func (b *Batcher) Init(id int) { - b.batches = make(map[string]*BatchMessage) - b.newBatches = make(chan *BatchMessage) - b.pendingBatches = make(chan *BatchMessage) - - readyBatches := make(chan *BatchMessage) - - if b.clk == nil { - b.clk = clock.New() - } - - go func() { - for { - select { - case batchMessage := <-readyBatches: - batchMessage.Send(func() { - delete(b.batches, batchMessage.Group) - }) - - case batchMessage := <-b.newBatches: - batchMessage.StartTimeout(b.clk, b.config.TimeoutMillis, readyBatches) - b.batches[batchMessage.Group] = batchMessage - b.wg.Done() - - case batchMessage := <-b.pendingBatches: - opts, err := batchMessage.PopOpts() - if err != nil { - break - } - - b.batches[batchMessage.Group].Opts.Push(opts) - b.batches[batchMessage.Group].Write(batchMessage.Buff.Bytes()) - - if b.batches[batchMessage.Group].MessageCount >= b.config.Limit { - b.batches[batchMessage.Group].Send(func() { - delete(b.batches, batchMessage.Group) - }) - } - - b.wg.Done() - } - } - }() -} - -// OnMessage is called when a new message is receive. Add the new message to -// a batch -func (b *Batcher) OnMessage(m types.Messenger, next types.Next, done types.Done) { - opts, _ := m.PopOpts() - group, ok := opts["batch_group"].(string) - if !ok { - next(m) - } - - data, _ := m.PopData() - batchMessage := &BatchMessage{ - Buff: bytes.NewBuffer(data), - Opts: lane.NewStack(), - MessageCount: 1, - Next: next, - Group: group, - } - - batchMessage.Opts.Push(opts) - - b.wg.Add(1) - if _, exists := b.batches[group]; exists { - b.pendingBatches <- batchMessage - } else { - b.newBatches <- batchMessage - } - - b.wg.Wait() -} diff --git a/components/batcher/batcher_test.go b/components/batcher/batcher_test.go deleted file mode 100644 index c8ccf60..0000000 --- a/components/batcher/batcher_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package batcher - -import ( - "testing" - "time" - - "github.com/benbjohnson/clock" - "github.com/redBorder/rbforwarder/types" - . "github.com/smartystreets/goconvey/convey" - "github.com/stretchr/testify/mock" -) - -type NexterDoner struct { - mock.Mock - nextCalled chan struct{} -} - -func (nd *NexterDoner) Next(m types.Messenger) { - nd.nextCalled <- struct{}{} - nd.Called(m) -} - -func (nd *NexterDoner) Done(m types.Messenger, code int, status string) { - nd.Called(m, code, status) -} - -type TestMessage struct { - mock.Mock -} - -func (m *TestMessage) PopData() (data []byte, err error) { - args := m.Called() - - return args.Get(0).([]byte), args.Error(1) -} - -func (m *TestMessage) PopOpts() (opts map[string]interface{}, err error) { - args := m.Called() - - return args.Get(0).(map[string]interface{}), args.Error(1) -} - -func (m *TestMessage) Reports() []types.Reporter { - return nil -} - -func TestBatcher(t *testing.T) { - Convey("Given a batcher", t, func() { - batcher := &Batcher{config: Config{ - TimeoutMillis: 1000, - Limit: 10, - MaxPendingBatches: 10, - }, - clk: clock.NewMock(), - } - - batcher.Init(0) - - Convey("When a message is received, but not yet sent", func() { - m := new(TestMessage) - m.On("PopData").Return([]byte("Hello World"), nil) - m.On("PopOpts").Return(map[string]interface{}{ - "batch_group": "group1", - }, nil) - - batcher.OnMessage(m, nil, nil) - - Convey("Message should be present on the batch", func() { - var data *BatchMessage - var exists bool - data, exists = batcher.batches["group1"] - - So(exists, ShouldBeTrue) - So(string(data.Buff.Bytes()), ShouldEqual, "Hello World") - So(len(batcher.batches), ShouldEqual, 1) - - m.AssertExpectations(t) - }) - }) - - Convey("When the max number of messages is reached", func() { - m := new(TestMessage) - m.On("PopData").Return([]byte("ABC"), nil) - m.On("PopOpts").Return(map[string]interface{}{ - "batch_group": "group1", - }, nil) - - nd := new(NexterDoner) - nd.nextCalled = make(chan struct{}, 1) - nd.On("Next", mock.MatchedBy(func(m *BatchMessage) bool { - data, _ := m.PopData() - return string(data) == "ABCABCABCABCABCABCABCABCABCABC" - })).Times(1) - - for i := 0; i < int(batcher.config.Limit); i++ { - batcher.OnMessage(m, nd.Next, nil) - } - - Convey("The batch should be sent", func() { - nd.AssertExpectations(t) - <-nd.nextCalled - So(batcher.batches["group1"], ShouldBeNil) - So(len(batcher.batches), ShouldEqual, 0) - }) - }) - - Convey("When the timeout expires", func() { - m := new(TestMessage) - m.On("PopData").Return([]byte("ABC"), nil) - m.On("PopOpts").Return(map[string]interface{}{ - "batch_group": "group1", - }, nil) - - nd := new(NexterDoner) - nd.nextCalled = make(chan struct{}, 1) - nd.On("Next", mock.MatchedBy(func(m *BatchMessage) bool { - data, _ := m.PopData() - return string(data) == "ABCABCABCABCABC" - })).Times(1) - - for i := 0; i < 5; i++ { - batcher.OnMessage(m, nd.Next, nil) - } - - clk := batcher.clk.(*clock.Mock) - - Convey("The batch should be sent", func() { - clk.Add(500 * time.Millisecond) - So(batcher.batches["group1"], ShouldNotBeNil) - clk.Add(500 * time.Millisecond) - <-nd.nextCalled - So(batcher.batches["group1"], ShouldBeNil) - So(len(batcher.batches), ShouldEqual, 0) - nd.AssertExpectations(t) - }) - }) - - Convey("When multiple messages are received with differente groups", func() { - m1 := new(TestMessage) - m1.On("PopData").Return([]byte("MESSAGE 1"), nil) - m1.On("PopOpts").Return(map[string]interface{}{ - "batch_group": "group1", - }, nil) - - m2 := new(TestMessage) - m2.On("PopData").Return([]byte("MESSAGE 2"), nil) - m2.On("PopOpts").Return(map[string]interface{}{ - "batch_group": "group2", - }, nil) - - m3 := new(TestMessage) - m3.On("PopData").Return([]byte("MESSAGE 3"), nil) - m3.On("PopOpts").Return(map[string]interface{}{ - "batch_group": "group2", - }, nil) - - nd := new(NexterDoner) - nd.nextCalled = make(chan struct{}, 2) - nd.On("Next", mock.AnythingOfType("*batcher.BatchMessage")).Times(2) - - for i := 0; i < 3; i++ { - batcher.OnMessage(m1, nd.Next, nil) - } - for i := 0; i < 3; i++ { - batcher.OnMessage(m2, nd.Next, nil) - } - batcher.OnMessage(m3, nd.Next, nil) - - Convey("Each message should be in its group", func() { - var err error - group1, err := batcher.batches["group1"].PopData() - So(err, ShouldBeNil) - group2, err := batcher.batches["group2"].PopData() - So(err, ShouldBeNil) - So(string(group1), ShouldEqual, "MESSAGE 1MESSAGE 1MESSAGE 1") - So(string(group2), ShouldEqual, "MESSAGE 2MESSAGE 2MESSAGE 2MESSAGE 3") - So(len(batcher.batches), ShouldEqual, 2) - m1.AssertExpectations(t) - m2.AssertExpectations(t) - m3.AssertExpectations(t) - }) - - Convey("After a timeout the messages should be sent", func() { - clk := batcher.clk.(*clock.Mock) - So(len(batcher.batches), ShouldEqual, 2) - clk.Add(1000 * time.Second) - <-nd.nextCalled - <-nd.nextCalled - So(batcher.batches["group1"], ShouldBeNil) - So(batcher.batches["group2"], ShouldBeNil) - So(len(batcher.batches), ShouldEqual, 0) - nd.AssertExpectations(t) - }) - }) - }) -} diff --git a/message.go b/message.go deleted file mode 100644 index b99f8aa..0000000 --- a/message.go +++ /dev/null @@ -1,74 +0,0 @@ -package rbforwarder - -import ( - "errors" - - "github.com/oleiade/lane" - "github.com/redBorder/rbforwarder/types" -) - -// message is used to send data through the pipeline -type message struct { - payload *lane.Stack - opts *lane.Stack - seq uint64 // Unique ID for the report, used to maintain sequence - status string // Result of the sending - code int // Result of the sending - retries int -} - -// PushData store data on an LIFO queue so the nexts handlers can use it -func (m *message) PushData(v []byte) { - if m.payload == nil { - m.payload = lane.NewStack() - } - - m.payload.Push(v) -} - -// PopData get the data stored by the previous handler -func (m *message) PopData() (ret []byte, err error) { - if m.payload == nil { - err = errors.New("Uninitialized payload") - return - } - - if m.payload.Empty() { - err = errors.New("No data") - return - } - - ret = m.payload.Pop().([]byte) - return -} - -// PopData get the data stored by the previous handler -func (m *message) PopOpts() (ret map[string]interface{}, err error) { - if m.opts == nil { - err = errors.New("Uninitialized options") - return - } - - if m.opts.Empty() { - err = errors.New("No options") - return - } - - ret = m.opts.Pop().(map[string]interface{}) - return -} - -func (m message) Reports() []types.Reporter { - var reports []types.Reporter - - for !m.opts.Empty() { - reports = append(reports, report{ - code: m.code, - status: m.status, - retries: m.retries, - opts: m.opts.Pop().(map[string]interface{}), - }) - } - - return reports -} diff --git a/pipeline.go b/pipeline.go index 6dd3d0d..1ca7eb0 100644 --- a/pipeline.go +++ b/pipeline.go @@ -3,24 +3,25 @@ package rbforwarder import ( "sync" + "github.com/oleiade/lane" "github.com/redBorder/rbforwarder/types" ) type component struct { - pool chan chan *message + pool chan chan *types.Message workers int } // pipeline contains the components type pipeline struct { components []component - input chan *message - retry chan *message - output chan *message + input chan *types.Message + retry chan *types.Message + output chan *types.Message } // newPipeline creates a new Backend -func newPipeline(input, retry, output chan *message) *pipeline { +func newPipeline(input, retry, output chan *types.Message) *pipeline { var wg sync.WaitGroup p := &pipeline{ input: input, @@ -65,7 +66,7 @@ func (p *pipeline) PushComponent(composser types.Composer, w int) { var wg sync.WaitGroup c := component{ workers: w, - pool: make(chan chan *message, w), + pool: make(chan chan *types.Message, w), } index := len(p.components) @@ -74,23 +75,29 @@ func (p *pipeline) PushComponent(composser types.Composer, w int) { for i := 0; i < w; i++ { composser.Init(i) - worker := make(chan *message) + worker := make(chan *types.Message) p.components[index].pool <- worker wg.Add(1) go func(i int) { wg.Done() for m := range worker { - composser.OnMessage(m, func(m types.Messenger) { + composser.OnMessage(m, func(m *types.Message) { if len(p.components) >= index { nextWorker := <-p.components[index+1].pool - nextWorker <- m.(*message) + nextWorker <- m } - }, func(m types.Messenger, code int, status string) { - rbmessage := m.(*message) - rbmessage.code = code - rbmessage.status = status - p.output <- rbmessage + }, func(m *types.Message, code int, status string) { + reports := lane.NewStack() + for !m.Reports.Empty() { + rep := m.Reports.Pop().(report) + rep.code = code + rep.status = status + reports.Push(rep) + } + + m.Reports = reports + p.output <- m }) p.components[index].pool <- worker diff --git a/rbforwarder.go b/rbforwarder.go index eaf2422..7699ee1 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -30,9 +30,9 @@ type RBForwarder struct { // NewRBForwarder creates a new Forwarder object func NewRBForwarder(config Config) *RBForwarder { - produces := make(chan *message, config.QueueSize) - retries := make(chan *message, config.QueueSize) - reports := make(chan *message, config.QueueSize) + produces := make(chan *types.Message, config.QueueSize) + retries := make(chan *types.Message, config.QueueSize) + reports := make(chan *types.Message, config.QueueSize) f := &RBForwarder{ working: 1, @@ -71,13 +71,13 @@ func (f *RBForwarder) PushComponents(components []types.Composer, w []int) { // GetReports is used by the source to get a report for a sent message. // Reports are delivered on the same order that was sent -func (f *RBForwarder) GetReports() <-chan types.Reporter { +func (f *RBForwarder) GetReports() <-chan interface{} { return f.r.GetReports() } // GetOrderedReports is the same as GetReports() but the reports are delivered // in order -func (f *RBForwarder) GetOrderedReports() <-chan types.Reporter { +func (f *RBForwarder) GetOrderedReports() <-chan interface{} { return f.r.GetOrderedReports() } @@ -90,13 +90,17 @@ func (f *RBForwarder) Produce(buf []byte, options map[string]interface{}) error seq := f.currentProducedID f.currentProducedID++ - message := &message{ - seq: seq, - opts: lane.NewStack(), + message := &types.Message{ + Payload: lane.NewStack(), + Reports: lane.NewStack(), + Opts: lane.NewStack(), } - message.PushData(buf) - message.opts.Push(options) + message.Payload.Push(buf) + message.Opts.Push(options) + message.Reports.Push(report{ + seq: seq, + }) f.p.input <- message diff --git a/rbforwarder_test.go b/rbforwarder_test.go index 927e3b7..0a42f50 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -18,17 +18,15 @@ func (c *MockMiddleComponent) Init(id int) error { } func (c *MockMiddleComponent) OnMessage( - m types.Messenger, + m *types.Message, next types.Next, done types.Done, ) { c.Called(m) data, _ := m.PopData() processedData := "-> [" + string(data) + "] <-" - message := m.(*message) - message.PushData([]byte(processedData)) + m.PushData([]byte(processedData)) next(m) - } type MockComponent struct { @@ -46,7 +44,7 @@ func (c *MockComponent) Init(id int) error { } func (c *MockComponent) OnMessage( - m types.Messenger, + m *types.Message, next types.Next, done types.Done, ) { @@ -87,7 +85,7 @@ func TestRBForwarder(t *testing.T) { component.status = "OK" component.statusCode = 0 - component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")).Times(1) + component.On("OnMessage", mock.AnythingOfType("*types.Message")).Times(1) err := rbforwarder.Produce( []byte("Hello World"), @@ -105,7 +103,6 @@ func TestRBForwarder(t *testing.T) { So(lastReport, ShouldNotBeNil) So(reports, ShouldEqual, 1) - So(lastReport.opts["message_id"], ShouldEqual, "test123") So(lastReport.code, ShouldEqual, 0) So(lastReport.status, ShouldEqual, "OK") So(err, ShouldBeNil) @@ -132,57 +129,34 @@ func TestRBForwarder(t *testing.T) { //////////////////////////////////////////////////////////////////////////// Convey("When calling OnMessage() with options", func() { - component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) + component.On("OnMessage", mock.AnythingOfType("*types.Message")) - err := rbforwarder.Produce( - []byte("Hello World"), - map[string]interface{}{"option": "example_option"}, - ) + // err := rbforwarder.Produce( + // []byte("Hello World"), + // map[string]interface{}{"option": "example_option"}, + // ) rbforwarder.Close() - Convey("Should be possible to read an option", func() { - for report := range rbforwarder.GetReports() { - So(report.GetOpts(), ShouldNotBeNil) - So(report.GetOpts()["option"], ShouldEqual, "example_option") - } - - So(err, ShouldBeNil) - component.AssertExpectations(t) - }) + Convey("Should be possible to read an option", nil) - Convey("Should not be possible to read an nonexistent option", func() { - for report := range rbforwarder.GetReports() { - So(report.GetOpts(), ShouldNotBeNil) - So(report.GetOpts()["invalid"], ShouldBeEmpty) - } + Convey("Should not be possible to read an nonexistent option", nil) - So(err, ShouldBeNil) - component.AssertExpectations(t) - }) }) // //////////////////////////////////////////////////////////////////////////// Convey("When calling OnMessage() without options", func() { - component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) + component.On("OnMessage", mock.AnythingOfType("*types.Message")) - err := rbforwarder.Produce( - []byte("Hello World"), - nil, - ) + // err := rbforwarder.Produce( + // []byte("Hello World"), + // nil, + // ) rbforwarder.Close() - Convey("Should not be possible to read the option", func() { - for report := range rbforwarder.GetReports() { - So(err, ShouldBeNil) - So(report.GetOpts(), ShouldBeNil) - } - - So(err, ShouldBeNil) - component.AssertExpectations(t) - }) + Convey("Should not be possible to read the option", nil) }) // //////////////////////////////////////////////////////////////////////////// @@ -191,7 +165,7 @@ func TestRBForwarder(t *testing.T) { component.status = "Fake Error" component.statusCode = 99 - component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")).Times(4) + component.On("OnMessage", mock.AnythingOfType("*types.Message")).Times(4) err := rbforwarder.Produce( []byte("Hello World"), @@ -211,7 +185,6 @@ func TestRBForwarder(t *testing.T) { So(lastReport, ShouldNotBeNil) So(reports, ShouldEqual, 1) - So(lastReport.opts["message_id"], ShouldEqual, "test123") So(lastReport.status, ShouldEqual, "Fake Error") So(lastReport.code, ShouldEqual, 99) So(lastReport.retries, ShouldEqual, numRetries) @@ -225,7 +198,7 @@ func TestRBForwarder(t *testing.T) { Convey("When 10000 messages are produced", func() { var numErr int - component.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")). + component.On("OnMessage", mock.AnythingOfType("*types.Message")). Return(nil). Times(numMessages) @@ -252,26 +225,27 @@ func TestRBForwarder(t *testing.T) { component.AssertExpectations(t) }) - Convey("10000 reports should be received in order", func() { - ordered := true - var reports int - - for report := range rbforwarder.GetOrderedReports() { - if report.GetOpts()["message_id"] != reports { - ordered = false - } - reports++ - if reports >= numMessages { - rbforwarder.Close() - } - } - - So(numErr, ShouldBeZeroValue) - So(ordered, ShouldBeTrue) - So(reports, ShouldEqual, numMessages) - - component.AssertExpectations(t) - }) + Convey("10000 reports should be received in order", nil) + // func() { + // ordered := true + // var reports int + // + // for rep := range rbforwarder.GetOrderedReports() { + // if rep.(report).GetOpts()["message_id"] != reports { + // ordered = false + // } + // reports++ + // if reports >= numMessages { + // rbforwarder.Close() + // } + // } + // + // So(numErr, ShouldBeZeroValue) + // So(ordered, ShouldBeTrue) + // So(reports, ShouldEqual, numMessages) + // + // component.AssertExpectations(t) + // }) }) }) @@ -310,8 +284,8 @@ func TestRBForwarder(t *testing.T) { component2.status = "OK" component2.statusCode = 0 - component1.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) - component2.On("OnMessage", mock.AnythingOfType("*rbforwarder.message")) + component1.On("OnMessage", mock.AnythingOfType("*types.Message")) + component2.On("OnMessage", mock.AnythingOfType("*types.Message")) err := rbforwarder.Produce( []byte("Hello World"), @@ -322,11 +296,10 @@ func TestRBForwarder(t *testing.T) { Convey("\"Hello World\" message should be processed by the pipeline", func() { reports := 0 - for report := range rbforwarder.GetReports() { + for rep := range rbforwarder.GetReports() { reports++ - So(report.GetOpts()["message_id"], ShouldEqual, "test123") - code, status, _ := report.Status() + code, status, _ := rep.(report).Status() So(code, ShouldEqual, 0) So(status, ShouldEqual, "OK") } diff --git a/report.go b/report.go index 8df6dc3..651c959 100644 --- a/report.go +++ b/report.go @@ -1,16 +1,12 @@ package rbforwarder type report struct { + seq uint64 code int status string retries int - opts map[string]interface{} } func (r report) Status() (code int, status string, retries int) { return r.code, r.status, r.retries } - -func (r report) GetOpts() map[string]interface{} { - return r.opts -} diff --git a/reportHandler.go b/reportHandler.go index b105b58..43d6dd9 100644 --- a/reportHandler.go +++ b/reportHandler.go @@ -11,12 +11,12 @@ import ( // of the pipeline. The first element of the pipeline can know the status // of the produced message using GetReports() or GetOrderedReports() type reportHandler struct { - input chan *message // Receive messages from pipeline - retries chan *message // Send messages back to the pipeline - out chan *message // Send reports to the user + input chan *types.Message // Receive messages from pipeline + retries chan *types.Message // Send messages back to the pipeline + out chan *types.Message // Send reports to the user - queued map[uint64]types.Reporter // Store pending reports - currentReport uint64 // Last delivered report + queued map[uint64]interface{} // Store pending reports + currentReport uint64 // Last delivered report maxRetries int backoff int @@ -27,15 +27,15 @@ type reportHandler struct { // newReportHandler creates a new instance of reportHandler func newReporter( maxRetries, backoff int, - input, retries chan *message, + input, retries chan *types.Message, ) *reportHandler { r := &reportHandler{ input: input, retries: retries, - out: make(chan *message, 100), // NOTE Temp channel size + out: make(chan *types.Message, 100), // NOTE Temp channel size - queued: make(map[uint64]types.Reporter), + queued: make(map[uint64]interface{}), maxRetries: maxRetries, backoff: backoff, @@ -45,23 +45,26 @@ func newReporter( // Get reports from the handler channel for m := range r.input { // If the message has status code 0 (success) send the report to the user - if m.code == 0 || r.maxRetries == 0 { + rep := m.Reports.Head().(report) + if rep.code == 0 || r.maxRetries == 0 { r.out <- m continue } // If the message has status code != 0 (fail) but has been retried the // maximum number or retries also send it to the user - if r.maxRetries > 0 && m.retries >= r.maxRetries { + if r.maxRetries > 0 && rep.retries >= r.maxRetries { r.out <- m continue } - // In othe case retry the message sending it again to the pipeline + // In other case retry the message sending it again to the pipeline r.wg.Add(1) - go func(m *message) { + go func(m *types.Message) { defer r.wg.Done() - m.retries++ + rep := m.Reports.Pop().(report) + rep.retries++ + m.Reports.Push(rep) <-time.After(time.Duration(r.backoff) * time.Second) r.retries <- m }(m) @@ -77,13 +80,14 @@ func newReporter( return r } -func (r *reportHandler) GetReports() chan types.Reporter { - reports := make(chan types.Reporter) +func (r *reportHandler) GetReports() chan interface{} { + reports := make(chan interface{}) go func() { for message := range r.out { - for _, report := range message.Reports() { - reports <- report + for !message.Reports.Empty() { + rep := message.Reports.Pop().(report) + reports <- rep } } @@ -93,19 +97,20 @@ func (r *reportHandler) GetReports() chan types.Reporter { return reports } -func (r *reportHandler) GetOrderedReports() chan types.Reporter { - reports := make(chan types.Reporter) +func (r *reportHandler) GetOrderedReports() chan interface{} { + reports := make(chan interface{}) go func() { for message := range r.out { - for _, report := range message.Reports() { - if message.seq == r.currentReport { + for !message.Reports.Empty() { + rep := message.Reports.Pop().(report) + if rep.seq == r.currentReport { // The message is the expected. Send it. - reports <- report + reports <- rep r.currentReport++ } else { // This message is not the expected. Store it. - r.queued[message.seq] = report + r.queued[rep.seq] = rep } // Check if there are stored messages and send them. diff --git a/types/composer.go b/types/composer.go index e0bd721..7999ffe 100644 --- a/types/composer.go +++ b/types/composer.go @@ -2,16 +2,16 @@ package types // Next should be called by a component in order to pass the message to the next // component in the pipeline. -type Next func(Messenger) +type Next func(*Message) // Done should be called by a component in order to return the message to the // message handler. Can be used by the last component to inform that the // message processing is done o by a middle component to inform an error. -type Done func(Messenger, int, string) +type Done func(*Message, int, string) // Composer represents a component in the pipeline that performs a work on // a message type Composer interface { Init(int) error - OnMessage(Messenger, Next, Done) + OnMessage(*Message, Next, Done) } diff --git a/types/message.go b/types/message.go new file mode 100644 index 0000000..80a720f --- /dev/null +++ b/types/message.go @@ -0,0 +1,64 @@ +package types + +import ( + "errors" + + "github.com/oleiade/lane" +) + +// Message is used to send data through the pipeline +type Message struct { + Payload *lane.Stack + Opts *lane.Stack + Reports *lane.Stack +} + +// PushData store data on an LIFO queue so the nexts handlers can use it +func (m *Message) PushData(v []byte) { + if m.Payload == nil { + m.Payload = lane.NewStack() + } + + m.Payload.Push(v) +} + +// PopData get the data stored by the previous handler +func (m *Message) PopData() (ret []byte, err error) { + if m.Payload == nil { + err = errors.New("Uninitialized payload") + return + } + + if m.Payload.Empty() { + err = errors.New("No data") + return + } + + ret = m.Payload.Pop().([]byte) + return +} + +// PopData get the data stored by the previous handler +func (m *Message) PopOpts() (ret map[string]interface{}, err error) { + if m.Opts == nil { + err = errors.New("Uninitialized options") + return + } + + if m.Opts.Empty() { + err = errors.New("No options") + return + } + + ret = m.Opts.Pop().(map[string]interface{}) + return +} + +func (m *Message) PushOpts(Opts map[string]interface{}) error { + if m.Opts == nil { + return errors.New("Uninitialized options") + } + + m.Opts.Push(Opts) + return nil +} diff --git a/message_test.go b/types/message_test.go similarity index 65% rename from message_test.go rename to types/message_test.go index 4a19c4b..c6d3b83 100644 --- a/message_test.go +++ b/types/message_test.go @@ -1,4 +1,4 @@ -package rbforwarder +package types import ( "testing" @@ -16,12 +16,12 @@ func TestMessage(t *testing.T) { } Convey("When the data is stored in a message", func() { - m := new(message) - m.payload = lane.NewStack() - m.opts = lane.NewStack() + m := new(Message) + m.Payload = lane.NewStack() + m.Opts = lane.NewStack() - m.payload.Push([]byte(payload)) - m.opts.Push(metadata) + m.Payload.Push([]byte(payload)) + m.Opts.Push(metadata) Convey("Then the data can be recovered through messenger methods", func() { data, err := m.PopData() @@ -40,7 +40,7 @@ func TestMessage(t *testing.T) { }) Convey("Given a message with nil data and opts", t, func() { - m := new(message) + m := new(Message) Convey("When trying to get message data", func() { data, err1 := m.PopData() @@ -58,9 +58,9 @@ func TestMessage(t *testing.T) { }) Convey("Given a message with no options or no data", t, func() { - m := &message{ - payload: lane.NewStack(), - opts: lane.NewStack(), + m := &Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), } Convey("When trying to get message data", func() { @@ -79,35 +79,29 @@ func TestMessage(t *testing.T) { }) Convey("Given a delivered message", t, func() { - m := &message{ - payload: lane.NewStack(), - opts: lane.NewStack(), - code: 0, - status: "testing", - retries: 10, + m := &Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), } - m.payload.Push([]byte("This message is delivered")) - m.opts.Push(map[string]interface{}{ + m.Reports.Push("This is a report") + m.Reports.Push("This is a report") + m.Reports.Push("This is a report") + m.Payload.Push([]byte("This message is delivered")) + m.Opts.Push(map[string]interface{}{ "key1": "value1", "key2": "value2", }) - Convey("When the user needs a report for the message", func() { - r := m.Reports() - - Convey("Then a report can be obtained for the message", func() { - So(len(r), ShouldEqual, 1) - - code, status, retries := r[0].Status() - So(code, ShouldEqual, 0) - So(status, ShouldEqual, "testing") - So(retries, ShouldEqual, 10) - - opts := r[0].GetOpts() - So(opts["key1"], ShouldEqual, "value1") - So(opts["key2"], ShouldEqual, "value2") - }) + Convey("Then a report can be obtained for the message", func() { + var i int + for !m.Reports.Empty() { + i++ + rep := m.Reports.Pop().(string) + So(rep, ShouldEqual, "This is a report") + } + So(i, ShouldEqual, 3) }) }) } diff --git a/types/messenger.go b/types/messenger.go deleted file mode 100644 index 7b01b55..0000000 --- a/types/messenger.go +++ /dev/null @@ -1,8 +0,0 @@ -package types - -// Messenger is used by modules to handle messages -type Messenger interface { - PopData() ([]byte, error) - PopOpts() (map[string]interface{}, error) - Reports() []Reporter -} diff --git a/types/reporter.go b/types/reporter.go deleted file mode 100644 index a670f5d..0000000 --- a/types/reporter.go +++ /dev/null @@ -1,9 +0,0 @@ -package types - -// Reporter returns information about a processed message. The GetOpts method -// should return, at least, the same info that was pushed to the original -// message from the user. -type Reporter interface { - Status() (code int, status string, retries int) - GetOpts() map[string]interface{} -} From 399037140f34ae80e30ba6215825d6286c590cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 10 Aug 2016 10:00:56 +0200 Subject: [PATCH 10/36] :sparkles: Add opaque to produce function It can be recovered on reports --- components/batch/batch.go | 6 +- components/batch/batcher.go | 36 +++---- components/batch/batcher_test.go | 35 ++++-- rbforwarder.go | 11 +- rbforwarder_test.go | 104 +++++++++--------- report.go | 6 +- types/message.go | 104 +++++++++--------- types/message_test.go | 180 +++++++++++++------------------ 8 files changed, 233 insertions(+), 249 deletions(-) diff --git a/components/batch/batch.go b/components/batch/batch.go index ffe7291..65b1e50 100644 --- a/components/batch/batch.go +++ b/components/batch/batch.go @@ -26,10 +26,6 @@ func NewBatch(m *types.Message, group string, next types.Next, clk clock.Clock, MessageCount: 1, } - if clk == nil { - clk = clock.New() - } - if timeoutMillis != 0 { timer := clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) @@ -60,7 +56,7 @@ func (b *Batch) Add(m *types.Message) { buff := bytes.NewBuffer(currentPayload) buff.Write(newPayload) - b.Message.PushData(buff.Bytes()) + b.Message.Payload.Push(buff.Bytes()) b.Message.Opts.Push(newOptions) b.Message.Reports.Push(newReport) diff --git a/components/batch/batcher.go b/components/batch/batcher.go index 99e49b5..259d8af 100644 --- a/components/batch/batcher.go +++ b/components/batch/batcher.go @@ -1,8 +1,6 @@ package batcher import ( - "sync" - "github.com/benbjohnson/clock" "github.com/redBorder/rbforwarder/types" ) @@ -12,7 +10,6 @@ type Batcher struct { id int // Worker ID batches map[string]*Batch // Collection of batches pending readyBatches chan *Batch - wg sync.WaitGroup clk clock.Clock config Config // Batcher configuration @@ -22,12 +19,10 @@ type Batcher struct { // - New messages that will be added to a existing or new batch of messages // - A batch of messages that is ready to send (i.e. batch timeout has expired) func (b *Batcher) Init(id int) { + b.id = id b.batches = make(map[string]*Batch) b.readyBatches = make(chan *Batch) - - if b.clk == nil { - b.clk = clock.New() - } + b.clk = clock.New() go func() { for batch := range b.readyBatches { @@ -41,19 +36,20 @@ func (b *Batcher) Init(id int) { // OnMessage is called when a new message is receive. Add the new message to // a batch func (b *Batcher) OnMessage(m *types.Message, next types.Next, done types.Done) { - opts := m.Opts.Head().(map[string]interface{}) - group, exists := opts["batch_group"].(string) - if !exists { - next(m) - return - } - - if batch, exists := b.batches[group]; exists { - batch.Add(m) - if batch.MessageCount >= b.config.Limit { - b.readyBatches <- batch + if opts, ok := m.Opts.Head().(map[string]interface{}); ok { + if group, exists := opts["batch_group"].(string); exists { + if batch, exists := b.batches[group]; exists { + batch.Add(m) + if batch.MessageCount >= b.config.Limit { + b.readyBatches <- batch + } + } else { + b.batches[group] = NewBatch(m, group, next, b.clk, b.config.TimeoutMillis, b.readyBatches) + } + + return } - } else { - b.batches[group] = NewBatch(m, group, next, b.clk, b.config.TimeoutMillis, b.readyBatches) } + + next(m) } diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go index e94728d..7ab0915 100644 --- a/components/batch/batcher_test.go +++ b/components/batch/batcher_test.go @@ -23,15 +23,38 @@ func (nd *NexterDoner) Next(m *types.Message) { func TestBatcher(t *testing.T) { Convey("Given a batcher", t, func() { - batcher := &Batcher{config: Config{ - TimeoutMillis: 1000, - Limit: 10, - MaxPendingBatches: 10, - }, - clk: clock.NewMock(), + batcher := &Batcher{ + config: Config{ + TimeoutMillis: 1000, + Limit: 10, + MaxPendingBatches: 10, + }, } batcher.Init(0) + batcher.clk = clock.NewMock() + + Convey("When a message is received with no batch group", func() { + m := &types.Message{ + Payload: lane.NewStack(), + Opts: lane.NewStack(), + Reports: lane.NewStack(), + } + m.Payload.Push([]byte("Hello World")) + + nd := new(NexterDoner) + nd.nextCalled = make(chan *types.Message, 1) + nd.On("Next", mock.AnythingOfType("*types.Message")).Times(1) + + batcher.OnMessage(m, nd.Next, nil) + + Convey("Message should be present on the batch", func() { + nd.AssertExpectations(t) + m := <-nd.nextCalled + So(len(batcher.batches), ShouldEqual, 0) + So(string(m.Payload.Pop().([]byte)), ShouldEqual, "Hello World") + }) + }) Convey("When a message is received, but not yet sent", func() { m := &types.Message{ diff --git a/rbforwarder.go b/rbforwarder.go index 7699ee1..142c95c 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -82,7 +82,7 @@ func (f *RBForwarder) GetOrderedReports() <-chan interface{} { } // Produce is used by the source to send messages to the backend -func (f *RBForwarder) Produce(buf []byte, options map[string]interface{}) error { +func (f *RBForwarder) Produce(buf []byte, options map[string]interface{}, opaque interface{}) error { if atomic.LoadUint32(&f.working) == 0 { return errors.New("Forwarder has been closed") } @@ -95,12 +95,15 @@ func (f *RBForwarder) Produce(buf []byte, options map[string]interface{}) error Reports: lane.NewStack(), Opts: lane.NewStack(), } + r := report{ + seq: seq, + opaque: lane.NewStack(), + } + r.opaque.Push(opaque) message.Payload.Push(buf) message.Opts.Push(options) - message.Reports.Push(report{ - seq: seq, - }) + message.Reports.Push(r) f.p.input <- message diff --git a/rbforwarder_test.go b/rbforwarder_test.go index 0a42f50..74c8848 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -23,9 +23,9 @@ func (c *MockMiddleComponent) OnMessage( done types.Done, ) { c.Called(m) - data, _ := m.PopData() + data := m.Payload.Pop().([]byte) processedData := "-> [" + string(data) + "] <-" - m.PushData([]byte(processedData)) + m.Payload.Push([]byte(processedData)) next(m) } @@ -49,9 +49,10 @@ func (c *MockComponent) OnMessage( done types.Done, ) { c.Called(m) + if data, ok := m.Payload.Pop().([]byte); ok { + c.channel <- string(data) + } - data, _ := m.PopData() - c.channel <- string(data) done(m, c.statusCode, c.status) } @@ -90,6 +91,7 @@ func TestRBForwarder(t *testing.T) { err := rbforwarder.Produce( []byte("Hello World"), map[string]interface{}{"message_id": "test123"}, + nil, ) Convey("\"Hello World\" message should be get by the worker", func() { @@ -119,6 +121,7 @@ func TestRBForwarder(t *testing.T) { err := rbforwarder.Produce( []byte("Hello World"), map[string]interface{}{"message_id": "test123"}, + nil, ) Convey("Should error", func() { @@ -128,38 +131,32 @@ func TestRBForwarder(t *testing.T) { //////////////////////////////////////////////////////////////////////////// - Convey("When calling OnMessage() with options", func() { + Convey("When calling OnMessage() with opaque", func() { component.On("OnMessage", mock.AnythingOfType("*types.Message")) - // err := rbforwarder.Produce( - // []byte("Hello World"), - // map[string]interface{}{"option": "example_option"}, - // ) - - rbforwarder.Close() - - Convey("Should be possible to read an option", nil) - - Convey("Should not be possible to read an nonexistent option", nil) - - }) - - // //////////////////////////////////////////////////////////////////////////// - - Convey("When calling OnMessage() without options", func() { - component.On("OnMessage", mock.AnythingOfType("*types.Message")) + err := rbforwarder.Produce( + []byte("Hello World"), + nil, + "This is an opaque", + ) - // err := rbforwarder.Produce( - // []byte("Hello World"), - // nil, - // ) + Convey("Should be possible to read the opaque", func() { + So(err, ShouldBeNil) - rbforwarder.Close() + var reports int + var lastReport report + for r := range rbforwarder.GetReports() { + reports++ + lastReport = r.(report) + rbforwarder.Close() + } - Convey("Should not be possible to read the option", nil) + opaque := lastReport.opaque.Pop().(string) + So(opaque, ShouldEqual, "This is an opaque") + }) }) - // //////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////// Convey("When a message fails to send", func() { component.status = "Fake Error" @@ -170,6 +167,7 @@ func TestRBForwarder(t *testing.T) { err := rbforwarder.Produce( []byte("Hello World"), map[string]interface{}{"message_id": "test123"}, + nil, ) Convey("The message should be retried", func() { @@ -204,7 +202,8 @@ func TestRBForwarder(t *testing.T) { for i := 0; i < numMessages; i++ { if err := rbforwarder.Produce([]byte("Hello World"), - map[string]interface{}{"message_id": i}, + nil, + i, ); err != nil { numErr++ } @@ -225,27 +224,26 @@ func TestRBForwarder(t *testing.T) { component.AssertExpectations(t) }) - Convey("10000 reports should be received in order", nil) - // func() { - // ordered := true - // var reports int - // - // for rep := range rbforwarder.GetOrderedReports() { - // if rep.(report).GetOpts()["message_id"] != reports { - // ordered = false - // } - // reports++ - // if reports >= numMessages { - // rbforwarder.Close() - // } - // } - // - // So(numErr, ShouldBeZeroValue) - // So(ordered, ShouldBeTrue) - // So(reports, ShouldEqual, numMessages) - // - // component.AssertExpectations(t) - // }) + Convey("10000 reports should be received in order", func() { + ordered := true + var reports int + + for rep := range rbforwarder.GetOrderedReports() { + if rep.(report).opaque.Pop().(int) != reports { + ordered = false + } + reports++ + if reports >= numMessages { + rbforwarder.Close() + } + } + + So(numErr, ShouldBeZeroValue) + So(ordered, ShouldBeTrue) + So(reports, ShouldEqual, numMessages) + + component.AssertExpectations(t) + }) }) }) @@ -290,6 +288,7 @@ func TestRBForwarder(t *testing.T) { err := rbforwarder.Produce( []byte("Hello World"), map[string]interface{}{"message_id": "test123"}, + nil, ) rbforwarder.Close() @@ -299,7 +298,8 @@ func TestRBForwarder(t *testing.T) { for rep := range rbforwarder.GetReports() { reports++ - code, status, _ := rep.(report).Status() + code := rep.(report).code + status := rep.(report).status So(code, ShouldEqual, 0) So(status, ShouldEqual, "OK") } diff --git a/report.go b/report.go index 651c959..6d8d5ac 100644 --- a/report.go +++ b/report.go @@ -1,12 +1,12 @@ package rbforwarder +import "github.com/oleiade/lane" + type report struct { seq uint64 code int status string retries int -} -func (r report) Status() (code int, status string, retries int) { - return r.code, r.status, r.retries + opaque *lane.Stack } diff --git a/types/message.go b/types/message.go index 80a720f..f43be3e 100644 --- a/types/message.go +++ b/types/message.go @@ -1,10 +1,6 @@ package types -import ( - "errors" - - "github.com/oleiade/lane" -) +import "github.com/oleiade/lane" // Message is used to send data through the pipeline type Message struct { @@ -13,52 +9,52 @@ type Message struct { Reports *lane.Stack } -// PushData store data on an LIFO queue so the nexts handlers can use it -func (m *Message) PushData(v []byte) { - if m.Payload == nil { - m.Payload = lane.NewStack() - } - - m.Payload.Push(v) -} - -// PopData get the data stored by the previous handler -func (m *Message) PopData() (ret []byte, err error) { - if m.Payload == nil { - err = errors.New("Uninitialized payload") - return - } - - if m.Payload.Empty() { - err = errors.New("No data") - return - } - - ret = m.Payload.Pop().([]byte) - return -} - -// PopData get the data stored by the previous handler -func (m *Message) PopOpts() (ret map[string]interface{}, err error) { - if m.Opts == nil { - err = errors.New("Uninitialized options") - return - } - - if m.Opts.Empty() { - err = errors.New("No options") - return - } - - ret = m.Opts.Pop().(map[string]interface{}) - return -} - -func (m *Message) PushOpts(Opts map[string]interface{}) error { - if m.Opts == nil { - return errors.New("Uninitialized options") - } - - m.Opts.Push(Opts) - return nil -} +// // PushData store data on an LIFO queue so the nexts handlers can use it +// func (m *Message) PushData(v []byte) { +// if m.Payload == nil { +// m.Payload = lane.NewStack() +// } +// +// m.Payload.Push(v) +// } +// +// // PopData get the data stored by the previous handler +// func (m *Message) PopData() (ret []byte, err error) { +// if m.Payload == nil { +// err = errors.New("Uninitialized payload") +// return +// } +// +// if m.Payload.Empty() { +// err = errors.New("No data") +// return +// } +// +// ret = m.Payload.Pop().([]byte) +// return +// } +// +// // PopData get the data stored by the previous handler +// func (m *Message) PopOpts() (ret map[string]interface{}, err error) { +// if m.Opts == nil { +// err = errors.New("Uninitialized options") +// return +// } +// +// if m.Opts.Empty() { +// err = errors.New("No options") +// return +// } +// +// ret = m.Opts.Pop().(map[string]interface{}) +// return +// } +// +// func (m *Message) PushOpts(Opts map[string]interface{}) error { +// if m.Opts == nil { +// return errors.New("Uninitialized options") +// } +// +// m.Opts.Push(Opts) +// return nil +// } diff --git a/types/message_test.go b/types/message_test.go index c6d3b83..a3c451a 100644 --- a/types/message_test.go +++ b/types/message_test.go @@ -1,107 +1,77 @@ package types -import ( - "testing" - - "github.com/oleiade/lane" - . "github.com/smartystreets/goconvey/convey" -) - -func TestMessage(t *testing.T) { - Convey("Given a payload and some metadata", t, func() { - payload := "This is the payload" - metadata := map[string]interface{}{ - "key1": "value1", - "key2": "value2", - } - - Convey("When the data is stored in a message", func() { - m := new(Message) - m.Payload = lane.NewStack() - m.Opts = lane.NewStack() - - m.Payload.Push([]byte(payload)) - m.Opts.Push(metadata) - - Convey("Then the data can be recovered through messenger methods", func() { - data, err := m.PopData() - So(err, ShouldBeNil) - - opts, err := m.PopOpts() - So(err, ShouldBeNil) - - So(string(data), ShouldEqual, payload) - So(opts, ShouldNotBeNil) - - So(opts["key1"], ShouldEqual, "value1") - So(opts["key2"], ShouldEqual, "value2") - }) - }) - }) - - Convey("Given a message with nil data and opts", t, func() { - m := new(Message) - - Convey("When trying to get message data", func() { - data, err1 := m.PopData() - opts, err2 := m.PopOpts() - - Convey("Then should error", func() { - So(err1, ShouldNotBeNil) - So(err2, ShouldNotBeNil) - So(err1.Error(), ShouldEqual, "Uninitialized payload") - So(err2.Error(), ShouldEqual, "Uninitialized options") - So(data, ShouldBeNil) - So(opts, ShouldBeNil) - }) - }) - }) - - Convey("Given a message with no options or no data", t, func() { - m := &Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - } - - Convey("When trying to get message data", func() { - data, err1 := m.PopData() - opts, err2 := m.PopOpts() - - Convey("Then should error", func() { - So(err1, ShouldNotBeNil) - So(err2, ShouldNotBeNil) - So(err1.Error(), ShouldEqual, "No data") - So(err2.Error(), ShouldEqual, "No options") - So(data, ShouldBeNil) - So(opts, ShouldBeNil) - }) - }) - }) - - Convey("Given a delivered message", t, func() { - m := &Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), - } - - m.Reports.Push("This is a report") - m.Reports.Push("This is a report") - m.Reports.Push("This is a report") - m.Payload.Push([]byte("This message is delivered")) - m.Opts.Push(map[string]interface{}{ - "key1": "value1", - "key2": "value2", - }) - - Convey("Then a report can be obtained for the message", func() { - var i int - for !m.Reports.Empty() { - i++ - rep := m.Reports.Pop().(string) - So(rep, ShouldEqual, "This is a report") - } - So(i, ShouldEqual, 3) - }) - }) -} +// import . "github.com/smartystreets/goconvey/convey" + +// func TestMessage(t *testing.T) { +// Convey("Given a payload and some metadata", t, func() { +// payload := "This is the payload" +// metadata := map[string]interface{}{ +// "key1": "value1", +// "key2": "value2", +// } +// +// Convey("When the data is stored in a message", func() { +// m := new(Message) +// m.Payload = lane.NewStack() +// m.Opts = lane.NewStack() +// +// m.Payload.Push([]byte(payload)) +// m.Opts.Push(metadata) +// +// Convey("Then the data can be recovered through messenger methods", func() { +// data := m.Payload.Pop().([]byte) +// opts := m.Opts.Pop().(map[string]interface{}) +// +// So(string(data), ShouldEqual, payload) +// So(opts, ShouldNotBeNil) +// +// So(opts["key1"], ShouldEqual, "value1") +// So(opts["key2"], ShouldEqual, "value2") +// }) +// }) +// }) +// +// Convey("Given a message with no options or no data", t, func() { +// m := &Message{ +// Payload: lane.NewStack(), +// Opts: lane.NewStack(), +// } +// +// Convey("When trying to get message data", func() { +// _, dataOk := m.Payload.Pop().([]byte) +// _, optsOk := m.Opts.Pop().(map[string]interface{}) +// +// Convey("Then should error", func() { +// So(dataOk, ShouldBeFalse) +// So(optsOk, ShouldBeFalse) +// }) +// }) +// }) +// +// Convey("Given a delivered message", t, func() { +// m := &Message{ +// Payload: lane.NewStack(), +// Opts: lane.NewStack(), +// Reports: lane.NewStack(), +// } +// +// m.Reports.Push("This is a report") +// m.Reports.Push("This is a report") +// m.Reports.Push("This is a report") +// m.Payload.Push([]byte("This message is delivered")) +// m.Opts.Push(map[string]interface{}{ +// "key1": "value1", +// "key2": "value2", +// }) +// +// Convey("Then a report can be obtained for the message", func() { +// var i int +// for !m.Reports.Empty() { +// i++ +// rep := m.Reports.Pop().(string) +// So(rep, ShouldEqual, "This is a report") +// } +// So(i, ShouldEqual, 3) +// }) +// }) +// } From b2de50a607675d9bcdecf2b8f68d7655c8a70423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 10 Aug 2016 11:19:26 +0200 Subject: [PATCH 11/36] :sparkles: Make message payload private It is now exposed via a method. Assertions are not necessary anymore and there is error handling. --- components/batch/batch.go | 16 ++-- components/batch/batcher.go | 20 +++-- components/batch/batcher_test.go | 132 +++++++++++++------------------ rbforwarder.go | 17 ++-- rbforwarder_test.go | 12 ++- types/message.go | 84 ++++++++------------ types/message_test.go | 108 ++++++++----------------- 7 files changed, 149 insertions(+), 240 deletions(-) diff --git a/components/batch/batch.go b/components/batch/batch.go index 65b1e50..158b0c2 100644 --- a/components/batch/batch.go +++ b/components/batch/batch.go @@ -12,6 +12,7 @@ import ( type Batch struct { Group string Message *types.Message + Buff *bytes.Buffer MessageCount uint // Current number of messages in the buffer Next types.Next // Call to pass the message to the next handler } @@ -19,11 +20,13 @@ type Batch struct { // NewBatch creates a new instance of Batch func NewBatch(m *types.Message, group string, next types.Next, clk clock.Clock, timeoutMillis uint, ready chan *Batch) *Batch { + payload, _ := m.PopPayload() b := &Batch{ Group: group, Next: next, Message: m, MessageCount: 1, + Buff: bytes.NewBuffer(payload), } if timeoutMillis != 0 { @@ -42,23 +45,18 @@ func NewBatch(m *types.Message, group string, next types.Next, clk clock.Clock, // Send the batch of messages to the next handler in the pipeline func (b *Batch) Send(cb func()) { + b.Message.PushPayload(b.Buff.Bytes()) cb() b.Next(b.Message) } // Add merges a new message in the buffer func (b *Batch) Add(m *types.Message) { - newPayload := m.Payload.Pop().([]byte) - newOptions := m.Opts.Pop().(map[string]interface{}) newReport := m.Reports.Pop() - - currentPayload := b.Message.Payload.Pop().([]byte) - buff := bytes.NewBuffer(currentPayload) - buff.Write(newPayload) - - b.Message.Payload.Push(buff.Bytes()) - b.Message.Opts.Push(newOptions) b.Message.Reports.Push(newReport) + newPayload, _ := m.PopPayload() + b.Buff.Write(newPayload) + b.MessageCount++ } diff --git a/components/batch/batcher.go b/components/batch/batcher.go index 259d8af..eaf4fcf 100644 --- a/components/batch/batcher.go +++ b/components/batch/batcher.go @@ -36,19 +36,17 @@ func (b *Batcher) Init(id int) { // OnMessage is called when a new message is receive. Add the new message to // a batch func (b *Batcher) OnMessage(m *types.Message, next types.Next, done types.Done) { - if opts, ok := m.Opts.Head().(map[string]interface{}); ok { - if group, exists := opts["batch_group"].(string); exists { - if batch, exists := b.batches[group]; exists { - batch.Add(m) - if batch.MessageCount >= b.config.Limit { - b.readyBatches <- batch - } - } else { - b.batches[group] = NewBatch(m, group, next, b.clk, b.config.TimeoutMillis, b.readyBatches) + if group, exists := m.Opts["batch_group"].(string); exists { + if batch, exists := b.batches[group]; exists { + batch.Add(m) + if batch.MessageCount >= b.config.Limit { + b.readyBatches <- batch } - - return + } else { + b.batches[group] = NewBatch(m, group, next, b.clk, b.config.TimeoutMillis, b.readyBatches) } + + return } next(m) diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go index 7ab0915..81c97e9 100644 --- a/components/batch/batcher_test.go +++ b/components/batch/batcher_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/benbjohnson/clock" - "github.com/oleiade/lane" "github.com/redBorder/rbforwarder/types" . "github.com/smartystreets/goconvey/convey" "github.com/stretchr/testify/mock" @@ -35,12 +34,8 @@ func TestBatcher(t *testing.T) { batcher.clk = clock.NewMock() Convey("When a message is received with no batch group", func() { - m := &types.Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), - } - m.Payload.Push([]byte("Hello World")) + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) nd := new(NexterDoner) nd.nextCalled = make(chan *types.Message, 1) @@ -52,36 +47,35 @@ func TestBatcher(t *testing.T) { nd.AssertExpectations(t) m := <-nd.nextCalled So(len(batcher.batches), ShouldEqual, 0) - So(string(m.Payload.Pop().([]byte)), ShouldEqual, "Hello World") + payload, err := m.PopPayload() + So(err, ShouldBeNil) + So(string(payload), ShouldEqual, "Hello World") }) }) Convey("When a message is received, but not yet sent", func() { - m := &types.Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), - } - m.Payload.Push([]byte("Hello World")) - m.Opts.Push(map[string]interface{}{ + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) + m.Opts = map[string]interface{}{ "batch_group": "group1", - }) + } m.Reports.Push("Report") batcher.OnMessage(m, nil, nil) Convey("Message should be present on the batch", func() { - var batch *Batch - var exists bool - batch, exists = batcher.batches["group1"] - + batch, exists := batcher.batches["group1"] So(exists, ShouldBeTrue) - data := batch.Message.Payload.Pop().([]byte) - opts := batch.Message.Opts.Pop().(map[string]interface{}) - report := batch.Message.Reports.Pop().(string) + + data := batch.Buff.Bytes() So(string(data), ShouldEqual, "Hello World") + + opts := batch.Message.Opts So(opts["batch_group"], ShouldEqual, "group1") + + report := batch.Message.Reports.Pop().(string) So(report, ShouldEqual, "Report") + So(len(batcher.batches), ShouldEqual, 1) }) }) @@ -90,15 +84,11 @@ func TestBatcher(t *testing.T) { var messages []*types.Message for i := 0; i < int(batcher.config.Limit); i++ { - m := &types.Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), - } - m.Payload.Push([]byte("ABC")) - m.Opts.Push(map[string]interface{}{ + m := types.NewMessage() + m.PushPayload([]byte("ABC")) + m.Opts = map[string]interface{}{ "batch_group": "group1", - }) + } m.Reports.Push("Report") messages = append(messages, m) @@ -115,13 +105,11 @@ func TestBatcher(t *testing.T) { Convey("The batch should be sent", func() { m := <-nd.nextCalled nd.AssertExpectations(t) - data := m.Payload.Pop().([]byte) - optsSize := m.Opts.Size() - reportsSize := m.Reports.Size() + data, err := m.PopPayload() + + So(err, ShouldBeNil) So(string(data), ShouldEqual, "ABCABCABCABCABCABCABCABCABCABC") - So(m.Payload.Empty(), ShouldBeTrue) - So(optsSize, ShouldEqual, batcher.config.Limit) - So(reportsSize, ShouldEqual, batcher.config.Limit) + So(m.Reports.Size(), ShouldEqual, batcher.config.Limit) So(batcher.batches["group1"], ShouldBeNil) So(len(batcher.batches), ShouldEqual, 0) }) @@ -131,15 +119,11 @@ func TestBatcher(t *testing.T) { var messages []*types.Message for i := 0; i < 5; i++ { - m := &types.Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), - } - m.Payload.Push([]byte("ABC")) - m.Opts.Push(map[string]interface{}{ + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) + m.Opts = map[string]interface{}{ "batch_group": "group1", - }) + } m.Reports.Push("Report") messages = append(messages, m) @@ -167,35 +151,21 @@ func TestBatcher(t *testing.T) { }) Convey("When multiple messages are received with differente groups", func() { - m1 := &types.Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), - } - m1.Payload.Push([]byte("MESSAGE 1")) - m1.Opts.Push(map[string]interface{}{ + m1 := types.NewMessage() + m1.PushPayload([]byte("MESSAGE 1")) + m1.Opts = map[string]interface{}{ "batch_group": "group1", - }) - - m2 := &types.Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), } - m2.Payload.Push([]byte("MESSAGE 2")) - m2.Opts.Push(map[string]interface{}{ + m2 := types.NewMessage() + m2.PushPayload([]byte("MESSAGE 2")) + m2.Opts = map[string]interface{}{ "batch_group": "group2", - }) - - m3 := &types.Message{ - Payload: lane.NewStack(), - Opts: lane.NewStack(), - Reports: lane.NewStack(), } - m3.Payload.Push([]byte("MESSAGE 3")) - m3.Opts.Push(map[string]interface{}{ + m3 := types.NewMessage() + m3.PushPayload([]byte("MESSAGE 3")) + m3.Opts = map[string]interface{}{ "batch_group": "group2", - }) + } nd := new(NexterDoner) nd.nextCalled = make(chan *types.Message, 2) @@ -206,31 +176,35 @@ func TestBatcher(t *testing.T) { batcher.OnMessage(m3, nd.Next, nil) Convey("Each message should be in its group", func() { - var err error - group1 := batcher.batches["group1"].Message.Payload.Pop().([]byte) - So(err, ShouldBeNil) - - group2 := batcher.batches["group2"].Message.Payload.Pop().([]byte) - So(err, ShouldBeNil) - + group1 := batcher.batches["group1"].Buff.Bytes() So(string(group1), ShouldEqual, "MESSAGE 1") + + group2 := batcher.batches["group2"].Buff.Bytes() So(string(group2), ShouldEqual, "MESSAGE 2MESSAGE 3") + So(len(batcher.batches), ShouldEqual, 2) }) Convey("After a timeout the messages should be sent", func() { clk := batcher.clk.(*clock.Mock) So(len(batcher.batches), ShouldEqual, 2) + clk.Add(time.Duration(batcher.config.TimeoutMillis) * time.Millisecond) + group1 := <-nd.nextCalled - group1Data := group1.Payload.Pop().([]byte) - So(string(group1Data), ShouldEqual, "MESSAGE 1") + group1Data, err := group1.PopPayload() + So(err, ShouldBeNil) + group2 := <-nd.nextCalled - group2Data := group2.Payload.Pop().([]byte) + group2Data, err := group2.PopPayload() + So(err, ShouldBeNil) + + So(string(group1Data), ShouldEqual, "MESSAGE 1") So(string(group2Data), ShouldEqual, "MESSAGE 2MESSAGE 3") So(batcher.batches["group1"], ShouldBeNil) So(batcher.batches["group2"], ShouldBeNil) So(len(batcher.batches), ShouldEqual, 0) + nd.AssertExpectations(t) }) }) diff --git a/rbforwarder.go b/rbforwarder.go index 142c95c..18e5881 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -82,30 +82,25 @@ func (f *RBForwarder) GetOrderedReports() <-chan interface{} { } // Produce is used by the source to send messages to the backend -func (f *RBForwarder) Produce(buf []byte, options map[string]interface{}, opaque interface{}) error { +func (f *RBForwarder) Produce(data []byte, opts map[string]interface{}, opaque interface{}) error { if atomic.LoadUint32(&f.working) == 0 { return errors.New("Forwarder has been closed") } seq := f.currentProducedID f.currentProducedID++ - - message := &types.Message{ - Payload: lane.NewStack(), - Reports: lane.NewStack(), - Opts: lane.NewStack(), - } + m := types.NewMessage() r := report{ seq: seq, opaque: lane.NewStack(), } + m.PushPayload(data) + m.Opts = opts + m.Reports.Push(r) r.opaque.Push(opaque) - message.Payload.Push(buf) - message.Opts.Push(options) - message.Reports.Push(r) - f.p.input <- message + f.p.input <- m return nil } diff --git a/rbforwarder_test.go b/rbforwarder_test.go index 74c8848..88a012e 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -23,9 +23,11 @@ func (c *MockMiddleComponent) OnMessage( done types.Done, ) { c.Called(m) - data := m.Payload.Pop().([]byte) - processedData := "-> [" + string(data) + "] <-" - m.Payload.Push([]byte(processedData)) + if data, err := m.PopPayload(); err == nil { + processedData := "-> [" + string(data) + "] <-" + m.PushPayload([]byte(processedData)) + } + next(m) } @@ -49,8 +51,10 @@ func (c *MockComponent) OnMessage( done types.Done, ) { c.Called(m) - if data, ok := m.Payload.Pop().([]byte); ok { + if data, err := m.PopPayload(); err == nil { c.channel <- string(data) + } else { + c.channel <- err.Error() } done(m, c.statusCode, c.status) diff --git a/types/message.go b/types/message.go index f43be3e..bb1a412 100644 --- a/types/message.go +++ b/types/message.go @@ -1,60 +1,40 @@ package types -import "github.com/oleiade/lane" +import ( + "errors" + + "github.com/oleiade/lane" +) // Message is used to send data through the pipeline type Message struct { - Payload *lane.Stack - Opts *lane.Stack + Opts map[string]interface{} Reports *lane.Stack + + payload *lane.Stack } -// // PushData store data on an LIFO queue so the nexts handlers can use it -// func (m *Message) PushData(v []byte) { -// if m.Payload == nil { -// m.Payload = lane.NewStack() -// } -// -// m.Payload.Push(v) -// } -// -// // PopData get the data stored by the previous handler -// func (m *Message) PopData() (ret []byte, err error) { -// if m.Payload == nil { -// err = errors.New("Uninitialized payload") -// return -// } -// -// if m.Payload.Empty() { -// err = errors.New("No data") -// return -// } -// -// ret = m.Payload.Pop().([]byte) -// return -// } -// -// // PopData get the data stored by the previous handler -// func (m *Message) PopOpts() (ret map[string]interface{}, err error) { -// if m.Opts == nil { -// err = errors.New("Uninitialized options") -// return -// } -// -// if m.Opts.Empty() { -// err = errors.New("No options") -// return -// } -// -// ret = m.Opts.Pop().(map[string]interface{}) -// return -// } -// -// func (m *Message) PushOpts(Opts map[string]interface{}) error { -// if m.Opts == nil { -// return errors.New("Uninitialized options") -// } -// -// m.Opts.Push(Opts) -// return nil -// } +// NewMessage creates a new instance of Message +func NewMessage() *Message { + return &Message{ + payload: lane.NewStack(), + Reports: lane.NewStack(), + } +} + +// PushPayload store data on an LIFO queue so the nexts handlers can use it +func (m *Message) PushPayload(data []byte) { + m.payload.Push(data) +} + +// PopPayload get the data stored by the previous handler +func (m *Message) PopPayload() (data []byte, err error) { + if m.payload.Empty() { + err = errors.New("No payload available") + return + } + + data = m.payload.Pop().([]byte) + + return +} diff --git a/types/message_test.go b/types/message_test.go index a3c451a..d5ad5e1 100644 --- a/types/message_test.go +++ b/types/message_test.go @@ -1,77 +1,37 @@ package types -// import . "github.com/smartystreets/goconvey/convey" +import ( + "testing" -// func TestMessage(t *testing.T) { -// Convey("Given a payload and some metadata", t, func() { -// payload := "This is the payload" -// metadata := map[string]interface{}{ -// "key1": "value1", -// "key2": "value2", -// } -// -// Convey("When the data is stored in a message", func() { -// m := new(Message) -// m.Payload = lane.NewStack() -// m.Opts = lane.NewStack() -// -// m.Payload.Push([]byte(payload)) -// m.Opts.Push(metadata) -// -// Convey("Then the data can be recovered through messenger methods", func() { -// data := m.Payload.Pop().([]byte) -// opts := m.Opts.Pop().(map[string]interface{}) -// -// So(string(data), ShouldEqual, payload) -// So(opts, ShouldNotBeNil) -// -// So(opts["key1"], ShouldEqual, "value1") -// So(opts["key2"], ShouldEqual, "value2") -// }) -// }) -// }) -// -// Convey("Given a message with no options or no data", t, func() { -// m := &Message{ -// Payload: lane.NewStack(), -// Opts: lane.NewStack(), -// } -// -// Convey("When trying to get message data", func() { -// _, dataOk := m.Payload.Pop().([]byte) -// _, optsOk := m.Opts.Pop().(map[string]interface{}) -// -// Convey("Then should error", func() { -// So(dataOk, ShouldBeFalse) -// So(optsOk, ShouldBeFalse) -// }) -// }) -// }) -// -// Convey("Given a delivered message", t, func() { -// m := &Message{ -// Payload: lane.NewStack(), -// Opts: lane.NewStack(), -// Reports: lane.NewStack(), -// } -// -// m.Reports.Push("This is a report") -// m.Reports.Push("This is a report") -// m.Reports.Push("This is a report") -// m.Payload.Push([]byte("This message is delivered")) -// m.Opts.Push(map[string]interface{}{ -// "key1": "value1", -// "key2": "value2", -// }) -// -// Convey("Then a report can be obtained for the message", func() { -// var i int -// for !m.Reports.Empty() { -// i++ -// rep := m.Reports.Pop().(string) -// So(rep, ShouldEqual, "This is a report") -// } -// So(i, ShouldEqual, 3) -// }) -// }) -// } + . "github.com/smartystreets/goconvey/convey" +) + +func TestMessage(t *testing.T) { + Convey("Given a payload", t, func() { + payload := "This is the payload" + + Convey("When the data is stored in a message", func() { + m := NewMessage() + m.PushPayload([]byte(payload)) + + Convey("Then the data can be recovered through messenger methods", func() { + data, err := m.PopPayload() + So(err, ShouldBeNil) + So(string(data), ShouldEqual, payload) + }) + }) + }) + + Convey("Given a message with no options or no data", t, func() { + m := NewMessage() + + Convey("When trying to get message data", func() { + _, err := m.PopPayload() + + Convey("Then should error", func() { + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "No payload available") + }) + }) + }) +} From 19f84a553d9a2bdeb9f618554d3ebe1198f0b73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 10 Aug 2016 13:01:54 +0200 Subject: [PATCH 12/36] :bug: Fix missing initialization of map --- types/message.go | 1 + types/message_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/types/message.go b/types/message.go index bb1a412..d1593ee 100644 --- a/types/message.go +++ b/types/message.go @@ -19,6 +19,7 @@ func NewMessage() *Message { return &Message{ payload: lane.NewStack(), Reports: lane.NewStack(), + Opts: make(map[string]interface{}), } } diff --git a/types/message_test.go b/types/message_test.go index d5ad5e1..c4406ea 100644 --- a/types/message_test.go +++ b/types/message_test.go @@ -7,6 +7,19 @@ import ( ) func TestMessage(t *testing.T) { + Convey("Given a message", t, func() { + + Convey("When fields are accessed", func() { + m := NewMessage() + + Convey("Then fields has to be initialized", func() { + So(m.Opts, ShouldNotBeNil) + So(m.payload, ShouldNotBeNil) + So(m.Reports, ShouldNotBeNil) + }) + }) + }) + Convey("Given a payload", t, func() { payload := "This is the payload" From 544cacfd28bd9be9e6a0acb93d0a20d9d4c56bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 10 Aug 2016 13:05:12 +0200 Subject: [PATCH 13/36] :sparkles: Add http sender component Closes #3 --- components/httpsender/httpSender.go | 71 +++++++ components/httpsender/httpSender_test.go | 251 +++++++++++++++++++++++ 2 files changed, 322 insertions(+) create mode 100644 components/httpsender/httpSender.go create mode 100644 components/httpsender/httpSender_test.go diff --git a/components/httpsender/httpSender.go b/components/httpsender/httpSender.go new file mode 100644 index 0000000..6505806 --- /dev/null +++ b/components/httpsender/httpSender.go @@ -0,0 +1,71 @@ +package httpsender + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "net/http" + + "github.com/asaskevich/govalidator" + "github.com/redBorder/rbforwarder/types" +) + +// HTTPSender is a component for the rbforwarder pipeline that sends messages +// to an HTTP endpoint. It's a final component, so it will call Done() instead +// of Next() and further components shuld not be added after this component. +type HTTPSender struct { + id int + err error + URL string + client *http.Client +} + +// Init initializes the HTTP component +func (s *HTTPSender) Init(id int) { + s.id = id + + if govalidator.IsURL(s.URL) { + s.client = &http.Client{} + } else { + s.err = errors.New("Invalid URL") + } +} + +// OnMessage is called when a new message should be sent via HTTP +func (s *HTTPSender) OnMessage(m *types.Message, next types.Next, done types.Done) { + var u string + + if s.err != nil { + done(m, 2, s.err.Error()) + return + } + + data, err := m.PopPayload() + if err != nil { + done(m, 3, "Can't get payload of message: "+err.Error()) + return + } + + if endpoint, exists := m.Opts["http_endpoint"]; exists { + u = s.URL + "/" + endpoint.(string) + } else { + u = s.URL + } + + buf := bytes.NewBuffer(data) + res, err := s.client.Post(u, "", buf) + if err != nil { + done(m, 1, "HTTPSender error: "+err.Error()) + return + } + io.Copy(ioutil.Discard, res.Body) + res.Body.Close() + + if res.StatusCode >= 400 { + done(m, res.StatusCode, "HTTPSender error: "+res.Status) + return + } + + done(m, 0, res.Status) +} diff --git a/components/httpsender/httpSender_test.go b/components/httpsender/httpSender_test.go new file mode 100644 index 0000000..e1a4bb1 --- /dev/null +++ b/components/httpsender/httpSender_test.go @@ -0,0 +1,251 @@ +package httpsender + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/redBorder/rbforwarder/types" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/mock" +) + +type Doner struct { + mock.Mock + doneCalled chan struct { + code int + status string + } +} + +func (d *Doner) Done(m *types.Message, code int, status string) { + d.Called(m, code, status) + d.doneCalled <- struct { + code int + status string + }{ + code, + status, + } +} + +func NewTestClient(code int, cb func(*http.Request)) *http.Client { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + cb(r) + })) + + transport := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + return &http.Client{Transport: transport} +} + +func TestHTTPSender(t *testing.T) { + Convey("Given an HTTP sender with defined URL", t, func() { + sender := &HTTPSender{ + URL: "http://example.com", + } + + Convey("When is initialized", func() { + sender.Init(0) + + Convey("Then the config should be ok", func() { + So(sender.client, ShouldNotBeNil) + }) + }) + + Convey("When a message is sent and the response code is >= 400", func() { + var url string + sender.Init(0) + + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) + sender.client = NewTestClient(401, func(req *http.Request) { + url = req.URL.String() + }) + + d := &Doner{ + doneCalled: make(chan struct { + code int + status string + }, 1), + } + + d.On("Done", mock.AnythingOfType("*types.Message"), + mock.AnythingOfType("int"), mock.AnythingOfType("string")) + + sender.OnMessage(m, nil, d.Done) + + Convey("Then the reporth should contain info about the error", func() { + result := <-d.doneCalled + So(result.status, ShouldEqual, "HTTPSender error: 401 Unauthorized") + So(result.code, ShouldEqual, 401) + So(url, ShouldEqual, "http://example.com/") + + d.AssertExpectations(t) + }) + }) + + Convey("When a message is received without endpoint option", func() { + var url string + sender.Init(0) + + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) + + sender.client = NewTestClient(200, func(req *http.Request) { + url = req.URL.String() + }) + + d := &Doner{ + doneCalled: make(chan struct { + code int + status string + }, 1), + } + d.On("Done", mock.AnythingOfType("*types.Message"), + mock.AnythingOfType("int"), mock.AnythingOfType("string")) + + sender.OnMessage(m, nil, d.Done) + + Convey("Then the message should be sent via HTTP to the URL", func() { + result := <-d.doneCalled + So(result.status, ShouldEqual, "200 OK") + So(result.code, ShouldBeZeroValue) + So(url, ShouldEqual, "http://example.com/") + + d.AssertExpectations(t) + }) + }) + + Convey("When a message is received with endpoint option", func() { + var url string + sender.Init(0) + + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) + m.Opts["http_endpoint"] = "endpoint1" + + sender.client = NewTestClient(200, func(req *http.Request) { + url = req.URL.String() + }) + + d := &Doner{ + doneCalled: make(chan struct { + code int + status string + }, 1), + } + d.On("Done", mock.AnythingOfType("*types.Message"), + mock.AnythingOfType("int"), mock.AnythingOfType("string")) + + sender.OnMessage(m, nil, d.Done) + + Convey("Then the message should be sent to the URL with endpoint as suffix", func() { + result := <-d.doneCalled + So(result.status, ShouldEqual, "200 OK") + So(result.code, ShouldBeZeroValue) + So(url, ShouldEqual, "http://example.com/endpoint1") + + d.AssertExpectations(t) + }) + }) + + Convey("When a message without payload is received", func() { + var url string + sender.Init(0) + + m := types.NewMessage() + + sender.client = NewTestClient(200, func(req *http.Request) { + url = req.URL.String() + }) + + d := &Doner{ + doneCalled: make(chan struct { + code int + status string + }, 1), + } + d.On("Done", mock.AnythingOfType("*types.Message"), + mock.AnythingOfType("int"), mock.AnythingOfType("string")) + + sender.OnMessage(m, nil, d.Done) + + Convey("Then the message should not be sent", func() { + result := <-d.doneCalled + So(result.status, ShouldEqual, "Can't get payload of message: No payload available") + So(result.code, ShouldBeGreaterThan, 0) + So(url, ShouldBeEmpty) + + d.AssertExpectations(t) + }) + }) + + Convey("When a the HTTP client fails", func() { + sender.Init(0) + + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) + + sender.client = NewTestClient(200, func(req *http.Request) { + req.Write(nil) + }) + + d := &Doner{ + doneCalled: make(chan struct { + code int + status string + }, 1), + } + d.On("Done", mock.AnythingOfType("*types.Message"), + mock.AnythingOfType("int"), mock.AnythingOfType("string")) + + sender.OnMessage(m, nil, d.Done) + + Convey("Then the message should not be sent", func() { + result := <-d.doneCalled + So(result.status, ShouldEqual, "HTTPSender error: Post http://example.com: EOF") + So(result.code, ShouldBeGreaterThan, 0) + + d.AssertExpectations(t) + }) + }) + }) + + Convey("Given an HTTP sender with invalid URL", t, func() { + sender := &HTTPSender{} + sender.Init(0) + + Convey("When try to send messages", func() { + m := types.NewMessage() + m.PushPayload([]byte("Hello World")) + m.Opts["http_endpoint"] = "endpoint1" + + d := &Doner{ + doneCalled: make(chan struct { + code int + status string + }, 1), + } + + d.On("Done", mock.AnythingOfType("*types.Message"), + mock.AnythingOfType("int"), mock.AnythingOfType("string")) + + sender.OnMessage(m, nil, d.Done) + + Convey("Then should fail to send messages", func() { + So(sender.err, ShouldNotBeNil) + result := <-d.doneCalled + So(result.status, ShouldEqual, "Invalid URL") + So(result.code, ShouldBeGreaterThan, 0) + }) + }) + }) +} From 989a4191362197bdc038c23cecd521caf2534968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Thu, 11 Aug 2016 11:06:25 +0200 Subject: [PATCH 14/36] :lipstick: Rename types -> utils --- components/batch/batch.go | 10 +++--- components/batch/batcher.go | 4 +-- components/batch/batcher_test.go | 40 ++++++++++++------------ components/httpsender/httpSender.go | 4 +-- components/httpsender/httpSender_test.go | 28 ++++++++--------- pipeline.go | 22 ++++++------- rbforwarder.go | 12 +++---- rbforwarder_test.go | 30 +++++++++--------- reportHandler.go | 14 ++++----- {types => utils}/composer.go | 2 +- {types => utils}/message.go | 2 +- {types => utils}/message_test.go | 2 +- 12 files changed, 85 insertions(+), 85 deletions(-) rename {types => utils}/composer.go (97%) rename {types => utils}/message.go (98%) rename {types => utils}/message_test.go (98%) diff --git a/components/batch/batch.go b/components/batch/batch.go index 158b0c2..edd2b89 100644 --- a/components/batch/batch.go +++ b/components/batch/batch.go @@ -5,20 +5,20 @@ import ( "time" "github.com/benbjohnson/clock" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" ) // Batch groups multiple messages type Batch struct { Group string - Message *types.Message + Message *utils.Message Buff *bytes.Buffer MessageCount uint // Current number of messages in the buffer - Next types.Next // Call to pass the message to the next handler + Next utils.Next // Call to pass the message to the next handler } // NewBatch creates a new instance of Batch -func NewBatch(m *types.Message, group string, next types.Next, clk clock.Clock, +func NewBatch(m *utils.Message, group string, next utils.Next, clk clock.Clock, timeoutMillis uint, ready chan *Batch) *Batch { payload, _ := m.PopPayload() b := &Batch{ @@ -51,7 +51,7 @@ func (b *Batch) Send(cb func()) { } // Add merges a new message in the buffer -func (b *Batch) Add(m *types.Message) { +func (b *Batch) Add(m *utils.Message) { newReport := m.Reports.Pop() b.Message.Reports.Push(newReport) diff --git a/components/batch/batcher.go b/components/batch/batcher.go index eaf4fcf..e36ccc4 100644 --- a/components/batch/batcher.go +++ b/components/batch/batcher.go @@ -2,7 +2,7 @@ package batcher import ( "github.com/benbjohnson/clock" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" ) // Batcher allows to merge multiple messages in a single one @@ -35,7 +35,7 @@ func (b *Batcher) Init(id int) { // OnMessage is called when a new message is receive. Add the new message to // a batch -func (b *Batcher) OnMessage(m *types.Message, next types.Next, done types.Done) { +func (b *Batcher) OnMessage(m *utils.Message, next utils.Next, done utils.Done) { if group, exists := m.Opts["batch_group"].(string); exists { if batch, exists := b.batches[group]; exists { batch.Add(m) diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go index 81c97e9..c9280bb 100644 --- a/components/batch/batcher_test.go +++ b/components/batch/batcher_test.go @@ -5,17 +5,17 @@ import ( "time" "github.com/benbjohnson/clock" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" . "github.com/smartystreets/goconvey/convey" "github.com/stretchr/testify/mock" ) type NexterDoner struct { mock.Mock - nextCalled chan *types.Message + nextCalled chan *utils.Message } -func (nd *NexterDoner) Next(m *types.Message) { +func (nd *NexterDoner) Next(m *utils.Message) { nd.Called(m) nd.nextCalled <- m } @@ -34,12 +34,12 @@ func TestBatcher(t *testing.T) { batcher.clk = clock.NewMock() Convey("When a message is received with no batch group", func() { - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) nd := new(NexterDoner) - nd.nextCalled = make(chan *types.Message, 1) - nd.On("Next", mock.AnythingOfType("*types.Message")).Times(1) + nd.nextCalled = make(chan *utils.Message, 1) + nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) batcher.OnMessage(m, nd.Next, nil) @@ -54,7 +54,7 @@ func TestBatcher(t *testing.T) { }) Convey("When a message is received, but not yet sent", func() { - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) m.Opts = map[string]interface{}{ "batch_group": "group1", @@ -81,10 +81,10 @@ func TestBatcher(t *testing.T) { }) Convey("When the max number of messages is reached", func() { - var messages []*types.Message + var messages []*utils.Message for i := 0; i < int(batcher.config.Limit); i++ { - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("ABC")) m.Opts = map[string]interface{}{ "batch_group": "group1", @@ -95,8 +95,8 @@ func TestBatcher(t *testing.T) { } nd := new(NexterDoner) - nd.nextCalled = make(chan *types.Message) - nd.On("Next", mock.AnythingOfType("*types.Message")).Times(1) + nd.nextCalled = make(chan *utils.Message) + nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) for i := 0; i < int(batcher.config.Limit); i++ { batcher.OnMessage(messages[i], nd.Next, nil) @@ -116,10 +116,10 @@ func TestBatcher(t *testing.T) { }) Convey("When the timeout expires", func() { - var messages []*types.Message + var messages []*utils.Message for i := 0; i < 5; i++ { - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) m.Opts = map[string]interface{}{ "batch_group": "group1", @@ -130,8 +130,8 @@ func TestBatcher(t *testing.T) { } nd := new(NexterDoner) - nd.nextCalled = make(chan *types.Message, 1) - nd.On("Next", mock.AnythingOfType("*types.Message")).Times(1) + nd.nextCalled = make(chan *utils.Message, 1) + nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) for i := 0; i < 5; i++ { batcher.OnMessage(messages[i], nd.Next, nil) @@ -151,25 +151,25 @@ func TestBatcher(t *testing.T) { }) Convey("When multiple messages are received with differente groups", func() { - m1 := types.NewMessage() + m1 := utils.NewMessage() m1.PushPayload([]byte("MESSAGE 1")) m1.Opts = map[string]interface{}{ "batch_group": "group1", } - m2 := types.NewMessage() + m2 := utils.NewMessage() m2.PushPayload([]byte("MESSAGE 2")) m2.Opts = map[string]interface{}{ "batch_group": "group2", } - m3 := types.NewMessage() + m3 := utils.NewMessage() m3.PushPayload([]byte("MESSAGE 3")) m3.Opts = map[string]interface{}{ "batch_group": "group2", } nd := new(NexterDoner) - nd.nextCalled = make(chan *types.Message, 2) - nd.On("Next", mock.AnythingOfType("*types.Message")).Times(2) + nd.nextCalled = make(chan *utils.Message, 2) + nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(2) batcher.OnMessage(m1, nd.Next, nil) batcher.OnMessage(m2, nd.Next, nil) diff --git a/components/httpsender/httpSender.go b/components/httpsender/httpSender.go index 6505806..ee3bd1e 100644 --- a/components/httpsender/httpSender.go +++ b/components/httpsender/httpSender.go @@ -8,7 +8,7 @@ import ( "net/http" "github.com/asaskevich/govalidator" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" ) // HTTPSender is a component for the rbforwarder pipeline that sends messages @@ -33,7 +33,7 @@ func (s *HTTPSender) Init(id int) { } // OnMessage is called when a new message should be sent via HTTP -func (s *HTTPSender) OnMessage(m *types.Message, next types.Next, done types.Done) { +func (s *HTTPSender) OnMessage(m *utils.Message, next utils.Next, done utils.Done) { var u string if s.err != nil { diff --git a/components/httpsender/httpSender_test.go b/components/httpsender/httpSender_test.go index e1a4bb1..be0c950 100644 --- a/components/httpsender/httpSender_test.go +++ b/components/httpsender/httpSender_test.go @@ -6,7 +6,7 @@ import ( "net/url" "testing" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" . "github.com/smartystreets/goconvey/convey" "github.com/stretchr/testify/mock" ) @@ -19,7 +19,7 @@ type Doner struct { } } -func (d *Doner) Done(m *types.Message, code int, status string) { +func (d *Doner) Done(m *utils.Message, code int, status string) { d.Called(m, code, status) d.doneCalled <- struct { code int @@ -64,7 +64,7 @@ func TestHTTPSender(t *testing.T) { var url string sender.Init(0) - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) sender.client = NewTestClient(401, func(req *http.Request) { url = req.URL.String() @@ -77,7 +77,7 @@ func TestHTTPSender(t *testing.T) { }, 1), } - d.On("Done", mock.AnythingOfType("*types.Message"), + d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) sender.OnMessage(m, nil, d.Done) @@ -96,7 +96,7 @@ func TestHTTPSender(t *testing.T) { var url string sender.Init(0) - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) sender.client = NewTestClient(200, func(req *http.Request) { @@ -109,7 +109,7 @@ func TestHTTPSender(t *testing.T) { status string }, 1), } - d.On("Done", mock.AnythingOfType("*types.Message"), + d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) sender.OnMessage(m, nil, d.Done) @@ -128,7 +128,7 @@ func TestHTTPSender(t *testing.T) { var url string sender.Init(0) - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) m.Opts["http_endpoint"] = "endpoint1" @@ -142,7 +142,7 @@ func TestHTTPSender(t *testing.T) { status string }, 1), } - d.On("Done", mock.AnythingOfType("*types.Message"), + d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) sender.OnMessage(m, nil, d.Done) @@ -161,7 +161,7 @@ func TestHTTPSender(t *testing.T) { var url string sender.Init(0) - m := types.NewMessage() + m := utils.NewMessage() sender.client = NewTestClient(200, func(req *http.Request) { url = req.URL.String() @@ -173,7 +173,7 @@ func TestHTTPSender(t *testing.T) { status string }, 1), } - d.On("Done", mock.AnythingOfType("*types.Message"), + d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) sender.OnMessage(m, nil, d.Done) @@ -191,7 +191,7 @@ func TestHTTPSender(t *testing.T) { Convey("When a the HTTP client fails", func() { sender.Init(0) - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) sender.client = NewTestClient(200, func(req *http.Request) { @@ -204,7 +204,7 @@ func TestHTTPSender(t *testing.T) { status string }, 1), } - d.On("Done", mock.AnythingOfType("*types.Message"), + d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) sender.OnMessage(m, nil, d.Done) @@ -224,7 +224,7 @@ func TestHTTPSender(t *testing.T) { sender.Init(0) Convey("When try to send messages", func() { - m := types.NewMessage() + m := utils.NewMessage() m.PushPayload([]byte("Hello World")) m.Opts["http_endpoint"] = "endpoint1" @@ -235,7 +235,7 @@ func TestHTTPSender(t *testing.T) { }, 1), } - d.On("Done", mock.AnythingOfType("*types.Message"), + d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) sender.OnMessage(m, nil, d.Done) diff --git a/pipeline.go b/pipeline.go index 1ca7eb0..3656db0 100644 --- a/pipeline.go +++ b/pipeline.go @@ -4,24 +4,24 @@ import ( "sync" "github.com/oleiade/lane" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" ) type component struct { - pool chan chan *types.Message + pool chan chan *utils.Message workers int } // pipeline contains the components type pipeline struct { components []component - input chan *types.Message - retry chan *types.Message - output chan *types.Message + input chan *utils.Message + retry chan *utils.Message + output chan *utils.Message } // newPipeline creates a new Backend -func newPipeline(input, retry, output chan *types.Message) *pipeline { +func newPipeline(input, retry, output chan *utils.Message) *pipeline { var wg sync.WaitGroup p := &pipeline{ input: input, @@ -62,11 +62,11 @@ func newPipeline(input, retry, output chan *types.Message) *pipeline { } // PushComponent adds a new component to the pipeline -func (p *pipeline) PushComponent(composser types.Composer, w int) { +func (p *pipeline) PushComponent(composser utils.Composer, w int) { var wg sync.WaitGroup c := component{ workers: w, - pool: make(chan chan *types.Message, w), + pool: make(chan chan *utils.Message, w), } index := len(p.components) @@ -75,19 +75,19 @@ func (p *pipeline) PushComponent(composser types.Composer, w int) { for i := 0; i < w; i++ { composser.Init(i) - worker := make(chan *types.Message) + worker := make(chan *utils.Message) p.components[index].pool <- worker wg.Add(1) go func(i int) { wg.Done() for m := range worker { - composser.OnMessage(m, func(m *types.Message) { + composser.OnMessage(m, func(m *utils.Message) { if len(p.components) >= index { nextWorker := <-p.components[index+1].pool nextWorker <- m } - }, func(m *types.Message, code int, status string) { + }, func(m *utils.Message, code int, status string) { reports := lane.NewStack() for !m.Reports.Empty() { rep := m.Reports.Pop().(report) diff --git a/rbforwarder.go b/rbforwarder.go index 18e5881..2d015d9 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -6,7 +6,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/oleiade/lane" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" ) // Version is the current tag @@ -30,9 +30,9 @@ type RBForwarder struct { // NewRBForwarder creates a new Forwarder object func NewRBForwarder(config Config) *RBForwarder { - produces := make(chan *types.Message, config.QueueSize) - retries := make(chan *types.Message, config.QueueSize) - reports := make(chan *types.Message, config.QueueSize) + produces := make(chan *utils.Message, config.QueueSize) + retries := make(chan *utils.Message, config.QueueSize) + reports := make(chan *utils.Message, config.QueueSize) f := &RBForwarder{ working: 1, @@ -63,7 +63,7 @@ func (f *RBForwarder) Close() { } // PushComponents adds a new component to the pipeline -func (f *RBForwarder) PushComponents(components []types.Composer, w []int) { +func (f *RBForwarder) PushComponents(components []utils.Composer, w []int) { for i, component := range components { f.p.PushComponent(component, w[i]) } @@ -89,7 +89,7 @@ func (f *RBForwarder) Produce(data []byte, opts map[string]interface{}, opaque i seq := f.currentProducedID f.currentProducedID++ - m := types.NewMessage() + m := utils.NewMessage() r := report{ seq: seq, opaque: lane.NewStack(), diff --git a/rbforwarder_test.go b/rbforwarder_test.go index 88a012e..aece360 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -3,7 +3,7 @@ package rbforwarder import ( "testing" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" . "github.com/smartystreets/goconvey/convey" "github.com/stretchr/testify/mock" ) @@ -18,9 +18,9 @@ func (c *MockMiddleComponent) Init(id int) error { } func (c *MockMiddleComponent) OnMessage( - m *types.Message, - next types.Next, - done types.Done, + m *utils.Message, + next utils.Next, + done utils.Done, ) { c.Called(m) if data, err := m.PopPayload(); err == nil { @@ -46,9 +46,9 @@ func (c *MockComponent) Init(id int) error { } func (c *MockComponent) OnMessage( - m *types.Message, - next types.Next, - done types.Done, + m *utils.Message, + next utils.Next, + done utils.Done, ) { c.Called(m) if data, err := m.PopPayload(); err == nil { @@ -77,7 +77,7 @@ func TestRBForwarder(t *testing.T) { component.On("Init").Return(nil).Times(numWorkers) - var components []types.Composer + var components []utils.Composer var instances []int components = append(components, component) instances = append(instances, numWorkers) @@ -90,7 +90,7 @@ func TestRBForwarder(t *testing.T) { component.status = "OK" component.statusCode = 0 - component.On("OnMessage", mock.AnythingOfType("*types.Message")).Times(1) + component.On("OnMessage", mock.AnythingOfType("*utils.Message")).Times(1) err := rbforwarder.Produce( []byte("Hello World"), @@ -136,7 +136,7 @@ func TestRBForwarder(t *testing.T) { //////////////////////////////////////////////////////////////////////////// Convey("When calling OnMessage() with opaque", func() { - component.On("OnMessage", mock.AnythingOfType("*types.Message")) + component.On("OnMessage", mock.AnythingOfType("*utils.Message")) err := rbforwarder.Produce( []byte("Hello World"), @@ -166,7 +166,7 @@ func TestRBForwarder(t *testing.T) { component.status = "Fake Error" component.statusCode = 99 - component.On("OnMessage", mock.AnythingOfType("*types.Message")).Times(4) + component.On("OnMessage", mock.AnythingOfType("*utils.Message")).Times(4) err := rbforwarder.Produce( []byte("Hello World"), @@ -200,7 +200,7 @@ func TestRBForwarder(t *testing.T) { Convey("When 10000 messages are produced", func() { var numErr int - component.On("OnMessage", mock.AnythingOfType("*types.Message")). + component.On("OnMessage", mock.AnythingOfType("*utils.Message")). Return(nil). Times(numMessages) @@ -271,7 +271,7 @@ func TestRBForwarder(t *testing.T) { component2.On("Init").Return(nil) } - var components []types.Composer + var components []utils.Composer var instances []int components = append(components, component1) @@ -286,8 +286,8 @@ func TestRBForwarder(t *testing.T) { component2.status = "OK" component2.statusCode = 0 - component1.On("OnMessage", mock.AnythingOfType("*types.Message")) - component2.On("OnMessage", mock.AnythingOfType("*types.Message")) + component1.On("OnMessage", mock.AnythingOfType("*utils.Message")) + component2.On("OnMessage", mock.AnythingOfType("*utils.Message")) err := rbforwarder.Produce( []byte("Hello World"), diff --git a/reportHandler.go b/reportHandler.go index 43d6dd9..be1eb5f 100644 --- a/reportHandler.go +++ b/reportHandler.go @@ -4,16 +4,16 @@ import ( "sync" "time" - "github.com/redBorder/rbforwarder/types" + "github.com/redBorder/rbforwarder/utils" ) // reportHandler is used to handle the reports produced by the last element // of the pipeline. The first element of the pipeline can know the status // of the produced message using GetReports() or GetOrderedReports() type reportHandler struct { - input chan *types.Message // Receive messages from pipeline - retries chan *types.Message // Send messages back to the pipeline - out chan *types.Message // Send reports to the user + input chan *utils.Message // Receive messages from pipeline + retries chan *utils.Message // Send messages back to the pipeline + out chan *utils.Message // Send reports to the user queued map[uint64]interface{} // Store pending reports currentReport uint64 // Last delivered report @@ -27,13 +27,13 @@ type reportHandler struct { // newReportHandler creates a new instance of reportHandler func newReporter( maxRetries, backoff int, - input, retries chan *types.Message, + input, retries chan *utils.Message, ) *reportHandler { r := &reportHandler{ input: input, retries: retries, - out: make(chan *types.Message, 100), // NOTE Temp channel size + out: make(chan *utils.Message, 100), // NOTE Temp channel size queued: make(map[uint64]interface{}), @@ -60,7 +60,7 @@ func newReporter( // In other case retry the message sending it again to the pipeline r.wg.Add(1) - go func(m *types.Message) { + go func(m *utils.Message) { defer r.wg.Done() rep := m.Reports.Pop().(report) rep.retries++ diff --git a/types/composer.go b/utils/composer.go similarity index 97% rename from types/composer.go rename to utils/composer.go index 7999ffe..1bf7487 100644 --- a/types/composer.go +++ b/utils/composer.go @@ -1,4 +1,4 @@ -package types +package utils // Next should be called by a component in order to pass the message to the next // component in the pipeline. diff --git a/types/message.go b/utils/message.go similarity index 98% rename from types/message.go rename to utils/message.go index d1593ee..c06b042 100644 --- a/types/message.go +++ b/utils/message.go @@ -1,4 +1,4 @@ -package types +package utils import ( "errors" diff --git a/types/message_test.go b/utils/message_test.go similarity index 98% rename from types/message_test.go rename to utils/message_test.go index c4406ea..e99b570 100644 --- a/types/message_test.go +++ b/utils/message_test.go @@ -1,4 +1,4 @@ -package types +package utils import ( "testing" From 522441b086ece4d763a5a0a3c58d05839135617f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Thu, 11 Aug 2016 11:08:26 +0200 Subject: [PATCH 15/36] :sparkles: Add limiter component --- components/limiter/config.go | 8 ++ components/limiter/limiter.go | 58 +++++++++++ components/limiter/limiter_test.go | 152 +++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 components/limiter/config.go create mode 100644 components/limiter/limiter.go create mode 100644 components/limiter/limiter_test.go diff --git a/components/limiter/config.go b/components/limiter/config.go new file mode 100644 index 0000000..a7fc890 --- /dev/null +++ b/components/limiter/config.go @@ -0,0 +1,8 @@ +package limiter + +// Config stores the config for a Limiter +type Config struct { + MessageLimit uint64 + BytesLimit uint64 + Burst uint64 +} diff --git a/components/limiter/limiter.go b/components/limiter/limiter.go new file mode 100644 index 0000000..0da7d13 --- /dev/null +++ b/components/limiter/limiter.go @@ -0,0 +1,58 @@ +package limiter + +import ( + "time" + + "github.com/benbjohnson/clock" + "github.com/redBorder/rbforwarder/utils" +) + +// Limiter is a component that blocks the pipeline to ensure a maximum number +// of messages are being processed in a time. You may spawn ONLY ONE worker +type Limiter struct { + id int + currentMessages uint64 + currentBytes uint64 + config Config + keepSending chan struct{} + paused bool + clk clock.Clock +} + +// Init initializes the limiter +func (l *Limiter) Init(id int) { + l.id = id + l.keepSending = make(chan struct{}, l.config.Burst) + l.paused = false + + go func() { + for { + <-l.clk.Timer(1 * time.Second).C + l.keepSending <- struct{}{} + } + }() +} + +// OnMessage will block the pipeline when the message rate is too high +func (l *Limiter) OnMessage(m *utils.Message, next utils.Next, done utils.Done) { + if l.paused { + <-l.keepSending + l.currentMessages = 0 + l.currentBytes = 0 + l.paused = false + } + + l.currentMessages++ + if l.config.BytesLimit > 0 { + if payload, err := m.PopPayload(); err == nil { + l.currentBytes += uint64(len(payload)) + m.PushPayload(payload) + } + } + next(m) + + if l.config.MessageLimit > 0 && l.currentMessages >= l.config.MessageLimit || + l.config.BytesLimit > 0 && l.currentBytes >= l.config.BytesLimit { + l.paused = true + } +} diff --git a/components/limiter/limiter_test.go b/components/limiter/limiter_test.go new file mode 100644 index 0000000..ce58650 --- /dev/null +++ b/components/limiter/limiter_test.go @@ -0,0 +1,152 @@ +package limiter + +import ( + "testing" + "time" + + "github.com/benbjohnson/clock" + "github.com/redBorder/rbforwarder/utils" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/mock" +) + +type Nexter struct { + mock.Mock + nextCalled chan *utils.Message +} + +func (n *Nexter) Next(m *utils.Message) { + n.Called(m) + n.nextCalled <- m +} + +func TestHTTPSender(t *testing.T) { + Convey("Given an Limiter with 100 messages per second without burst", t, func() { + limiter := &Limiter{ + config: Config{ + MessageLimit: 100, + Burst: 1, + }, + clk: clock.NewMock(), + } + limiter.Init(0) + + Convey("When the limit number of messages are reached", func() { + clk := limiter.clk.(*clock.Mock) + n := Nexter{ + nextCalled: make(chan *utils.Message, limiter.config.MessageLimit*2), + } + n.On("Next", mock.AnythingOfType("*utils.Message")) + + for i := uint64(0); i < limiter.config.MessageLimit; i++ { + limiter.OnMessage(nil, n.Next, nil) + } + + Convey("Then the limiter should be paused", func() { + So(limiter.currentMessages, ShouldEqual, limiter.config.MessageLimit) + So(limiter.paused, ShouldBeTrue) + }) + + Convey("Then after 1 second the limiter should be ready again", func() { + clk.Add(1 * time.Second) + limiter.OnMessage(nil, n.Next, nil) + So(limiter.currentMessages, ShouldEqual, 1) + So(limiter.paused, ShouldBeFalse) + }) + }) + }) + + Convey("Given an Limiter with 1000 bytes per second without burst", t, func() { + limiter := &Limiter{ + config: Config{ + BytesLimit: 1000, + Burst: 1, + }, + clk: clock.NewMock(), + } + limiter.Init(0) + + Convey("When messages are sent", func() { + n := Nexter{ + nextCalled: make(chan *utils.Message, 100), + } + n.On("Next", mock.AnythingOfType("*utils.Message")) + + Convey("Then the limiter should not be paused after 750 bytes", func() { + for i := uint64(0); i < 3; i++ { + m := utils.NewMessage() + payload := make([]byte, 250) + m.PushPayload(payload) + limiter.OnMessage(m, n.Next, nil) + } + + So(limiter.currentBytes, ShouldEqual, 750) + So(limiter.paused, ShouldBeFalse) + }) + + Convey("Then the limiter should be paused after 1000 bytes", func() { + for i := uint64(0); i < 4; i++ { + m := utils.NewMessage() + payload := make([]byte, 250) + m.PushPayload(payload) + limiter.OnMessage(m, n.Next, nil) + } + + So(limiter.currentBytes, ShouldEqual, 1000) + So(limiter.paused, ShouldBeTrue) + }) + + Convey("Then after 1 second the limiter should be ready again", func() { + clk := limiter.clk.(*clock.Mock) + clk.Add(1 * time.Second) + + m := utils.NewMessage() + payload := make([]byte, 250) + m.PushPayload(payload) + limiter.OnMessage(m, n.Next, nil) + + So(limiter.currentBytes, ShouldEqual, 250) + So(limiter.paused, ShouldBeFalse) + }) + }) + }) + + Convey("Given a limiter with burst", t, func() { + limiter := &Limiter{ + config: Config{ + MessageLimit: 100, + Burst: 2, + }, + clk: clock.NewMock(), + } + limiter.Init(0) + + clk := limiter.clk.(*clock.Mock) + clk.Add(0) + clk.Add(2 * time.Second) + + Convey("When the limit number of messages are reached", func() { + n := Nexter{ + nextCalled: make(chan *utils.Message, limiter.config.MessageLimit*2), + } + n.On("Next", mock.AnythingOfType("*utils.Message")) + + for i := uint64(0); i < limiter.config.MessageLimit; i++ { + limiter.OnMessage(nil, n.Next, nil) + } + + Convey("Then should be 2 burst available", func() { + So(len(limiter.keepSending), ShouldEqual, 2) + }) + Convey("Then messages are not blocked after the limit", func() { + for i := uint64(0); i < limiter.config.MessageLimit; i++ { + limiter.OnMessage(nil, n.Next, nil) + } + So(limiter.currentMessages, ShouldEqual, 100) + }) + Convey("Then the limiter blocks again after reaching limit a second time", func() { + So(limiter.paused, ShouldBeTrue) + }) + }) + }) +} From 730bd0526bbaf8128461b6e48df88f69f4ec393a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 11:34:01 +0200 Subject: [PATCH 16/36] :bug: New batcher implementation Fixes #29 --- components/batch/batch.go | 6 +-- components/batch/batcher.go | 74 +++++++++++++++++++++++--------- components/batch/batcher_test.go | 24 ++++++++--- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/components/batch/batch.go b/components/batch/batch.go index edd2b89..1be7469 100644 --- a/components/batch/batch.go +++ b/components/batch/batch.go @@ -15,6 +15,7 @@ type Batch struct { Buff *bytes.Buffer MessageCount uint // Current number of messages in the buffer Next utils.Next // Call to pass the message to the next handler + Timer *clock.Timer } // NewBatch creates a new instance of Batch @@ -30,10 +31,10 @@ func NewBatch(m *utils.Message, group string, next utils.Next, clk clock.Clock, } if timeoutMillis != 0 { - timer := clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) + b.Timer = clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) go func() { - <-timer.C + <-b.Timer.C if b.MessageCount > 0 { ready <- b } @@ -47,7 +48,6 @@ func NewBatch(m *utils.Message, group string, next utils.Next, clk clock.Clock, func (b *Batch) Send(cb func()) { b.Message.PushPayload(b.Buff.Bytes()) cb() - b.Next(b.Message) } // Add merges a new message in the buffer diff --git a/components/batch/batcher.go b/components/batch/batcher.go index e36ccc4..81e9ae0 100644 --- a/components/batch/batcher.go +++ b/components/batch/batcher.go @@ -1,18 +1,25 @@ package batcher import ( + "sync" + "github.com/benbjohnson/clock" "github.com/redBorder/rbforwarder/utils" ) // Batcher allows to merge multiple messages in a single one type Batcher struct { - id int // Worker ID + id int // Worker ID + wg sync.WaitGroup batches map[string]*Batch // Collection of batches pending readyBatches chan *Batch clk clock.Clock + incoming chan struct { + m *utils.Message + next utils.Next + } - config Config // Batcher configuration + Config Config // Batcher configuration } // Init starts a gorutine that can receive: @@ -22,13 +29,46 @@ func (b *Batcher) Init(id int) { b.id = id b.batches = make(map[string]*Batch) b.readyBatches = make(chan *Batch) - b.clk = clock.New() + b.incoming = make(chan struct { + m *utils.Message + next utils.Next + }) + if b.clk == nil { + b.clk = clock.New() + } go func() { - for batch := range b.readyBatches { - batch.Send(func() { - delete(b.batches, batch.Group) - }) + for { + select { + case message := <-b.incoming: + group, exists := message.m.Opts["batch_group"].(string) + if !exists { + message.next(message.m) + } else { + if batch, exists := b.batches[group]; exists { + batch.Add(message.m) + + if batch.MessageCount >= b.Config.Limit { + batch.Send(func() { + delete(b.batches, group) + batch.Timer.Stop() + batch.Next(batch.Message) + }) + } + } else { + b.batches[group] = NewBatch(message.m, group, message.next, b.clk, + b.Config.TimeoutMillis, b.readyBatches) + } + } + + b.wg.Done() + + case batch := <-b.readyBatches: + batch.Send(func() { + delete(b.batches, batch.Group) + batch.Next(batch.Message) + }) + } } }() } @@ -36,18 +76,10 @@ func (b *Batcher) Init(id int) { // OnMessage is called when a new message is receive. Add the new message to // a batch func (b *Batcher) OnMessage(m *utils.Message, next utils.Next, done utils.Done) { - if group, exists := m.Opts["batch_group"].(string); exists { - if batch, exists := b.batches[group]; exists { - batch.Add(m) - if batch.MessageCount >= b.config.Limit { - b.readyBatches <- batch - } - } else { - b.batches[group] = NewBatch(m, group, next, b.clk, b.config.TimeoutMillis, b.readyBatches) - } - - return - } - - next(m) + b.wg.Add(1) + b.incoming <- struct { + m *utils.Message + next utils.Next + }{m, next} + b.wg.Wait() } diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go index c9280bb..5427593 100644 --- a/components/batch/batcher_test.go +++ b/components/batch/batcher_test.go @@ -23,7 +23,7 @@ func (nd *NexterDoner) Next(m *utils.Message) { func TestBatcher(t *testing.T) { Convey("Given a batcher", t, func() { batcher := &Batcher{ - config: Config{ + Config: Config{ TimeoutMillis: 1000, Limit: 10, MaxPendingBatches: 10, @@ -83,7 +83,7 @@ func TestBatcher(t *testing.T) { Convey("When the max number of messages is reached", func() { var messages []*utils.Message - for i := 0; i < int(batcher.config.Limit); i++ { + for i := 0; i < int(batcher.Config.Limit); i++ { m := utils.NewMessage() m.PushPayload([]byte("ABC")) m.Opts = map[string]interface{}{ @@ -95,10 +95,10 @@ func TestBatcher(t *testing.T) { } nd := new(NexterDoner) - nd.nextCalled = make(chan *utils.Message) + nd.nextCalled = make(chan *utils.Message, 1) nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) - for i := 0; i < int(batcher.config.Limit); i++ { + for i := 0; i < int(batcher.Config.Limit); i++ { batcher.OnMessage(messages[i], nd.Next, nil) } @@ -109,7 +109,7 @@ func TestBatcher(t *testing.T) { So(err, ShouldBeNil) So(string(data), ShouldEqual, "ABCABCABCABCABCABCABCABCABCABC") - So(m.Reports.Size(), ShouldEqual, batcher.config.Limit) + So(m.Reports.Size(), ShouldEqual, batcher.Config.Limit) So(batcher.batches["group1"], ShouldBeNil) So(len(batcher.batches), ShouldEqual, 0) }) @@ -150,19 +150,23 @@ func TestBatcher(t *testing.T) { }) }) - Convey("When multiple messages are received with differente groups", func() { + Convey("When multiple messages are received with differents groups", func() { m1 := utils.NewMessage() m1.PushPayload([]byte("MESSAGE 1")) + m1.Reports.Push("Report 1") m1.Opts = map[string]interface{}{ "batch_group": "group1", } + m2 := utils.NewMessage() m2.PushPayload([]byte("MESSAGE 2")) + m2.Reports.Push("Report 2") m2.Opts = map[string]interface{}{ "batch_group": "group2", } m3 := utils.NewMessage() m3.PushPayload([]byte("MESSAGE 3")) + m3.Reports.Push("Report 3") m3.Opts = map[string]interface{}{ "batch_group": "group2", } @@ -189,15 +193,21 @@ func TestBatcher(t *testing.T) { clk := batcher.clk.(*clock.Mock) So(len(batcher.batches), ShouldEqual, 2) - clk.Add(time.Duration(batcher.config.TimeoutMillis) * time.Millisecond) + clk.Add(time.Duration(batcher.Config.TimeoutMillis) * time.Millisecond) group1 := <-nd.nextCalled group1Data, err := group1.PopPayload() + report1 := group1.Reports.Pop().(string) So(err, ShouldBeNil) + So(report1, ShouldEqual, "Report 1") group2 := <-nd.nextCalled group2Data, err := group2.PopPayload() So(err, ShouldBeNil) + report3 := group2.Reports.Pop().(string) + So(report3, ShouldEqual, "Report 3") + report2 := group2.Reports.Pop().(string) + So(report2, ShouldEqual, "Report 2") So(string(group1Data), ShouldEqual, "MESSAGE 1") So(string(group2Data), ShouldEqual, "MESSAGE 2MESSAGE 3") From 70c9a8decc49f88323711fb3f5937a4e5ec1ce68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 11:40:06 +0200 Subject: [PATCH 17/36] :sparkles: Add new example using HTTP and batch --- examples/http_send.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 examples/http_send.go diff --git a/examples/http_send.go b/examples/http_send.go new file mode 100644 index 0000000..78a71ec --- /dev/null +++ b/examples/http_send.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + + "github.com/redBorder/rbforwarder" + "github.com/redBorder/rbforwarder/components/batch" + "github.com/redBorder/rbforwarder/components/httpsender" +) + +func main() { + var components []interface{} + var workers []int + + f := rbforwarder.NewRBForwarder(rbforwarder.Config{ + Retries: 3, + Backoff: 5, + QueueSize: 10000, + }) + + batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 2, + }, + } + components = append(components, batch) + workers = append(workers, 1) + + sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", + } + components = append(components, sender) + workers = append(workers, 1) + + f.PushComponents(components, workers) + + opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", + } + + for i := 0; i < 10; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) + } + + for report := range f.GetReports() { + r := report.(rbforwarder.Report) + // fmt.Printf("MESSAGE: %d\n", r.Opaque.(int)) + // fmt.Printf("CODE: %d\n", r.Code) + // fmt.Printf("STATUS: %s\n", r.Status) + if r.Opaque.(int) == 9999 { + break + } + } +} From 9cb42c06815e00827ef7bd5d130670a94dc6b606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 12:02:06 +0200 Subject: [PATCH 18/36] :bug: Make rbforwarder.Report public It was impossible to obtain a report for the user --- pipeline.go | 7 ++++--- rbforwarder.go | 10 ++++------ rbforwarder_test.go | 43 +++++++++++++++++++++---------------------- report.go | 11 +++++------ reportHandler.go | 10 +++++----- utils/composer.go | 2 +- 6 files changed, 40 insertions(+), 43 deletions(-) diff --git a/pipeline.go b/pipeline.go index 3656db0..2f4ca84 100644 --- a/pipeline.go +++ b/pipeline.go @@ -89,10 +89,11 @@ func (p *pipeline) PushComponent(composser utils.Composer, w int) { } }, func(m *utils.Message, code int, status string) { reports := lane.NewStack() + for !m.Reports.Empty() { - rep := m.Reports.Pop().(report) - rep.code = code - rep.status = status + rep := m.Reports.Pop().(Report) + rep.Code = code + rep.Status = status reports.Push(rep) } diff --git a/rbforwarder.go b/rbforwarder.go index 2d015d9..72b2dc9 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -5,7 +5,6 @@ import ( "sync/atomic" "github.com/Sirupsen/logrus" - "github.com/oleiade/lane" "github.com/redBorder/rbforwarder/utils" ) @@ -63,9 +62,9 @@ func (f *RBForwarder) Close() { } // PushComponents adds a new component to the pipeline -func (f *RBForwarder) PushComponents(components []utils.Composer, w []int) { +func (f *RBForwarder) PushComponents(components []interface{}, w []int) { for i, component := range components { - f.p.PushComponent(component, w[i]) + f.p.PushComponent(component.(utils.Composer), w[i]) } } @@ -90,15 +89,14 @@ func (f *RBForwarder) Produce(data []byte, opts map[string]interface{}, opaque i seq := f.currentProducedID f.currentProducedID++ m := utils.NewMessage() - r := report{ + r := Report{ seq: seq, - opaque: lane.NewStack(), + Opaque: opaque, } m.PushPayload(data) m.Opts = opts m.Reports.Push(r) - r.opaque.Push(opaque) f.p.input <- m diff --git a/rbforwarder_test.go b/rbforwarder_test.go index aece360..eb63d1b 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -12,9 +12,9 @@ type MockMiddleComponent struct { mock.Mock } -func (c *MockMiddleComponent) Init(id int) error { - args := c.Called() - return args.Error(0) +func (c *MockMiddleComponent) Init(id int) { + c.Called() + return } func (c *MockMiddleComponent) OnMessage( @@ -40,9 +40,8 @@ type MockComponent struct { statusCode int } -func (c *MockComponent) Init(id int) error { - args := c.Called() - return args.Error(0) +func (c *MockComponent) Init(id int) { + c.Called() } func (c *MockComponent) OnMessage( @@ -77,7 +76,7 @@ func TestRBForwarder(t *testing.T) { component.On("Init").Return(nil).Times(numWorkers) - var components []utils.Composer + var components []interface{} var instances []int components = append(components, component) instances = append(instances, numWorkers) @@ -99,18 +98,18 @@ func TestRBForwarder(t *testing.T) { ) Convey("\"Hello World\" message should be get by the worker", func() { - var lastReport report + var lastReport Report var reports int for r := range rbforwarder.GetReports() { reports++ - lastReport = r.(report) + lastReport = r.(Report) rbforwarder.Close() } So(lastReport, ShouldNotBeNil) So(reports, ShouldEqual, 1) - So(lastReport.code, ShouldEqual, 0) - So(lastReport.status, ShouldEqual, "OK") + So(lastReport.Code, ShouldEqual, 0) + So(lastReport.Status, ShouldEqual, "OK") So(err, ShouldBeNil) component.AssertExpectations(t) @@ -148,14 +147,14 @@ func TestRBForwarder(t *testing.T) { So(err, ShouldBeNil) var reports int - var lastReport report + var lastReport Report for r := range rbforwarder.GetReports() { reports++ - lastReport = r.(report) + lastReport = r.(Report) rbforwarder.Close() } - opaque := lastReport.opaque.Pop().(string) + opaque := lastReport.Opaque.(string) So(opaque, ShouldEqual, "This is an opaque") }) }) @@ -178,17 +177,17 @@ func TestRBForwarder(t *testing.T) { So(err, ShouldBeNil) var reports int - var lastReport report + var lastReport Report for r := range rbforwarder.GetReports() { reports++ - lastReport = r.(report) + lastReport = r.(Report) rbforwarder.Close() } So(lastReport, ShouldNotBeNil) So(reports, ShouldEqual, 1) - So(lastReport.status, ShouldEqual, "Fake Error") - So(lastReport.code, ShouldEqual, 99) + So(lastReport.Status, ShouldEqual, "Fake Error") + So(lastReport.Code, ShouldEqual, 99) So(lastReport.retries, ShouldEqual, numRetries) component.AssertExpectations(t) @@ -233,7 +232,7 @@ func TestRBForwarder(t *testing.T) { var reports int for rep := range rbforwarder.GetOrderedReports() { - if rep.(report).opaque.Pop().(int) != reports { + if rep.(Report).Opaque.(int) != reports { ordered = false } reports++ @@ -271,7 +270,7 @@ func TestRBForwarder(t *testing.T) { component2.On("Init").Return(nil) } - var components []utils.Composer + var components []interface{} var instances []int components = append(components, component1) @@ -302,8 +301,8 @@ func TestRBForwarder(t *testing.T) { for rep := range rbforwarder.GetReports() { reports++ - code := rep.(report).code - status := rep.(report).status + code := rep.(Report).Code + status := rep.(Report).Status So(code, ShouldEqual, 0) So(status, ShouldEqual, "OK") } diff --git a/report.go b/report.go index 6d8d5ac..d72865c 100644 --- a/report.go +++ b/report.go @@ -1,12 +1,11 @@ package rbforwarder -import "github.com/oleiade/lane" +// Report contains information abot a delivered message +type Report struct { + Code int + Status string + Opaque interface{} -type report struct { seq uint64 - code int - status string retries int - - opaque *lane.Stack } diff --git a/reportHandler.go b/reportHandler.go index be1eb5f..cb11688 100644 --- a/reportHandler.go +++ b/reportHandler.go @@ -45,8 +45,8 @@ func newReporter( // Get reports from the handler channel for m := range r.input { // If the message has status code 0 (success) send the report to the user - rep := m.Reports.Head().(report) - if rep.code == 0 || r.maxRetries == 0 { + rep := m.Reports.Head().(Report) + if rep.Code == 0 || r.maxRetries == 0 { r.out <- m continue } @@ -62,7 +62,7 @@ func newReporter( r.wg.Add(1) go func(m *utils.Message) { defer r.wg.Done() - rep := m.Reports.Pop().(report) + rep := m.Reports.Pop().(Report) rep.retries++ m.Reports.Push(rep) <-time.After(time.Duration(r.backoff) * time.Second) @@ -86,7 +86,7 @@ func (r *reportHandler) GetReports() chan interface{} { go func() { for message := range r.out { for !message.Reports.Empty() { - rep := message.Reports.Pop().(report) + rep := message.Reports.Pop().(Report) reports <- rep } } @@ -103,7 +103,7 @@ func (r *reportHandler) GetOrderedReports() chan interface{} { go func() { for message := range r.out { for !message.Reports.Empty() { - rep := message.Reports.Pop().(report) + rep := message.Reports.Pop().(Report) if rep.seq == r.currentReport { // The message is the expected. Send it. reports <- rep diff --git a/utils/composer.go b/utils/composer.go index 1bf7487..1ba831c 100644 --- a/utils/composer.go +++ b/utils/composer.go @@ -12,6 +12,6 @@ type Done func(*Message, int, string) // Composer represents a component in the pipeline that performs a work on // a message type Composer interface { - Init(int) error + Init(int) OnMessage(*Message, Next, Done) } From 87779e4add42834824c7a4b726db8fef1dd30cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 12:03:46 +0200 Subject: [PATCH 19/36] :sparkles: Improve example --- examples/http_send.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/http_send.go b/examples/http_send.go index 78a71ec..e36a5e4 100644 --- a/examples/http_send.go +++ b/examples/http_send.go @@ -11,6 +11,7 @@ import ( func main() { var components []interface{} var workers []int + const numMessages = 100000 f := rbforwarder.NewRBForwarder(rbforwarder.Config{ Retries: 3, @@ -21,7 +22,7 @@ func main() { batch := &batcher.Batcher{ Config: batcher.Config{ TimeoutMillis: 1000, - Limit: 2, + Limit: 1000, }, } components = append(components, batch) @@ -40,18 +41,21 @@ func main() { "batch_group": "librb-http", } - for i := 0; i < 10; i++ { + for i := 0; i < numMessages; i++ { data := fmt.Sprintf("{\"message\": %d}", i) f.Produce([]byte(data), opts, i) } + var errors int for report := range f.GetReports() { r := report.(rbforwarder.Report) - // fmt.Printf("MESSAGE: %d\n", r.Opaque.(int)) - // fmt.Printf("CODE: %d\n", r.Code) - // fmt.Printf("STATUS: %s\n", r.Status) - if r.Opaque.(int) == 9999 { + if r.Code > 0 { + errors += r.Code + } + if r.Opaque.(int) == numMessages-1 { break } } + + fmt.Printf("Sent %d messages with %d errors\n", numMessages, errors) } From 9f9121c07a380805aa35f771f9ca4dd4bcac993c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 12:47:02 +0200 Subject: [PATCH 20/36] :sparkles: Make http client public --- components/httpsender/httpSender.go | 8 +++++--- components/httpsender/httpSender_test.go | 12 ++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/components/httpsender/httpSender.go b/components/httpsender/httpSender.go index ee3bd1e..7daabcd 100644 --- a/components/httpsender/httpSender.go +++ b/components/httpsender/httpSender.go @@ -18,7 +18,7 @@ type HTTPSender struct { id int err error URL string - client *http.Client + Client *http.Client } // Init initializes the HTTP component @@ -26,7 +26,9 @@ func (s *HTTPSender) Init(id int) { s.id = id if govalidator.IsURL(s.URL) { - s.client = &http.Client{} + if s.Client == nil { + s.Client = &http.Client{} + } } else { s.err = errors.New("Invalid URL") } @@ -54,7 +56,7 @@ func (s *HTTPSender) OnMessage(m *utils.Message, next utils.Next, done utils.Don } buf := bytes.NewBuffer(data) - res, err := s.client.Post(u, "", buf) + res, err := s.Client.Post(u, "", buf) if err != nil { done(m, 1, "HTTPSender error: "+err.Error()) return diff --git a/components/httpsender/httpSender_test.go b/components/httpsender/httpSender_test.go index be0c950..207aefe 100644 --- a/components/httpsender/httpSender_test.go +++ b/components/httpsender/httpSender_test.go @@ -56,7 +56,7 @@ func TestHTTPSender(t *testing.T) { sender.Init(0) Convey("Then the config should be ok", func() { - So(sender.client, ShouldNotBeNil) + So(sender.Client, ShouldNotBeNil) }) }) @@ -66,7 +66,7 @@ func TestHTTPSender(t *testing.T) { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - sender.client = NewTestClient(401, func(req *http.Request) { + sender.Client = NewTestClient(401, func(req *http.Request) { url = req.URL.String() }) @@ -99,7 +99,7 @@ func TestHTTPSender(t *testing.T) { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - sender.client = NewTestClient(200, func(req *http.Request) { + sender.Client = NewTestClient(200, func(req *http.Request) { url = req.URL.String() }) @@ -132,7 +132,7 @@ func TestHTTPSender(t *testing.T) { m.PushPayload([]byte("Hello World")) m.Opts["http_endpoint"] = "endpoint1" - sender.client = NewTestClient(200, func(req *http.Request) { + sender.Client = NewTestClient(200, func(req *http.Request) { url = req.URL.String() }) @@ -163,7 +163,7 @@ func TestHTTPSender(t *testing.T) { m := utils.NewMessage() - sender.client = NewTestClient(200, func(req *http.Request) { + sender.Client = NewTestClient(200, func(req *http.Request) { url = req.URL.String() }) @@ -194,7 +194,7 @@ func TestHTTPSender(t *testing.T) { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - sender.client = NewTestClient(200, func(req *http.Request) { + sender.Client = NewTestClient(200, func(req *http.Request) { req.Write(nil) }) From 7a0329dc7b0f8c6bfe6086c196d3454d6cb71022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 12:47:50 +0200 Subject: [PATCH 21/36] :racehorse: Add benchmarks --- rbforwarder_test.go | 169 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/rbforwarder_test.go b/rbforwarder_test.go index eb63d1b..1933f4d 100644 --- a/rbforwarder_test.go +++ b/rbforwarder_test.go @@ -1,8 +1,14 @@ package rbforwarder import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" "testing" + "github.com/redBorder/rbforwarder/components/batch" + "github.com/redBorder/rbforwarder/components/httpsender" "github.com/redBorder/rbforwarder/utils" . "github.com/smartystreets/goconvey/convey" "github.com/stretchr/testify/mock" @@ -319,3 +325,166 @@ func TestRBForwarder(t *testing.T) { }) }) } + +func BenchmarkNoBatch(b *testing.B) { + var components []interface{} + var workers []int + + f := NewRBForwarder(Config{ + Retries: 3, + Backoff: 5, + QueueSize: 1, + }) + + batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 10000, + }, + } + components = append(components, batch) + workers = append(workers, 1) + + sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + } + components = append(components, sender) + workers = append(workers, 1) + + f.PushComponents(components, workers) + + opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", + } + + for i := 0; i < b.N; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) + } + + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) == b.N-1 { + break + } + } +} + +func NewTestClient(code int, cb func(*http.Request)) *http.Client { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + cb(r) + })) + + transport := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + return &http.Client{Transport: transport} +} + +func BenchmarkLittleBatch(b *testing.B) { + var components []interface{} + var workers []int + + f := NewRBForwarder(Config{ + Retries: 3, + Backoff: 5, + QueueSize: b.N / 100, + }) + + batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 10000, + }, + } + components = append(components, batch) + workers = append(workers, 1) + + sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + } + components = append(components, sender) + workers = append(workers, 1) + + f.PushComponents(components, workers) + + opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", + } + + for i := 0; i < b.N; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) + } + + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) == b.N-1 { + break + } + } +} + +func BenchmarkBigBatch(b *testing.B) { + var components []interface{} + var workers []int + + f := NewRBForwarder(Config{ + Retries: 3, + Backoff: 5, + QueueSize: b.N / 10, + }) + + batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 10000, + }, + } + components = append(components, batch) + workers = append(workers, 1) + + sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + } + components = append(components, sender) + workers = append(workers, 1) + + f.PushComponents(components, workers) + + opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", + } + + for i := 0; i < b.N; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) + } + + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) == b.N-1 { + break + } + } +} From 76dde601b7db7a32a7009f848795d680a909623b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 13:01:44 +0200 Subject: [PATCH 22/36] :memo: Add example to README.md --- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/README.md b/README.md index d0450cd..406aafe 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,59 @@ _The application is on hard development, breaking changes expected until 1.0._ | 0.8 | JSON component | _Pending_ | | 0.9 | MQTT component | _Pending_ | | 1.0 | Kafka component | _Pending_ | + +## Usage + +```go + // Array of components to use and workers +var components []interface{} +var workers []int + +// Create an instance of rbforwarder +f := rbforwarder.NewRBForwarder(rbforwarder.Config{ + Retries: 3, // Number of retries before give up + Backoff: 5, // Time to wait before retry a message + QueueSize: 10000, // Max messageon queue, produce will block when the queue is full +}) + +// Create a batcher component and add it to the components array +batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 1000, + }, +} +components = append(components, batch) +workers = append(workers, 1) + +// Create a http component and add it to the components array +sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", +} +components = append(components, sender) +workers = append(workers, 1) + +// Push the component array and workers to the pipeline +f.PushComponents(components, workers) + +opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", +} + +// Produce messages. It won't block until the queue is full. +f.Produce([]byte("{\"message\": 1}"), opts, 1) +f.Produce([]byte("{\"message\": 2}"), opts, 2) +f.Produce([]byte("{\"message\": 3}"), opts, 3) +f.Produce([]byte("{\"message\": 4}"), opts, 4) +f.Produce([]byte("{\"message\": 5}"), opts, 5) + +// Read reports. You should do this on a separate gorutine so you make sure +// that you won't block +for report := range f.GetReports() { + r := report.(rbforwarder.Report) + if r.Opaque.(int) == numMessages-1 { + break + } +} +``` From 1e987fcdc05f4c24c4ff9e90be7674486fd6c595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 13:46:19 +0200 Subject: [PATCH 23/36] :sparkles: Add compression feature to batcher --- components/batch/batch.go | 32 ++++++++++++--- components/batch/batcher.go | 4 +- components/batch/batcher_test.go | 70 ++++++++++++++++++++++++++++++-- components/batch/config.go | 1 + 4 files changed, 96 insertions(+), 11 deletions(-) diff --git a/components/batch/batch.go b/components/batch/batch.go index 1be7469..9cd6fe4 100644 --- a/components/batch/batch.go +++ b/components/batch/batch.go @@ -1,7 +1,10 @@ package batcher import ( + "bufio" "bytes" + "compress/zlib" + "io" "time" "github.com/benbjohnson/clock" @@ -11,25 +14,36 @@ import ( // Batch groups multiple messages type Batch struct { Group string + Deflate bool Message *utils.Message - Buff *bytes.Buffer + Buf *bytes.Buffer + Writer io.Writer MessageCount uint // Current number of messages in the buffer Next utils.Next // Call to pass the message to the next handler Timer *clock.Timer } // NewBatch creates a new instance of Batch -func NewBatch(m *utils.Message, group string, next utils.Next, clk clock.Clock, - timeoutMillis uint, ready chan *Batch) *Batch { +func NewBatch(m *utils.Message, group string, deflate bool, next utils.Next, + clk clock.Clock, timeoutMillis uint, ready chan *Batch) *Batch { payload, _ := m.PopPayload() b := &Batch{ Group: group, + Deflate: deflate, Next: next, Message: m, MessageCount: 1, - Buff: bytes.NewBuffer(payload), + Buf: new(bytes.Buffer), } + if b.Deflate { + b.Writer = zlib.NewWriter(b.Buf) + } else { + b.Writer = bufio.NewWriter(b.Buf) + } + + b.Writer.Write(payload) + if timeoutMillis != 0 { b.Timer = clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) @@ -46,7 +60,13 @@ func NewBatch(m *utils.Message, group string, next utils.Next, clk clock.Clock, // Send the batch of messages to the next handler in the pipeline func (b *Batch) Send(cb func()) { - b.Message.PushPayload(b.Buff.Bytes()) + if b.Deflate { + b.Writer.(*zlib.Writer).Flush() + } else { + b.Writer.(*bufio.Writer).Flush() + } + + b.Message.PushPayload(b.Buf.Bytes()) cb() } @@ -56,7 +76,7 @@ func (b *Batch) Add(m *utils.Message) { b.Message.Reports.Push(newReport) newPayload, _ := m.PopPayload() - b.Buff.Write(newPayload) + b.Writer.Write(newPayload) b.MessageCount++ } diff --git a/components/batch/batcher.go b/components/batch/batcher.go index 81e9ae0..a20110e 100644 --- a/components/batch/batcher.go +++ b/components/batch/batcher.go @@ -56,8 +56,8 @@ func (b *Batcher) Init(id int) { }) } } else { - b.batches[group] = NewBatch(message.m, group, message.next, b.clk, - b.Config.TimeoutMillis, b.readyBatches) + b.batches[group] = NewBatch(message.m, group, b.Config.Deflate, + message.next, b.clk, b.Config.TimeoutMillis, b.readyBatches) } } diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go index 5427593..0b85a9a 100644 --- a/components/batch/batcher_test.go +++ b/components/batch/batcher_test.go @@ -1,6 +1,9 @@ package batcher import ( + "bufio" + "bytes" + "compress/zlib" "testing" "time" @@ -27,6 +30,7 @@ func TestBatcher(t *testing.T) { TimeoutMillis: 1000, Limit: 10, MaxPendingBatches: 10, + Deflate: false, }, } @@ -67,7 +71,9 @@ func TestBatcher(t *testing.T) { batch, exists := batcher.batches["group1"] So(exists, ShouldBeTrue) - data := batch.Buff.Bytes() + batch.Writer.(*bufio.Writer).Flush() + + data := batch.Buf.Bytes() So(string(data), ShouldEqual, "Hello World") opts := batch.Message.Opts @@ -180,10 +186,12 @@ func TestBatcher(t *testing.T) { batcher.OnMessage(m3, nd.Next, nil) Convey("Each message should be in its group", func() { - group1 := batcher.batches["group1"].Buff.Bytes() + batcher.batches["group1"].Writer.(*bufio.Writer).Flush() + group1 := batcher.batches["group1"].Buf.Bytes() So(string(group1), ShouldEqual, "MESSAGE 1") - group2 := batcher.batches["group2"].Buff.Bytes() + batcher.batches["group2"].Writer.(*bufio.Writer).Flush() + group2 := batcher.batches["group2"].Buf.Bytes() So(string(group2), ShouldEqual, "MESSAGE 2MESSAGE 3") So(len(batcher.batches), ShouldEqual, 2) @@ -219,4 +227,60 @@ func TestBatcher(t *testing.T) { }) }) }) + + Convey("Given a batcher with compression", t, func() { + batcher := &Batcher{ + Config: Config{ + TimeoutMillis: 1000, + Limit: 10, + MaxPendingBatches: 10, + Deflate: true, + }, + } + + batcher.Init(0) + batcher.clk = clock.NewMock() + + Convey("When the max number of messages is reached", func() { + var messages []*utils.Message + + for i := 0; i < int(batcher.Config.Limit); i++ { + m := utils.NewMessage() + m.PushPayload([]byte("ABC")) + m.Opts = map[string]interface{}{ + "batch_group": "group1", + } + m.Reports.Push("Report") + + messages = append(messages, m) + } + + nd := new(NexterDoner) + nd.nextCalled = make(chan *utils.Message, 1) + nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) + + for i := 0; i < int(batcher.Config.Limit); i++ { + batcher.OnMessage(messages[i], nd.Next, nil) + } + + Convey("The batch should be sent compressed", func() { + decompressed := make([]byte, 30) + m := <-nd.nextCalled + nd.AssertExpectations(t) + + data, err := m.PopPayload() + buf := bytes.NewBuffer(data) + + r, err := zlib.NewReader(buf) + r.Read(decompressed) + r.Close() + + So(err, ShouldBeNil) + So(string(decompressed), ShouldEqual, "ABCABCABCABCABCABCABCABCABCABC") + So(m.Reports.Size(), ShouldEqual, batcher.Config.Limit) + So(batcher.batches["group1"], ShouldBeNil) + So(len(batcher.batches), ShouldEqual, 0) + }) + }) + }) } diff --git a/components/batch/config.go b/components/batch/config.go index f992080..879a159 100644 --- a/components/batch/config.go +++ b/components/batch/config.go @@ -2,6 +2,7 @@ package batcher // Config stores the config for a Batcher type Config struct { + Deflate bool TimeoutMillis uint Limit uint MaxPendingBatches uint From ca36edcb71b76ed73b7feb5c3eb6ed74d1299a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 16 Aug 2016 13:54:53 +0200 Subject: [PATCH 24/36] :white_check_mark: Add go 1.7 to the CI build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 88cd667..a538953 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ branches: go: - 1.5.4 - 1.6.3 + - 1.7 script: - $HOME/gopath/bin/goveralls From aac243271a76eff0862200a18b67e10fe712e9a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 09:57:19 +0200 Subject: [PATCH 25/36] :sparkles: Use concurrent map and next/done behavior change --- components/batch/batch.go | 23 +- components/batch/batcher.go | 73 ++-- components/batch/batcher_test.go | 220 +++++----- components/batch/config.go | 2 +- components/httpsender/httpSender.go | 22 +- components/httpsender/httpSender_test.go | 44 +- components/limiter/limiter.go | 2 +- components/limiter/limiter_test.go | 24 +- pipeline.go | 123 +++--- rbforwarder.go | 8 +- rbforwarder_test.go | 490 ----------------------- utils/composer.go | 6 +- utils/message.go | 5 +- 13 files changed, 297 insertions(+), 745 deletions(-) delete mode 100644 rbforwarder_test.go diff --git a/components/batch/batch.go b/components/batch/batch.go index 9cd6fe4..afcbf57 100644 --- a/components/batch/batch.go +++ b/components/batch/batch.go @@ -5,6 +5,8 @@ import ( "bytes" "compress/zlib" "io" + "sync" + "sync/atomic" "time" "github.com/benbjohnson/clock" @@ -18,22 +20,26 @@ type Batch struct { Message *utils.Message Buf *bytes.Buffer Writer io.Writer - MessageCount uint // Current number of messages in the buffer - Next utils.Next // Call to pass the message to the next handler + MessageCount uint64 // Current number of messages in the buffer + Done utils.Done // Call to pass the message to the next handler Timer *clock.Timer + Sent bool } // NewBatch creates a new instance of Batch -func NewBatch(m *utils.Message, group string, deflate bool, next utils.Next, +func NewBatch(m *utils.Message, group string, deflate bool, done utils.Done, clk clock.Clock, timeoutMillis uint, ready chan *Batch) *Batch { + var wg sync.WaitGroup + payload, _ := m.PopPayload() b := &Batch{ Group: group, Deflate: deflate, - Next: next, + Done: done, Message: m, MessageCount: 1, Buf: new(bytes.Buffer), + Sent: false, } if b.Deflate { @@ -44,17 +50,20 @@ func NewBatch(m *utils.Message, group string, deflate bool, next utils.Next, b.Writer.Write(payload) - if timeoutMillis != 0 { + if timeoutMillis > 0 { b.Timer = clk.Timer(time.Duration(timeoutMillis) * time.Millisecond) + wg.Add(1) go func() { + wg.Done() <-b.Timer.C - if b.MessageCount > 0 { + if atomic.LoadUint64(&b.MessageCount) > 0 { ready <- b } }() } + wg.Wait() return b } @@ -78,5 +87,5 @@ func (b *Batch) Add(m *utils.Message) { newPayload, _ := m.PopPayload() b.Writer.Write(newPayload) - b.MessageCount++ + atomic.AddUint64(&b.MessageCount, 1) } diff --git a/components/batch/batcher.go b/components/batch/batcher.go index a20110e..ebe98c4 100644 --- a/components/batch/batcher.go +++ b/components/batch/batcher.go @@ -5,81 +5,98 @@ import ( "github.com/benbjohnson/clock" "github.com/redBorder/rbforwarder/utils" + "github.com/streamrail/concurrent-map" ) // Batcher allows to merge multiple messages in a single one type Batcher struct { id int // Worker ID - wg sync.WaitGroup - batches map[string]*Batch // Collection of batches pending + batches cmap.ConcurrentMap readyBatches chan *Batch clk clock.Clock + finished chan struct{} incoming chan struct { m *utils.Message - next utils.Next + done utils.Done } Config Config // Batcher configuration } -// Init starts a gorutine that can receive: +// Spawn starts a gorutine that can receive: // - New messages that will be added to a existing or new batch of messages // - A batch of messages that is ready to send (i.e. batch timeout has expired) -func (b *Batcher) Init(id int) { +func (batcher *Batcher) Spawn(id int) utils.Composer { + var wg sync.WaitGroup + + b := *batcher + b.id = id - b.batches = make(map[string]*Batch) + b.batches = cmap.New() b.readyBatches = make(chan *Batch) + b.finished = make(chan struct{}) b.incoming = make(chan struct { m *utils.Message - next utils.Next + done utils.Done }) if b.clk == nil { b.clk = clock.New() } + wg.Add(1) go func() { + wg.Done() for { select { case message := <-b.incoming: - group, exists := message.m.Opts["batch_group"].(string) - if !exists { - message.next(message.m) + if !message.m.Opts.Has("batch_group") { + message.done(message.m, 0, "") } else { - if batch, exists := b.batches[group]; exists { + tmp, _ := message.m.Opts.Get("batch_group") + group := tmp.(string) + + if tmp, exists := b.batches.Get(group); exists { + batch := tmp.(*Batch) batch.Add(message.m) - if batch.MessageCount >= b.Config.Limit { + if batch.MessageCount >= b.Config.Limit && !batch.Sent { batch.Send(func() { - delete(b.batches, group) - batch.Timer.Stop() - batch.Next(batch.Message) + b.batches.Remove(group) + batch.Done(batch.Message, 0, "limit") + batch.Sent = true }) } } else { - b.batches[group] = NewBatch(message.m, group, b.Config.Deflate, - message.next, b.clk, b.Config.TimeoutMillis, b.readyBatches) + b.batches.Set(group, NewBatch(message.m, group, b.Config.Deflate, + message.done, b.clk, b.Config.TimeoutMillis, b.readyBatches)) } } - b.wg.Done() + b.finished <- struct{}{} case batch := <-b.readyBatches: - batch.Send(func() { - delete(b.batches, batch.Group) - batch.Next(batch.Message) - }) + if !batch.Sent { + batch.Send(func() { + b.batches.Remove(batch.Group) + batch.Done(batch.Message, 0, "timeout") + batch.Sent = true + }) + } } } }() + + wg.Wait() + return &b } // OnMessage is called when a new message is receive. Add the new message to // a batch -func (b *Batcher) OnMessage(m *utils.Message, next utils.Next, done utils.Done) { - b.wg.Add(1) - b.incoming <- struct { +func (batcher *Batcher) OnMessage(m *utils.Message, done utils.Done) { + batcher.incoming <- struct { m *utils.Message - next utils.Next - }{m, next} - b.wg.Wait() + done utils.Done + }{m, done} + + <-batcher.finished } diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go index 0b85a9a..656a8b8 100644 --- a/components/batch/batcher_test.go +++ b/components/batch/batcher_test.go @@ -10,17 +10,18 @@ import ( "github.com/benbjohnson/clock" "github.com/redBorder/rbforwarder/utils" . "github.com/smartystreets/goconvey/convey" + "github.com/streamrail/concurrent-map" "github.com/stretchr/testify/mock" ) -type NexterDoner struct { +type Doner struct { mock.Mock - nextCalled chan *utils.Message + doneCalled chan *utils.Message } -func (nd *NexterDoner) Next(m *utils.Message) { - nd.Called(m) - nd.nextCalled <- m +func (d *Doner) Done(m *utils.Message, code int, status string) { + d.Called(m, code, status) + d.doneCalled <- m } func TestBatcher(t *testing.T) { @@ -34,42 +35,42 @@ func TestBatcher(t *testing.T) { }, } - batcher.Init(0) - batcher.clk = clock.NewMock() + b := batcher.Spawn(0).(*Batcher) + b.clk = clock.NewMock() Convey("When a message is received with no batch group", func() { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - nd := new(NexterDoner) - nd.nextCalled = make(chan *utils.Message, 1) - nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) + d := new(Doner) + d.doneCalled = make(chan *utils.Message, 1) + d.On("Done", mock.AnythingOfType("*utils.Message"), 0, "").Times(1) - batcher.OnMessage(m, nd.Next, nil) + b.OnMessage(m, d.Done) + result := <-d.doneCalled - Convey("Message should be present on the batch", func() { - nd.AssertExpectations(t) - m := <-nd.nextCalled - So(len(batcher.batches), ShouldEqual, 0) - payload, err := m.PopPayload() + Convey("Then the message should be sent as is", func() { + So(b.batches.Count(), ShouldEqual, 0) + payload, err := result.PopPayload() So(err, ShouldBeNil) So(string(payload), ShouldEqual, "Hello World") + + d.AssertExpectations(t) }) }) - Convey("When a message is received, but not yet sent", func() { + Convey("When a message is received by the worker, but not yet sent", func() { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - m.Opts = map[string]interface{}{ - "batch_group": "group1", - } - m.Reports.Push("Report") + m.Opts = cmap.New() + m.Opts.Set("batch_group", "group1") - batcher.OnMessage(m, nil, nil) + b.OnMessage(m, nil) Convey("Message should be present on the batch", func() { - batch, exists := batcher.batches["group1"] + tmp, exists := b.batches.Get("group1") So(exists, ShouldBeTrue) + batch := tmp.(*Batch) batch.Writer.(*bufio.Writer).Flush() @@ -77,47 +78,45 @@ func TestBatcher(t *testing.T) { So(string(data), ShouldEqual, "Hello World") opts := batch.Message.Opts - So(opts["batch_group"], ShouldEqual, "group1") + group, ok := opts.Get("batch_group") + So(ok, ShouldBeTrue) + So(group.(string), ShouldEqual, "group1") - report := batch.Message.Reports.Pop().(string) - So(report, ShouldEqual, "Report") - - So(len(batcher.batches), ShouldEqual, 1) + So(b.batches.Count(), ShouldEqual, 1) }) }) Convey("When the max number of messages is reached", func() { var messages []*utils.Message - for i := 0; i < int(batcher.Config.Limit); i++ { + for i := 0; i < int(b.Config.Limit); i++ { m := utils.NewMessage() m.PushPayload([]byte("ABC")) - m.Opts = map[string]interface{}{ - "batch_group": "group1", - } - m.Reports.Push("Report") + m.Opts = cmap.New() + m.Opts.Set("batch_group", "group1") messages = append(messages, m) } - nd := new(NexterDoner) - nd.nextCalled = make(chan *utils.Message, 1) - nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) + d := new(Doner) + d.doneCalled = make(chan *utils.Message, 1) + d.On("Done", mock.AnythingOfType("*utils.Message"), 0, "limit").Times(1) - for i := 0; i < int(batcher.Config.Limit); i++ { - batcher.OnMessage(messages[i], nd.Next, nil) + for i := 0; i < int(b.Config.Limit); i++ { + b.OnMessage(messages[i], d.Done) } - Convey("The batch should be sent", func() { - m := <-nd.nextCalled - nd.AssertExpectations(t) + Convey("Then the batch should be sent", func() { + m := <-d.doneCalled data, err := m.PopPayload() So(err, ShouldBeNil) So(string(data), ShouldEqual, "ABCABCABCABCABCABCABCABCABCABC") - So(m.Reports.Size(), ShouldEqual, batcher.Config.Limit) - So(batcher.batches["group1"], ShouldBeNil) - So(len(batcher.batches), ShouldEqual, 0) + group1, _ := b.batches.Get("group1") + So(group1, ShouldBeNil) + So(b.batches.Count(), ShouldEqual, 0) + + d.AssertExpectations(t) }) }) @@ -127,32 +126,36 @@ func TestBatcher(t *testing.T) { for i := 0; i < 5; i++ { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - m.Opts = map[string]interface{}{ - "batch_group": "group1", - } - m.Reports.Push("Report") + m.Opts = cmap.New() + m.Opts.Set("batch_group", "group1") messages = append(messages, m) } - nd := new(NexterDoner) - nd.nextCalled = make(chan *utils.Message, 1) - nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) + d := new(Doner) + d.doneCalled = make(chan *utils.Message, 1) + d.On("Done", mock.AnythingOfType("*utils.Message"), 0, "timeout").Times(1) for i := 0; i < 5; i++ { - batcher.OnMessage(messages[i], nd.Next, nil) + b.OnMessage(messages[i], d.Done) } - clk := batcher.clk.(*clock.Mock) + clk := b.clk.(*clock.Mock) Convey("The batch should be sent", func() { clk.Add(500 * time.Millisecond) - So(batcher.batches["group1"], ShouldNotBeNil) + + group1, _ := b.batches.Get("group1") + So(group1.(*Batch), ShouldNotBeNil) + clk.Add(500 * time.Millisecond) - <-nd.nextCalled - So(batcher.batches["group1"], ShouldBeNil) - So(len(batcher.batches), ShouldEqual, 0) - nd.AssertExpectations(t) + + <-d.doneCalled + group1, _ = b.batches.Get("group1") + So(group1, ShouldBeNil) + So(b.batches.Count(), ShouldEqual, 0) + + d.AssertExpectations(t) }) }) @@ -160,56 +163,58 @@ func TestBatcher(t *testing.T) { m1 := utils.NewMessage() m1.PushPayload([]byte("MESSAGE 1")) m1.Reports.Push("Report 1") - m1.Opts = map[string]interface{}{ - "batch_group": "group1", - } + m1.Opts = cmap.New() + m1.Opts.Set("batch_group", "group1") m2 := utils.NewMessage() m2.PushPayload([]byte("MESSAGE 2")) m2.Reports.Push("Report 2") - m2.Opts = map[string]interface{}{ - "batch_group": "group2", - } + m2.Opts = cmap.New() + m2.Opts.Set("batch_group", "group2") + m3 := utils.NewMessage() m3.PushPayload([]byte("MESSAGE 3")) m3.Reports.Push("Report 3") - m3.Opts = map[string]interface{}{ - "batch_group": "group2", - } + m3.Opts = cmap.New() + m3.Opts.Set("batch_group", "group2") - nd := new(NexterDoner) - nd.nextCalled = make(chan *utils.Message, 2) - nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(2) + d := new(Doner) + d.doneCalled = make(chan *utils.Message, 2) + d.On("Done", mock.AnythingOfType("*utils.Message"), 0, "timeout").Times(2) - batcher.OnMessage(m1, nd.Next, nil) - batcher.OnMessage(m2, nd.Next, nil) - batcher.OnMessage(m3, nd.Next, nil) + b.OnMessage(m1, d.Done) + b.OnMessage(m2, d.Done) + b.OnMessage(m3, d.Done) - Convey("Each message should be in its group", func() { - batcher.batches["group1"].Writer.(*bufio.Writer).Flush() - group1 := batcher.batches["group1"].Buf.Bytes() + Convey("Then each message should be in its group", func() { + tmp, _ := b.batches.Get("group1") + batch := tmp.(*Batch) + batch.Writer.(*bufio.Writer).Flush() + group1 := batch.Buf.Bytes() So(string(group1), ShouldEqual, "MESSAGE 1") - batcher.batches["group2"].Writer.(*bufio.Writer).Flush() - group2 := batcher.batches["group2"].Buf.Bytes() + tmp, _ = b.batches.Get("group2") + batch = tmp.(*Batch) + batch.Writer.(*bufio.Writer).Flush() + group2 := batch.Buf.Bytes() So(string(group2), ShouldEqual, "MESSAGE 2MESSAGE 3") - So(len(batcher.batches), ShouldEqual, 2) + So(b.batches.Count(), ShouldEqual, 2) }) - Convey("After a timeout the messages should be sent", func() { - clk := batcher.clk.(*clock.Mock) - So(len(batcher.batches), ShouldEqual, 2) + Convey("Then after a timeout the messages should be sent", func() { + clk := b.clk.(*clock.Mock) + So(b.batches.Count(), ShouldEqual, 2) - clk.Add(time.Duration(batcher.Config.TimeoutMillis) * time.Millisecond) + clk.Add(time.Duration(b.Config.TimeoutMillis) * time.Millisecond) - group1 := <-nd.nextCalled + group1 := <-d.doneCalled group1Data, err := group1.PopPayload() report1 := group1.Reports.Pop().(string) So(err, ShouldBeNil) So(report1, ShouldEqual, "Report 1") - group2 := <-nd.nextCalled + group2 := <-d.doneCalled group2Data, err := group2.PopPayload() So(err, ShouldBeNil) report3 := group2.Reports.Pop().(string) @@ -219,11 +224,14 @@ func TestBatcher(t *testing.T) { So(string(group1Data), ShouldEqual, "MESSAGE 1") So(string(group2Data), ShouldEqual, "MESSAGE 2MESSAGE 3") - So(batcher.batches["group1"], ShouldBeNil) - So(batcher.batches["group2"], ShouldBeNil) - So(len(batcher.batches), ShouldEqual, 0) + batch, _ := b.batches.Get("group1") + So(batch, ShouldBeNil) + + batch, _ = b.batches.Get("group2") + So(batch, ShouldBeNil) + So(b.batches.Count(), ShouldEqual, 0) - nd.AssertExpectations(t) + d.AssertExpectations(t) }) }) }) @@ -238,36 +246,33 @@ func TestBatcher(t *testing.T) { }, } - batcher.Init(0) - batcher.clk = clock.NewMock() + b := batcher.Spawn(0).(*Batcher) + b.clk = clock.NewMock() Convey("When the max number of messages is reached", func() { var messages []*utils.Message - for i := 0; i < int(batcher.Config.Limit); i++ { + for i := 0; i < int(b.Config.Limit); i++ { m := utils.NewMessage() m.PushPayload([]byte("ABC")) - m.Opts = map[string]interface{}{ - "batch_group": "group1", - } + m.Opts = cmap.New() + m.Opts.Set("batch_group", "group1") m.Reports.Push("Report") messages = append(messages, m) } - nd := new(NexterDoner) - nd.nextCalled = make(chan *utils.Message, 1) - nd.On("Next", mock.AnythingOfType("*utils.Message")).Times(1) + d := new(Doner) + d.doneCalled = make(chan *utils.Message, 1) + d.On("Done", mock.AnythingOfType("*utils.Message"), 0, "limit").Times(1) - for i := 0; i < int(batcher.Config.Limit); i++ { - batcher.OnMessage(messages[i], nd.Next, nil) + for i := 0; i < int(b.Config.Limit); i++ { + b.OnMessage(messages[i], d.Done) } Convey("The batch should be sent compressed", func() { + m := <-d.doneCalled decompressed := make([]byte, 30) - m := <-nd.nextCalled - nd.AssertExpectations(t) - data, err := m.PopPayload() buf := bytes.NewBuffer(data) @@ -277,9 +282,12 @@ func TestBatcher(t *testing.T) { So(err, ShouldBeNil) So(string(decompressed), ShouldEqual, "ABCABCABCABCABCABCABCABCABCABC") - So(m.Reports.Size(), ShouldEqual, batcher.Config.Limit) - So(batcher.batches["group1"], ShouldBeNil) - So(len(batcher.batches), ShouldEqual, 0) + So(m.Reports.Size(), ShouldEqual, b.Config.Limit) + batch, _ := b.batches.Get("group1") + So(batch, ShouldBeNil) + So(b.batches.Count(), ShouldEqual, 0) + + d.AssertExpectations(t) }) }) }) diff --git a/components/batch/config.go b/components/batch/config.go index 879a159..3788c53 100644 --- a/components/batch/config.go +++ b/components/batch/config.go @@ -4,6 +4,6 @@ package batcher type Config struct { Deflate bool TimeoutMillis uint - Limit uint + Limit uint64 MaxPendingBatches uint } diff --git a/components/httpsender/httpSender.go b/components/httpsender/httpSender.go index 7daabcd..6ea0f67 100644 --- a/components/httpsender/httpSender.go +++ b/components/httpsender/httpSender.go @@ -21,8 +21,10 @@ type HTTPSender struct { Client *http.Client } -// Init initializes the HTTP component -func (s *HTTPSender) Init(id int) { +// Spawn initializes the HTTP component +func (httpsender *HTTPSender) Spawn(id int) utils.Composer { + s := *httpsender + s.id = id if govalidator.IsURL(s.URL) { @@ -32,14 +34,16 @@ func (s *HTTPSender) Init(id int) { } else { s.err = errors.New("Invalid URL") } + + return &s } // OnMessage is called when a new message should be sent via HTTP -func (s *HTTPSender) OnMessage(m *utils.Message, next utils.Next, done utils.Done) { +func (httpsender *HTTPSender) OnMessage(m *utils.Message, done utils.Done) { var u string - if s.err != nil { - done(m, 2, s.err.Error()) + if httpsender.err != nil { + done(m, 2, httpsender.err.Error()) return } @@ -49,14 +53,14 @@ func (s *HTTPSender) OnMessage(m *utils.Message, next utils.Next, done utils.Don return } - if endpoint, exists := m.Opts["http_endpoint"]; exists { - u = s.URL + "/" + endpoint.(string) + if endpoint, exists := m.Opts.Get("http_endpoint"); exists { + u = httpsender.URL + "/" + endpoint.(string) } else { - u = s.URL + u = httpsender.URL } buf := bytes.NewBuffer(data) - res, err := s.Client.Post(u, "", buf) + res, err := httpsender.Client.Post(u, "", buf) if err != nil { done(m, 1, "HTTPSender error: "+err.Error()) return diff --git a/components/httpsender/httpSender_test.go b/components/httpsender/httpSender_test.go index 207aefe..fbe84cf 100644 --- a/components/httpsender/httpSender_test.go +++ b/components/httpsender/httpSender_test.go @@ -53,20 +53,20 @@ func TestHTTPSender(t *testing.T) { } Convey("When is initialized", func() { - sender.Init(0) + s := sender.Spawn(0).(*HTTPSender) Convey("Then the config should be ok", func() { - So(sender.Client, ShouldNotBeNil) + So(s.Client, ShouldNotBeNil) }) }) Convey("When a message is sent and the response code is >= 400", func() { var url string - sender.Init(0) + s := sender.Spawn(0).(*HTTPSender) m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - sender.Client = NewTestClient(401, func(req *http.Request) { + s.Client = NewTestClient(401, func(req *http.Request) { url = req.URL.String() }) @@ -80,7 +80,7 @@ func TestHTTPSender(t *testing.T) { d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) - sender.OnMessage(m, nil, d.Done) + s.OnMessage(m, d.Done) Convey("Then the reporth should contain info about the error", func() { result := <-d.doneCalled @@ -94,12 +94,12 @@ func TestHTTPSender(t *testing.T) { Convey("When a message is received without endpoint option", func() { var url string - sender.Init(0) + s := sender.Spawn(0).(*HTTPSender) m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - sender.Client = NewTestClient(200, func(req *http.Request) { + s.Client = NewTestClient(200, func(req *http.Request) { url = req.URL.String() }) @@ -112,7 +112,7 @@ func TestHTTPSender(t *testing.T) { d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) - sender.OnMessage(m, nil, d.Done) + s.OnMessage(m, d.Done) Convey("Then the message should be sent via HTTP to the URL", func() { result := <-d.doneCalled @@ -126,13 +126,13 @@ func TestHTTPSender(t *testing.T) { Convey("When a message is received with endpoint option", func() { var url string - sender.Init(0) + s := sender.Spawn(0).(*HTTPSender) m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - m.Opts["http_endpoint"] = "endpoint1" + m.Opts.Set("http_endpoint", "endpoint1") - sender.Client = NewTestClient(200, func(req *http.Request) { + s.Client = NewTestClient(200, func(req *http.Request) { url = req.URL.String() }) @@ -145,7 +145,7 @@ func TestHTTPSender(t *testing.T) { d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) - sender.OnMessage(m, nil, d.Done) + s.OnMessage(m, d.Done) Convey("Then the message should be sent to the URL with endpoint as suffix", func() { result := <-d.doneCalled @@ -159,11 +159,11 @@ func TestHTTPSender(t *testing.T) { Convey("When a message without payload is received", func() { var url string - sender.Init(0) + s := sender.Spawn(0).(*HTTPSender) m := utils.NewMessage() - sender.Client = NewTestClient(200, func(req *http.Request) { + s.Client = NewTestClient(200, func(req *http.Request) { url = req.URL.String() }) @@ -176,7 +176,7 @@ func TestHTTPSender(t *testing.T) { d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) - sender.OnMessage(m, nil, d.Done) + s.OnMessage(m, d.Done) Convey("Then the message should not be sent", func() { result := <-d.doneCalled @@ -189,12 +189,12 @@ func TestHTTPSender(t *testing.T) { }) Convey("When a the HTTP client fails", func() { - sender.Init(0) + s := sender.Spawn(0).(*HTTPSender) m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - sender.Client = NewTestClient(200, func(req *http.Request) { + s.Client = NewTestClient(200, func(req *http.Request) { req.Write(nil) }) @@ -207,7 +207,7 @@ func TestHTTPSender(t *testing.T) { d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) - sender.OnMessage(m, nil, d.Done) + s.OnMessage(m, d.Done) Convey("Then the message should not be sent", func() { result := <-d.doneCalled @@ -221,12 +221,12 @@ func TestHTTPSender(t *testing.T) { Convey("Given an HTTP sender with invalid URL", t, func() { sender := &HTTPSender{} - sender.Init(0) + s := sender.Spawn(0).(*HTTPSender) Convey("When try to send messages", func() { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) - m.Opts["http_endpoint"] = "endpoint1" + m.Opts.Set("http_endpoint", "endpoint1") d := &Doner{ doneCalled: make(chan struct { @@ -238,10 +238,10 @@ func TestHTTPSender(t *testing.T) { d.On("Done", mock.AnythingOfType("*utils.Message"), mock.AnythingOfType("int"), mock.AnythingOfType("string")) - sender.OnMessage(m, nil, d.Done) + s.OnMessage(m, d.Done) Convey("Then should fail to send messages", func() { - So(sender.err, ShouldNotBeNil) + So(s.err, ShouldNotBeNil) result := <-d.doneCalled So(result.status, ShouldEqual, "Invalid URL") So(result.code, ShouldBeGreaterThan, 0) diff --git a/components/limiter/limiter.go b/components/limiter/limiter.go index 0da7d13..664519c 100644 --- a/components/limiter/limiter.go +++ b/components/limiter/limiter.go @@ -49,7 +49,7 @@ func (l *Limiter) OnMessage(m *utils.Message, next utils.Next, done utils.Done) m.PushPayload(payload) } } - next(m) + done(m, 0, "") if l.config.MessageLimit > 0 && l.currentMessages >= l.config.MessageLimit || l.config.BytesLimit > 0 && l.currentBytes >= l.config.BytesLimit { diff --git a/components/limiter/limiter_test.go b/components/limiter/limiter_test.go index ce58650..e69d9e7 100644 --- a/components/limiter/limiter_test.go +++ b/components/limiter/limiter_test.go @@ -15,8 +15,8 @@ type Nexter struct { nextCalled chan *utils.Message } -func (n *Nexter) Next(m *utils.Message) { - n.Called(m) +func (n *Nexter) Done(m *utils.Message, code int, status string) { + n.Called(m, code, status) n.nextCalled <- m } @@ -36,10 +36,10 @@ func TestHTTPSender(t *testing.T) { n := Nexter{ nextCalled: make(chan *utils.Message, limiter.config.MessageLimit*2), } - n.On("Next", mock.AnythingOfType("*utils.Message")) + n.On("Done", mock.AnythingOfType("*utils.Message"), 0, "") for i := uint64(0); i < limiter.config.MessageLimit; i++ { - limiter.OnMessage(nil, n.Next, nil) + limiter.OnMessage(nil, nil, n.Done) } Convey("Then the limiter should be paused", func() { @@ -49,7 +49,7 @@ func TestHTTPSender(t *testing.T) { Convey("Then after 1 second the limiter should be ready again", func() { clk.Add(1 * time.Second) - limiter.OnMessage(nil, n.Next, nil) + limiter.OnMessage(nil, nil, n.Done) So(limiter.currentMessages, ShouldEqual, 1) So(limiter.paused, ShouldBeFalse) }) @@ -70,14 +70,14 @@ func TestHTTPSender(t *testing.T) { n := Nexter{ nextCalled: make(chan *utils.Message, 100), } - n.On("Next", mock.AnythingOfType("*utils.Message")) + n.On("Done", mock.AnythingOfType("*utils.Message"), 0, "") Convey("Then the limiter should not be paused after 750 bytes", func() { for i := uint64(0); i < 3; i++ { m := utils.NewMessage() payload := make([]byte, 250) m.PushPayload(payload) - limiter.OnMessage(m, n.Next, nil) + limiter.OnMessage(m, nil, n.Done) } So(limiter.currentBytes, ShouldEqual, 750) @@ -89,7 +89,7 @@ func TestHTTPSender(t *testing.T) { m := utils.NewMessage() payload := make([]byte, 250) m.PushPayload(payload) - limiter.OnMessage(m, n.Next, nil) + limiter.OnMessage(m, nil, n.Done) } So(limiter.currentBytes, ShouldEqual, 1000) @@ -103,7 +103,7 @@ func TestHTTPSender(t *testing.T) { m := utils.NewMessage() payload := make([]byte, 250) m.PushPayload(payload) - limiter.OnMessage(m, n.Next, nil) + limiter.OnMessage(m, nil, n.Done) So(limiter.currentBytes, ShouldEqual, 250) So(limiter.paused, ShouldBeFalse) @@ -129,10 +129,10 @@ func TestHTTPSender(t *testing.T) { n := Nexter{ nextCalled: make(chan *utils.Message, limiter.config.MessageLimit*2), } - n.On("Next", mock.AnythingOfType("*utils.Message")) + n.On("Done", mock.AnythingOfType("*utils.Message"), 0, "") for i := uint64(0); i < limiter.config.MessageLimit; i++ { - limiter.OnMessage(nil, n.Next, nil) + limiter.OnMessage(nil, nil, n.Done) } Convey("Then should be 2 burst available", func() { @@ -140,7 +140,7 @@ func TestHTTPSender(t *testing.T) { }) Convey("Then messages are not blocked after the limit", func() { for i := uint64(0); i < limiter.config.MessageLimit; i++ { - limiter.OnMessage(nil, n.Next, nil) + limiter.OnMessage(nil, nil, n.Done) } So(limiter.currentMessages, ShouldEqual, 100) }) diff --git a/pipeline.go b/pipeline.go index 2f4ca84..bc84f68 100644 --- a/pipeline.go +++ b/pipeline.go @@ -1,20 +1,20 @@ package rbforwarder import ( - "sync" - "github.com/oleiade/lane" "github.com/redBorder/rbforwarder/utils" ) -type component struct { - pool chan chan *utils.Message - workers int +// Component contains information about a pipeline component +type Component struct { + workers int + composser utils.Composer + pool chan chan *utils.Message } // pipeline contains the components type pipeline struct { - components []component + components []Component input chan *utils.Message retry chan *utils.Message output chan *utils.Message @@ -22,20 +22,70 @@ type pipeline struct { // newPipeline creates a new Backend func newPipeline(input, retry, output chan *utils.Message) *pipeline { - var wg sync.WaitGroup - p := &pipeline{ + return &pipeline{ input: input, retry: retry, output: output, } +} - wg.Add(1) - go func() { - wg.Done() +// PushComponent adds a new component to the pipeline +func (p *pipeline) PushComponent(composser utils.Composer, w int) { + p.components = append(p.components, struct { + workers int + composser utils.Composer + pool chan chan *utils.Message + }{ + workers: w, + composser: composser, + pool: make(chan chan *utils.Message, w), + }) +} +func (p *pipeline) Run() { + for index, component := range p.components { + for w := 0; w < component.workers; w++ { + go func(w, index int, component Component) { + component.composser = component.composser.Spawn(w) + messages := make(chan *utils.Message) + component.pool <- messages + + for m := range messages { + component.composser.OnMessage(m, + // Done function + func(m *utils.Message, code int, status string) { + // If there is another component next in the pipeline send the + // messate to it. I other case send the message to the report + // handler + if len(p.components)-1 > index { + nextWorker := <-p.components[index+1].pool + nextWorker <- m + } else { + reports := lane.NewStack() + + for !m.Reports.Empty() { + rep := m.Reports.Pop().(Report) + rep.Code = code + rep.Status = status + reports.Push(rep) + } + + m.Reports = reports + p.output <- m + } + }) + + component.pool <- messages + } + }(w, index, component) + } + } + + go func() { for { select { case m, ok := <-p.input: + // If input channel has been closed, close output channel if !ok { for _, component := range p.components { @@ -56,55 +106,4 @@ func newPipeline(input, retry, output chan *utils.Message) *pipeline { } } }() - - wg.Wait() - return p -} - -// PushComponent adds a new component to the pipeline -func (p *pipeline) PushComponent(composser utils.Composer, w int) { - var wg sync.WaitGroup - c := component{ - workers: w, - pool: make(chan chan *utils.Message, w), - } - - index := len(p.components) - p.components = append(p.components, c) - - for i := 0; i < w; i++ { - composser.Init(i) - - worker := make(chan *utils.Message) - p.components[index].pool <- worker - - wg.Add(1) - go func(i int) { - wg.Done() - for m := range worker { - composser.OnMessage(m, func(m *utils.Message) { - if len(p.components) >= index { - nextWorker := <-p.components[index+1].pool - nextWorker <- m - } - }, func(m *utils.Message, code int, status string) { - reports := lane.NewStack() - - for !m.Reports.Empty() { - rep := m.Reports.Pop().(Report) - rep.Code = code - rep.Status = status - reports.Push(rep) - } - - m.Reports = reports - p.output <- m - }) - - p.components[index].pool <- worker - } - }(i) - } - - wg.Wait() } diff --git a/rbforwarder.go b/rbforwarder.go index 72b2dc9..f25610b 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -55,6 +55,11 @@ func NewRBForwarder(config Config) *RBForwarder { return f } +// Run starts getting messages +func (f *RBForwarder) Run() { + f.p.Run() +} + // Close stops pending actions func (f *RBForwarder) Close() { atomic.StoreUint32(&f.working, 0) @@ -93,9 +98,8 @@ func (f *RBForwarder) Produce(data []byte, opts map[string]interface{}, opaque i seq: seq, Opaque: opaque, } - m.PushPayload(data) - m.Opts = opts + m.Opts.MSet(opts) m.Reports.Push(r) f.p.input <- m diff --git a/rbforwarder_test.go b/rbforwarder_test.go deleted file mode 100644 index 1933f4d..0000000 --- a/rbforwarder_test.go +++ /dev/null @@ -1,490 +0,0 @@ -package rbforwarder - -import ( - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/redBorder/rbforwarder/components/batch" - "github.com/redBorder/rbforwarder/components/httpsender" - "github.com/redBorder/rbforwarder/utils" - . "github.com/smartystreets/goconvey/convey" - "github.com/stretchr/testify/mock" -) - -type MockMiddleComponent struct { - mock.Mock -} - -func (c *MockMiddleComponent) Init(id int) { - c.Called() - return -} - -func (c *MockMiddleComponent) OnMessage( - m *utils.Message, - next utils.Next, - done utils.Done, -) { - c.Called(m) - if data, err := m.PopPayload(); err == nil { - processedData := "-> [" + string(data) + "] <-" - m.PushPayload([]byte(processedData)) - } - - next(m) -} - -type MockComponent struct { - mock.Mock - - channel chan string - - status string - statusCode int -} - -func (c *MockComponent) Init(id int) { - c.Called() -} - -func (c *MockComponent) OnMessage( - m *utils.Message, - next utils.Next, - done utils.Done, -) { - c.Called(m) - if data, err := m.PopPayload(); err == nil { - c.channel <- string(data) - } else { - c.channel <- err.Error() - } - - done(m, c.statusCode, c.status) -} - -func TestRBForwarder(t *testing.T) { - Convey("Given a single component working pipeline", t, func() { - numMessages := 10000 - numWorkers := 10 - numRetries := 3 - - component := &MockComponent{ - channel: make(chan string, 10000), - } - - rbforwarder := NewRBForwarder(Config{ - Retries: numRetries, - QueueSize: numMessages, - }) - - component.On("Init").Return(nil).Times(numWorkers) - - var components []interface{} - var instances []int - components = append(components, component) - instances = append(instances, numWorkers) - - rbforwarder.PushComponents(components, instances) - - //////////////////////////////////////////////////////////////////////////// - - Convey("When a \"Hello World\" message is produced", func() { - component.status = "OK" - component.statusCode = 0 - - component.On("OnMessage", mock.AnythingOfType("*utils.Message")).Times(1) - - err := rbforwarder.Produce( - []byte("Hello World"), - map[string]interface{}{"message_id": "test123"}, - nil, - ) - - Convey("\"Hello World\" message should be get by the worker", func() { - var lastReport Report - var reports int - for r := range rbforwarder.GetReports() { - reports++ - lastReport = r.(Report) - rbforwarder.Close() - } - - So(lastReport, ShouldNotBeNil) - So(reports, ShouldEqual, 1) - So(lastReport.Code, ShouldEqual, 0) - So(lastReport.Status, ShouldEqual, "OK") - So(err, ShouldBeNil) - - component.AssertExpectations(t) - }) - }) - - // //////////////////////////////////////////////////////////////////////////// - - Convey("When a message is produced after close forwarder", func() { - rbforwarder.Close() - - err := rbforwarder.Produce( - []byte("Hello World"), - map[string]interface{}{"message_id": "test123"}, - nil, - ) - - Convey("Should error", func() { - So(err.Error(), ShouldEqual, "Forwarder has been closed") - }) - }) - - //////////////////////////////////////////////////////////////////////////// - - Convey("When calling OnMessage() with opaque", func() { - component.On("OnMessage", mock.AnythingOfType("*utils.Message")) - - err := rbforwarder.Produce( - []byte("Hello World"), - nil, - "This is an opaque", - ) - - Convey("Should be possible to read the opaque", func() { - So(err, ShouldBeNil) - - var reports int - var lastReport Report - for r := range rbforwarder.GetReports() { - reports++ - lastReport = r.(Report) - rbforwarder.Close() - } - - opaque := lastReport.Opaque.(string) - So(opaque, ShouldEqual, "This is an opaque") - }) - }) - - //////////////////////////////////////////////////////////////////////////// - - Convey("When a message fails to send", func() { - component.status = "Fake Error" - component.statusCode = 99 - - component.On("OnMessage", mock.AnythingOfType("*utils.Message")).Times(4) - - err := rbforwarder.Produce( - []byte("Hello World"), - map[string]interface{}{"message_id": "test123"}, - nil, - ) - - Convey("The message should be retried", func() { - So(err, ShouldBeNil) - - var reports int - var lastReport Report - for r := range rbforwarder.GetReports() { - reports++ - lastReport = r.(Report) - rbforwarder.Close() - } - - So(lastReport, ShouldNotBeNil) - So(reports, ShouldEqual, 1) - So(lastReport.Status, ShouldEqual, "Fake Error") - So(lastReport.Code, ShouldEqual, 99) - So(lastReport.retries, ShouldEqual, numRetries) - - component.AssertExpectations(t) - }) - }) - - //////////////////////////////////////////////////////////////////////////// - - Convey("When 10000 messages are produced", func() { - var numErr int - - component.On("OnMessage", mock.AnythingOfType("*utils.Message")). - Return(nil). - Times(numMessages) - - for i := 0; i < numMessages; i++ { - if err := rbforwarder.Produce([]byte("Hello World"), - nil, - i, - ); err != nil { - numErr++ - } - } - - Convey("10000 reports should be received", func() { - var reports int - for range rbforwarder.GetReports() { - reports++ - if reports >= numMessages { - rbforwarder.Close() - } - } - - So(numErr, ShouldBeZeroValue) - So(reports, ShouldEqual, numMessages) - - component.AssertExpectations(t) - }) - - Convey("10000 reports should be received in order", func() { - ordered := true - var reports int - - for rep := range rbforwarder.GetOrderedReports() { - if rep.(Report).Opaque.(int) != reports { - ordered = false - } - reports++ - if reports >= numMessages { - rbforwarder.Close() - } - } - - So(numErr, ShouldBeZeroValue) - So(ordered, ShouldBeTrue) - So(reports, ShouldEqual, numMessages) - - component.AssertExpectations(t) - }) - }) - }) - - Convey("Given a multi-component working pipeline", t, func() { - numMessages := 100 - numWorkers := 3 - numRetries := 3 - - component1 := &MockMiddleComponent{} - component2 := &MockComponent{ - channel: make(chan string, 10000), - } - - rbforwarder := NewRBForwarder(Config{ - Retries: numRetries, - QueueSize: numMessages, - }) - - for i := 0; i < numWorkers; i++ { - component1.On("Init").Return(nil) - component2.On("Init").Return(nil) - } - - var components []interface{} - var instances []int - - components = append(components, component1) - components = append(components, component2) - - instances = append(instances, numWorkers) - instances = append(instances, numWorkers) - - rbforwarder.PushComponents(components, instances) - - Convey("When a \"Hello World\" message is produced", func() { - component2.status = "OK" - component2.statusCode = 0 - - component1.On("OnMessage", mock.AnythingOfType("*utils.Message")) - component2.On("OnMessage", mock.AnythingOfType("*utils.Message")) - - err := rbforwarder.Produce( - []byte("Hello World"), - map[string]interface{}{"message_id": "test123"}, - nil, - ) - - rbforwarder.Close() - - Convey("\"Hello World\" message should be processed by the pipeline", func() { - reports := 0 - for rep := range rbforwarder.GetReports() { - reports++ - - code := rep.(Report).Code - status := rep.(Report).Status - So(code, ShouldEqual, 0) - So(status, ShouldEqual, "OK") - } - - m := <-component2.channel - - So(err, ShouldBeNil) - So(reports, ShouldEqual, 1) - So(m, ShouldEqual, "-> [Hello World] <-") - - component1.AssertExpectations(t) - component2.AssertExpectations(t) - }) - }) - }) -} - -func BenchmarkNoBatch(b *testing.B) { - var components []interface{} - var workers []int - - f := NewRBForwarder(Config{ - Retries: 3, - Backoff: 5, - QueueSize: 1, - }) - - batch := &batcher.Batcher{ - Config: batcher.Config{ - TimeoutMillis: 1000, - Limit: 10000, - }, - } - components = append(components, batch) - workers = append(workers, 1) - - sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), - } - components = append(components, sender) - workers = append(workers, 1) - - f.PushComponents(components, workers) - - opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", - } - - for i := 0; i < b.N; i++ { - data := fmt.Sprintf("{\"message\": %d}", i) - f.Produce([]byte(data), opts, i) - } - - for report := range f.GetReports() { - r := report.(Report) - if r.Code > 0 { - b.FailNow() - } - if r.Opaque.(int) == b.N-1 { - break - } - } -} - -func NewTestClient(code int, cb func(*http.Request)) *http.Client { - server := httptest.NewServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(code) - cb(r) - })) - - transport := &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, - } - - return &http.Client{Transport: transport} -} - -func BenchmarkLittleBatch(b *testing.B) { - var components []interface{} - var workers []int - - f := NewRBForwarder(Config{ - Retries: 3, - Backoff: 5, - QueueSize: b.N / 100, - }) - - batch := &batcher.Batcher{ - Config: batcher.Config{ - TimeoutMillis: 1000, - Limit: 10000, - }, - } - components = append(components, batch) - workers = append(workers, 1) - - sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), - } - components = append(components, sender) - workers = append(workers, 1) - - f.PushComponents(components, workers) - - opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", - } - - for i := 0; i < b.N; i++ { - data := fmt.Sprintf("{\"message\": %d}", i) - f.Produce([]byte(data), opts, i) - } - - for report := range f.GetReports() { - r := report.(Report) - if r.Code > 0 { - b.FailNow() - } - if r.Opaque.(int) == b.N-1 { - break - } - } -} - -func BenchmarkBigBatch(b *testing.B) { - var components []interface{} - var workers []int - - f := NewRBForwarder(Config{ - Retries: 3, - Backoff: 5, - QueueSize: b.N / 10, - }) - - batch := &batcher.Batcher{ - Config: batcher.Config{ - TimeoutMillis: 1000, - Limit: 10000, - }, - } - components = append(components, batch) - workers = append(workers, 1) - - sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), - } - components = append(components, sender) - workers = append(workers, 1) - - f.PushComponents(components, workers) - - opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", - } - - for i := 0; i < b.N; i++ { - data := fmt.Sprintf("{\"message\": %d}", i) - f.Produce([]byte(data), opts, i) - } - - for report := range f.GetReports() { - r := report.(Report) - if r.Code > 0 { - b.FailNow() - } - if r.Opaque.(int) == b.N-1 { - break - } - } -} diff --git a/utils/composer.go b/utils/composer.go index 1ba831c..424e869 100644 --- a/utils/composer.go +++ b/utils/composer.go @@ -2,7 +2,7 @@ package utils // Next should be called by a component in order to pass the message to the next // component in the pipeline. -type Next func(*Message) +type Next func() // Done should be called by a component in order to return the message to the // message handler. Can be used by the last component to inform that the @@ -12,6 +12,6 @@ type Done func(*Message, int, string) // Composer represents a component in the pipeline that performs a work on // a message type Composer interface { - Init(int) - OnMessage(*Message, Next, Done) + Spawn(int) Composer + OnMessage(*Message, Done) } diff --git a/utils/message.go b/utils/message.go index c06b042..78a6290 100644 --- a/utils/message.go +++ b/utils/message.go @@ -4,11 +4,12 @@ import ( "errors" "github.com/oleiade/lane" + "github.com/streamrail/concurrent-map" ) // Message is used to send data through the pipeline type Message struct { - Opts map[string]interface{} + Opts cmap.ConcurrentMap Reports *lane.Stack payload *lane.Stack @@ -19,7 +20,7 @@ func NewMessage() *Message { return &Message{ payload: lane.NewStack(), Reports: lane.NewStack(), - Opts: make(map[string]interface{}), + Opts: cmap.New(), } } From 97619dcfcabca0454411eafbcf557c5fcf732314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 10:02:32 +0200 Subject: [PATCH 26/36] :lipstick: Reorganize tests and benchmarks --- Makefile | 6 +- benchmarks_test.go | 178 +++++++++++++++++++++++ integration_test.go | 318 ++++++++++++++++++++++++++++++++++++++++++ pipeline_test.go | 1 + reportHandler_test.go | 1 + 5 files changed, 501 insertions(+), 3 deletions(-) create mode 100644 benchmarks_test.go create mode 100644 integration_test.go create mode 100644 pipeline_test.go create mode 100644 reportHandler_test.go diff --git a/Makefile b/Makefile index 95830e2..27de9bc 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,12 @@ errcheck: errcheck -ignoretests -verbose ./... vet: - @printf "$(MKL_YELLOW)Runing go vet$(MKL_CLR_RESET)\n" + @printf "$(MKL_YELLOW)Running go vet$(MKL_CLR_RESET)\n" go vet ./... test: - @printf "$(MKL_YELLOW)Runing tests$(MKL_CLR_RESET)\n" - go test -race ./... + @printf "$(MKL_YELLOW)Running tests$(MKL_CLR_RESET)\n" + go test -race ./... -tags=integration @printf "$(MKL_GREEN)Test passed$(MKL_CLR_RESET)\n" coverage: diff --git a/benchmarks_test.go b/benchmarks_test.go new file mode 100644 index 0000000..af4e91f --- /dev/null +++ b/benchmarks_test.go @@ -0,0 +1,178 @@ +package rbforwarder + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/redBorder/rbforwarder/components/batch" + "github.com/redBorder/rbforwarder/components/httpsender" +) + +func NewTestClient(code int, cb func(*http.Request)) *http.Client { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + cb(r) + })) + + transport := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + return &http.Client{Transport: transport} +} + +func BenchmarkNoBatch(b *testing.B) { + var components []interface{} + var workers []int + + f := NewRBForwarder(Config{ + Retries: 3, + Backoff: 5, + QueueSize: 1, + }) + + batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 10000, + }, + } + components = append(components, batch) + workers = append(workers, 1) + + sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + } + components = append(components, sender) + workers = append(workers, 1) + + f.PushComponents(components, workers) + f.Run() + + opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", + } + + for i := 0; i < b.N; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) + } + + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) == b.N-1 { + break + } + } +} + +func BenchmarkLittleBatch(b *testing.B) { + var components []interface{} + var workers []int + + f := NewRBForwarder(Config{ + Retries: 3, + Backoff: 5, + QueueSize: b.N / 100, + }) + + batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 10000, + }, + } + components = append(components, batch) + workers = append(workers, 1) + + sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + } + components = append(components, sender) + workers = append(workers, 1) + + f.PushComponents(components, workers) + f.Run() + + opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", + } + + for i := 0; i < b.N; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) + } + + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) == b.N-1 { + break + } + } +} + +func BenchmarkBigBatch(b *testing.B) { + var components []interface{} + var workers []int + + f := NewRBForwarder(Config{ + Retries: 3, + Backoff: 5, + QueueSize: b.N / 10, + }) + + batch := &batcher.Batcher{ + Config: batcher.Config{ + TimeoutMillis: 1000, + Limit: 10000, + }, + } + components = append(components, batch) + workers = append(workers, 1) + + sender := &httpsender.HTTPSender{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + } + components = append(components, sender) + workers = append(workers, 1) + + f.PushComponents(components, workers) + f.Run() + + opts := map[string]interface{}{ + "http_endpoint": "librb-http", + "batch_group": "librb-http", + } + + for i := 0; i < b.N; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) + } + + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) == b.N-1 { + break + } + } +} diff --git a/integration_test.go b/integration_test.go new file mode 100644 index 0000000..654987b --- /dev/null +++ b/integration_test.go @@ -0,0 +1,318 @@ +// +build integration + +package rbforwarder + +import ( + "testing" + + "github.com/redBorder/rbforwarder/utils" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/mock" +) + +type MockMiddleComponent struct { + mock.Mock +} + +func (c *MockMiddleComponent) Spawn(id int) utils.Composer { + args := c.Called() + return args.Get(0).(utils.Composer) +} + +func (c *MockMiddleComponent) OnMessage(m *utils.Message, done utils.Done) { + c.Called(m) + if data, err := m.PopPayload(); err == nil { + processedData := "-> [" + string(data) + "] <-" + m.PushPayload([]byte(processedData)) + } + + done(m, 0, "") +} + +type MockComponent struct { + mock.Mock + + channel chan string + + status string + statusCode int +} + +func (c *MockComponent) Spawn(id int) utils.Composer { + args := c.Called() + return args.Get(0).(utils.Composer) +} + +func (c *MockComponent) OnMessage(m *utils.Message, done utils.Done) { + c.Called(m) + if data, err := m.PopPayload(); err == nil { + c.channel <- string(data) + } else { + c.channel <- err.Error() + } + + done(m, c.statusCode, c.status) +} + +func TestRBForwarder(t *testing.T) { + Convey("Given a single component working pipeline", t, func() { + numMessages := 1000 + numWorkers := 100 + numRetries := 3 + + component := &MockComponent{ + channel: make(chan string, 10000), + } + + rbforwarder := NewRBForwarder(Config{ + Retries: numRetries, + QueueSize: numMessages, + }) + + component.On("Spawn").Return(component).Times(numWorkers) + + var components []interface{} + var instances []int + components = append(components, component) + instances = append(instances, numWorkers) + + rbforwarder.PushComponents(components, instances) + rbforwarder.Run() + + //////////////////////////////////////////////////////////////////////////// + + Convey("When a \"Hello World\" message is produced", func() { + component.status = "OK" + component.statusCode = 0 + + component.On("OnMessage", mock.AnythingOfType("*utils.Message")) + + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + nil, + ) + + Convey("Then \"Hello World\" message should be get by the worker", func() { + var lastReport Report + var reports int + for r := range rbforwarder.GetReports() { + reports++ + lastReport = r.(Report) + rbforwarder.Close() + } + + So(lastReport, ShouldNotBeNil) + So(reports, ShouldEqual, 1) + So(lastReport.Code, ShouldEqual, 0) + So(lastReport.Status, ShouldEqual, "OK") + So(err, ShouldBeNil) + + component.AssertExpectations(t) + }) + }) + + //////////////////////////////////////////////////////////////////////////// + + Convey("When a message is produced after close forwarder", func() { + rbforwarder.Close() + + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + nil, + ) + + Convey("Should error", func() { + So(err.Error(), ShouldEqual, "Forwarder has been closed") + }) + }) + + //////////////////////////////////////////////////////////////////////////// + + Convey("When calling OnMessage() with opaque", func() { + component.On("OnMessage", mock.AnythingOfType("*utils.Message")) + + err := rbforwarder.Produce( + []byte("Hello World"), + nil, + "This is an opaque", + ) + + Convey("Should be possible to read the opaque", func() { + So(err, ShouldBeNil) + + var reports int + var lastReport Report + for r := range rbforwarder.GetReports() { + reports++ + lastReport = r.(Report) + rbforwarder.Close() + } + + opaque := lastReport.Opaque.(string) + So(opaque, ShouldEqual, "This is an opaque") + }) + }) + + //////////////////////////////////////////////////////////////////////////// + + Convey("When a message fails to send", func() { + component.status = "Fake Error" + component.statusCode = 99 + + component.On("OnMessage", mock.AnythingOfType("*utils.Message")).Times(4) + + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + nil, + ) + + Convey("The message should be retried", func() { + So(err, ShouldBeNil) + + var reports int + var lastReport Report + for r := range rbforwarder.GetReports() { + reports++ + lastReport = r.(Report) + rbforwarder.Close() + } + + So(lastReport, ShouldNotBeNil) + So(reports, ShouldEqual, 1) + So(lastReport.Status, ShouldEqual, "Fake Error") + So(lastReport.Code, ShouldEqual, 99) + So(lastReport.retries, ShouldEqual, numRetries) + + component.AssertExpectations(t) + }) + }) + + //////////////////////////////////////////////////////////////////////////// + + Convey("When multiple messages are produced", func() { + var numErr int + + component.On("OnMessage", mock.AnythingOfType("*utils.Message")). + Return(nil). + Times(numMessages) + + for i := 0; i < numMessages; i++ { + if err := rbforwarder.Produce([]byte("Hello World"), + nil, + i, + ); err != nil { + numErr++ + } + } + + Convey("Then reports should be received", func() { + var reports int + for range rbforwarder.GetReports() { + reports++ + if reports >= numMessages { + rbforwarder.Close() + } + } + + So(numErr, ShouldBeZeroValue) + So(reports, ShouldEqual, numMessages) + + component.AssertExpectations(t) + }) + + Convey("Then reports should be received in order", func() { + ordered := true + var reports int + + for rep := range rbforwarder.GetOrderedReports() { + if rep.(Report).Opaque.(int) != reports { + ordered = false + } + reports++ + if reports >= numMessages { + rbforwarder.Close() + } + } + + So(numErr, ShouldBeZeroValue) + So(ordered, ShouldBeTrue) + So(reports, ShouldEqual, numMessages) + + component.AssertExpectations(t) + }) + }) + }) + + Convey("Given a multi-component working pipeline", t, func() { + numMessages := 100 + numWorkers := 3 + numRetries := 3 + + component1 := &MockMiddleComponent{} + component2 := &MockComponent{ + channel: make(chan string, 10000), + } + + rbforwarder := NewRBForwarder(Config{ + Retries: numRetries, + QueueSize: numMessages, + }) + + for i := 0; i < numWorkers; i++ { + component1.On("Spawn").Return(component1) + component2.On("Spawn").Return(component2) + } + + var components []interface{} + var instances []int + + components = append(components, component1) + components = append(components, component2) + + instances = append(instances, numWorkers) + instances = append(instances, numWorkers) + + rbforwarder.PushComponents(components, instances) + rbforwarder.Run() + + Convey("When a \"Hello World\" message is produced", func() { + component2.status = "OK" + component2.statusCode = 0 + + component1.On("OnMessage", mock.AnythingOfType("*utils.Message")) + component2.On("OnMessage", mock.AnythingOfType("*utils.Message")) + + err := rbforwarder.Produce( + []byte("Hello World"), + map[string]interface{}{"message_id": "test123"}, + nil, + ) + + rbforwarder.Close() + + Convey("\"Hello World\" message should be processed by the pipeline", func() { + reports := 0 + for rep := range rbforwarder.GetReports() { + reports++ + + code := rep.(Report).Code + status := rep.(Report).Status + So(code, ShouldEqual, 0) + So(status, ShouldEqual, "OK") + } + + m := <-component2.channel + + So(err, ShouldBeNil) + So(reports, ShouldEqual, 1) + So(m, ShouldEqual, "-> [Hello World] <-") + + component1.AssertExpectations(t) + component2.AssertExpectations(t) + }) + }) + }) +} diff --git a/pipeline_test.go b/pipeline_test.go new file mode 100644 index 0000000..92407a5 --- /dev/null +++ b/pipeline_test.go @@ -0,0 +1 @@ +package rbforwarder diff --git a/reportHandler_test.go b/reportHandler_test.go new file mode 100644 index 0000000..92407a5 --- /dev/null +++ b/reportHandler_test.go @@ -0,0 +1 @@ +package rbforwarder From d863af94715a5f5fe301ca722f0c287db00ca151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 10:14:36 +0200 Subject: [PATCH 27/36] :bug: Fix example --- examples/http_send.go | 58 +++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/examples/http_send.go b/examples/http_send.go index e36a5e4..a699f41 100644 --- a/examples/http_send.go +++ b/examples/http_send.go @@ -1,7 +1,10 @@ +// +build examples + package main import ( "fmt" + "sync" "github.com/redBorder/rbforwarder" "github.com/redBorder/rbforwarder/components/batch" @@ -9,30 +12,31 @@ import ( ) func main() { + var wg sync.WaitGroup var components []interface{} var workers []int const numMessages = 100000 f := rbforwarder.NewRBForwarder(rbforwarder.Config{ - Retries: 3, - Backoff: 5, - QueueSize: 10000, + Retries: 1, + Backoff: 1, + QueueSize: 1000, }) batch := &batcher.Batcher{ Config: batcher.Config{ - TimeoutMillis: 1000, - Limit: 1000, + TimeoutMillis: 100, + Limit: 10000, }, } components = append(components, batch) - workers = append(workers, 1) + workers = append(workers, 5) sender := &httpsender.HTTPSender{ URL: "http://localhost:8888", } components = append(components, sender) - workers = append(workers, 1) + workers = append(workers, 10) f.PushComponents(components, workers) @@ -41,21 +45,37 @@ func main() { "batch_group": "librb-http", } + f.Run() + + wg.Add(1) + go func() { + var errors int + var messages int + + fmt.Print("[") + for report := range f.GetReports() { + r := report.(rbforwarder.Report) + // fmt.Printf("[MESSAGE %d] %s\n", r.Opaque.(int), r.Status) + messages++ + if messages%(numMessages/20) == 0 { + fmt.Printf("=") + } + if r.Code > 0 { + errors += r.Code + } + if messages >= numMessages { + break + } + } + fmt.Print("] ") + fmt.Printf("Sent %d messages with %d errors\n", messages, errors) + wg.Done() + }() + for i := 0; i < numMessages; i++ { data := fmt.Sprintf("{\"message\": %d}", i) f.Produce([]byte(data), opts, i) } - var errors int - for report := range f.GetReports() { - r := report.(rbforwarder.Report) - if r.Code > 0 { - errors += r.Code - } - if r.Opaque.(int) == numMessages-1 { - break - } - } - - fmt.Printf("Sent %d messages with %d errors\n", numMessages, errors) + wg.Wait() } From 9b554bf33b4503d596a5b4bfe6992078d354b34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 10:47:24 +0200 Subject: [PATCH 28/36] :green_heart: Fix coverage --- .travis.yml | 8 +++----- Makefile | 14 ++++++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index a538953..4488f5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,13 +17,11 @@ script: install: - go get github.com/mattn/goveralls - - go get github.com/go-playground/overalls - go get -t ./... script: - - go test -v -race -cover ./... + - make test after_success: - - overalls -covermode=set -project=github.com/redBorder/rbforwarder - - go tool cover -func overalls.coverprofile - - goveralls -coverprofile=overalls.coverprofile -service=travis-ci + - make coverage + - goveralls -coverprofile=coverage-out -service=travis-ci diff --git a/Makefile b/Makefile index 27de9bc..51d302f 100644 --- a/Makefile +++ b/Makefile @@ -25,14 +25,20 @@ vet: test: @printf "$(MKL_YELLOW)Running tests$(MKL_CLR_RESET)\n" - go test -race ./... -tags=integration + go test -v -race -tags=integration ./... @printf "$(MKL_GREEN)Test passed$(MKL_CLR_RESET)\n" coverage: @printf "$(MKL_YELLOW)Computing coverage$(MKL_CLR_RESET)\n" - @overalls -covermode=set -project=github.com/redBorder/rbforwarder - @go tool cover -func overalls.coverprofile - @rm -f overalls.coverprofile + @go test -covermode=count -coverprofile=rbforwarder.part -tags=integration + @go test -covermode=count -coverprofile=batch.part -tags=integration ./components/batch + @go test -covermode=count -coverprofile=httpsender.part -tags=integration ./components/httpsender + @go test -covermode=count -coverprofile=limiter.part -tags=integration ./components/limiter + @go test -covermode=count -coverprofile=utils.part -tags=integration ./utils + @echo "mode: count" > coverage.out + @grep -h -v "mode: count" *.part >> coverage.out + @go tool cover -func coverage.out + @rm -f *.part coverage.out get: @printf "$(MKL_YELLOW)Installing deps$(MKL_CLR_RESET)\n" From 3426653a0890187e2a50eb5cb36d209fefc9235e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 10:51:31 +0200 Subject: [PATCH 29/36] :green_heart: Fix travis-ci wrong filename --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4488f5b..c1fe62f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,4 +24,4 @@ script: after_success: - make coverage - - goveralls -coverprofile=coverage-out -service=travis-ci + - goveralls -coverprofile=coverage.out -service=travis-ci From cea46422cdd4df77a1ef82b636ff3f2c4b82397d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 11:14:19 +0200 Subject: [PATCH 30/36] :green_heart: Fix makefile clearing files used on coverage --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 51d302f..eeb7432 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,6 @@ coverage: @echo "mode: count" > coverage.out @grep -h -v "mode: count" *.part >> coverage.out @go tool cover -func coverage.out - @rm -f *.part coverage.out get: @printf "$(MKL_YELLOW)Installing deps$(MKL_CLR_RESET)\n" From 63a849d89cd43b8b46d3f1d685324c6718312069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 12:14:45 +0200 Subject: [PATCH 31/36] :racehorse: Improve benchmarks --- benchmarks_test.go | 224 +++++++++++++++++++++++++++++++-------------- 1 file changed, 157 insertions(+), 67 deletions(-) diff --git a/benchmarks_test.go b/benchmarks_test.go index af4e91f..fc5a244 100644 --- a/benchmarks_test.go +++ b/benchmarks_test.go @@ -5,92 +5,155 @@ import ( "net/http" "net/http/httptest" "net/url" + "sync" "testing" "github.com/redBorder/rbforwarder/components/batch" "github.com/redBorder/rbforwarder/components/httpsender" + "github.com/redBorder/rbforwarder/utils" ) -func NewTestClient(code int, cb func(*http.Request)) *http.Client { - server := httptest.NewServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(code) - cb(r) - })) +type Null struct{} - transport := &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, +func (null *Null) Spawn(id int) utils.Composer { + n := *null + return &n +} + +func (null *Null) OnMessage(m *utils.Message, done utils.Done) { + done(m, 0, "") +} + +func BenchmarkQueue1(b *testing.B) { benchmarkQueue(1, b) } +func BenchmarkQueue10(b *testing.B) { benchmarkQueue(10, b) } +func BenchmarkQueue1000(b *testing.B) { benchmarkQueue(100, b) } +func BenchmarkQueue10000(b *testing.B) { benchmarkQueue(1000, b) } +func BenchmarkQueue100000(b *testing.B) { benchmarkQueue(10000, b) } + +func BenchmarkHTTP(b *testing.B) { benchmarkHTTP(b) } + +func BenchmarkBatch1(b *testing.B) { benchmarkBatch(1, b) } +func BenchmarkBatch10(b *testing.B) { benchmarkBatch(10, b) } +func BenchmarkBatch1000(b *testing.B) { benchmarkBatch(100, b) } +func BenchmarkBatch10000(b *testing.B) { benchmarkBatch(1000, b) } +func BenchmarkBatch100000(b *testing.B) { benchmarkBatch(10000, b) } + +func BenchmarkHTTPBatch1(b *testing.B) { benchmarkHTTPBatch(1, b) } +func BenchmarkHTTPBatch10(b *testing.B) { benchmarkHTTPBatch(10, b) } +func BenchmarkHTTPBatch1000(b *testing.B) { benchmarkHTTPBatch(100, b) } +func BenchmarkHTTPBatch10000(b *testing.B) { benchmarkHTTPBatch(1000, b) } +func BenchmarkHTTPBatch100000(b *testing.B) { benchmarkHTTPBatch(10000, b) } + +func benchmarkQueue(queue int, b *testing.B) { + var wg sync.WaitGroup + var components []interface{} + var workers []int + + f := NewRBForwarder(Config{ + Retries: 3, + Backoff: 5, + QueueSize: queue, + }) + + null := &Null{} + components = append(components, null) + workers = append(workers, 1) + + f.PushComponents(components, workers) + f.Run() + + opts := map[string]interface{}{} + + wg.Add(1) + b.ResetTimer() + go func() { + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) >= b.N-1 { + break + } + } + + wg.Done() + }() + + for i := 0; i < b.N; i++ { + data := fmt.Sprintf("{\"message\": %d}", i) + f.Produce([]byte(data), opts, i) } - return &http.Client{Transport: transport} + wg.Wait() } -func BenchmarkNoBatch(b *testing.B) { +func benchmarkBatch(batchSize int, b *testing.B) { + var wg sync.WaitGroup var components []interface{} var workers []int f := NewRBForwarder(Config{ Retries: 3, Backoff: 5, - QueueSize: 1, + QueueSize: 10000, }) batch := &batcher.Batcher{ Config: batcher.Config{ TimeoutMillis: 1000, - Limit: 10000, + Limit: uint64(batchSize), }, } components = append(components, batch) workers = append(workers, 1) - sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), - } - components = append(components, sender) - workers = append(workers, 1) - f.PushComponents(components, workers) f.Run() opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", + "batch_group": "test", } + wg.Add(1) + b.ResetTimer() + go func() { + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) >= b.N-1 { + break + } + } + + wg.Done() + }() + for i := 0; i < b.N; i++ { data := fmt.Sprintf("{\"message\": %d}", i) f.Produce([]byte(data), opts, i) } - for report := range f.GetReports() { - r := report.(Report) - if r.Code > 0 { - b.FailNow() - } - if r.Opaque.(int) == b.N-1 { - break - } - } + wg.Wait() } -func BenchmarkLittleBatch(b *testing.B) { +func benchmarkHTTPBatch(batchSize int, b *testing.B) { + var wg sync.WaitGroup var components []interface{} var workers []int f := NewRBForwarder(Config{ Retries: 3, Backoff: 5, - QueueSize: b.N / 100, + QueueSize: 10000, }) batch := &batcher.Batcher{ Config: batcher.Config{ TimeoutMillis: 1000, - Limit: 10000, + Limit: uint64(batchSize), }, } components = append(components, batch) @@ -107,45 +170,45 @@ func BenchmarkLittleBatch(b *testing.B) { f.Run() opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", + "http_endpoint": "test", + "batch_group": "test", } + wg.Add(1) + b.ResetTimer() + go func() { + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) >= b.N-1 { + break + } + } + + wg.Done() + }() + for i := 0; i < b.N; i++ { data := fmt.Sprintf("{\"message\": %d}", i) f.Produce([]byte(data), opts, i) } - for report := range f.GetReports() { - r := report.(Report) - if r.Code > 0 { - b.FailNow() - } - if r.Opaque.(int) == b.N-1 { - break - } - } + wg.Wait() } -func BenchmarkBigBatch(b *testing.B) { +func benchmarkHTTP(b *testing.B) { + var wg sync.WaitGroup var components []interface{} var workers []int f := NewRBForwarder(Config{ Retries: 3, Backoff: 5, - QueueSize: b.N / 10, + QueueSize: 10000, }) - batch := &batcher.Batcher{ - Config: batcher.Config{ - TimeoutMillis: 1000, - Limit: 10000, - }, - } - components = append(components, batch) - workers = append(workers, 1) - sender := &httpsender.HTTPSender{ URL: "http://localhost:8888", Client: NewTestClient(200, func(r *http.Request) {}), @@ -157,22 +220,49 @@ func BenchmarkBigBatch(b *testing.B) { f.Run() opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", + "http_endpoint": "test", } + wg.Add(1) + b.ResetTimer() + go func() { + for report := range f.GetReports() { + r := report.(Report) + if r.Code > 0 { + b.FailNow() + } + if r.Opaque.(int) >= b.N-1 { + break + } + } + + wg.Done() + }() + for i := 0; i < b.N; i++ { data := fmt.Sprintf("{\"message\": %d}", i) f.Produce([]byte(data), opts, i) } - for report := range f.GetReports() { - r := report.(Report) - if r.Code > 0 { - b.FailNow() - } - if r.Opaque.(int) == b.N-1 { - break - } + wg.Wait() +} + +//////////////////////////////////////////////////////////////////////////////// +/// Aux functions +//////////////////////////////////////////////////////////////////////////////// + +func NewTestClient(code int, cb func(*http.Request)) *http.Client { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + cb(r) + })) + + transport := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, } + + return &http.Client{Transport: transport} } From c96a57c18bbb8a4594d74b01aa117b2a69df606b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 12:56:32 +0200 Subject: [PATCH 32/36] :memo: Update README.md --- README.md | 89 ++++++++++--------------------------------------------- 1 file changed, 15 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 406aafe..f13fdf2 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,24 @@ ![](https://img.shields.io/packagist/l/doctrine/orm.svg?maxAge=2592000) -[![](https://travis-ci.org/redBorder/rbforwarder.svg?branch=develop)](https://travis-ci.org/redBorder/rbforwarder) +[![](https://travis-ci.org/redBorder/rbforwarder.svg?branch=master)](https://travis-ci.org/redBorder/rbforwarder) [![](https://goreportcard.com/badge/github.com/redBorder/rbforwarder)](https://goreportcard.com/report/github.com/redBorder/rbforwarder) -[![](https://coveralls.io/repos/github/redBorder/rbforwarder/badge.svg?branch=develop)](https://coveralls.io/github/redBorder/rbforwarder?branch=develop) +[![](https://coveralls.io/repos/github/redBorder/rbforwarder/badge.svg?branch=master)](https://coveralls.io/github/redBorder/rbforwarder?branch=develop) [![](https://godoc.org/github.com/redBorder/rbforwarder?status.svg)](https://godoc.org/github.com/redBorder/rbforwarder) # rbforwarder -**rbforwarder** is an extensible, protocol agnostic and easy to use tool for -processing data asynchronously. It allows you to create a custom pipeline in -a modular fashion. +**rbforwarder** is an extensible tool for processing data asynchronously. +It allows you to create a custom pipeline in a modular fashion. -For example, you can read data from a Kafka broker and use **rbforwarder** to -build a **pipeline** to decodes the JSON, filter or add fields, encode -the data again to JSON and send it using using multiple protocols `HTTP`, -`MQTT`, `AMQP`, etc. It's easy to write a pipeline **component**. +You can use **rbforwarder** to build a **pipeline** that decodes a JSON, filter +or add fields, encode the data again to JSON and send it using multiple +protocols `HTTP`, `MQTT`, `AMQP`, etc. It's easy to write a pipeline **component**. ## Built-in features - Support multiple **workers** for each components. -- Support **buffer pooling** for memory recycling. -- Asynchronous report system. Get responses on a separate gorutine. -- Built-in message retrying. The **rbforwarder** can retry messages on fail. -- Instrumentation to have an idea of how it is performing the application. +- Asynchronous report system. Get responses on a separate goroutine. +- Built-in message retrying. **rbforwarder** can retry messages on fail. +- Instrumentation, to have an overview of how the application is performing. ## Components @@ -33,6 +30,7 @@ the data again to JSON and send it using using multiple protocols `HTTP`, - JSON - Utility - Limiter + - Batcher (supports deflate) ## Road Map @@ -40,69 +38,12 @@ _The application is on hard development, breaking changes expected until 1.0._ |Milestone | Feature | Status | |----------|---------------------|-----------| -| 0.1 | Pipeline builder | Done | +| 0.1 | Pipeline | Done | | 0.2 | Reporter | Done | -| 0.3 | Buffer pool | _Pending_ | -| 0.4 | Batcher component | _Pending_ | -| 0.5 | Limiter component | _Pending_ | +| 0.3 | HTTP component | Done | +| 0.4 | Batcher component | Done | +| 0.5 | Limiter component | Done | | 0.6 | Instrumentation | _Pending_ | -| 0.7 | HTTP component | _Pending_ | | 0.8 | JSON component | _Pending_ | | 0.9 | MQTT component | _Pending_ | | 1.0 | Kafka component | _Pending_ | - -## Usage - -```go - // Array of components to use and workers -var components []interface{} -var workers []int - -// Create an instance of rbforwarder -f := rbforwarder.NewRBForwarder(rbforwarder.Config{ - Retries: 3, // Number of retries before give up - Backoff: 5, // Time to wait before retry a message - QueueSize: 10000, // Max messageon queue, produce will block when the queue is full -}) - -// Create a batcher component and add it to the components array -batch := &batcher.Batcher{ - Config: batcher.Config{ - TimeoutMillis: 1000, - Limit: 1000, - }, -} -components = append(components, batch) -workers = append(workers, 1) - -// Create a http component and add it to the components array -sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", -} -components = append(components, sender) -workers = append(workers, 1) - -// Push the component array and workers to the pipeline -f.PushComponents(components, workers) - -opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", -} - -// Produce messages. It won't block until the queue is full. -f.Produce([]byte("{\"message\": 1}"), opts, 1) -f.Produce([]byte("{\"message\": 2}"), opts, 2) -f.Produce([]byte("{\"message\": 3}"), opts, 3) -f.Produce([]byte("{\"message\": 4}"), opts, 4) -f.Produce([]byte("{\"message\": 5}"), opts, 5) - -// Read reports. You should do this on a separate gorutine so you make sure -// that you won't block -for report := range f.GetReports() { - r := report.(rbforwarder.Report) - if r.Opaque.(int) == numMessages-1 { - break - } -} -``` From 58350962242bfa5107946386b97f09c5c47dddd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Tue, 23 Aug 2016 12:57:27 +0200 Subject: [PATCH 33/36] :arrow_up: Bump version --- rbforwarder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbforwarder.go b/rbforwarder.go index f25610b..9a6fa47 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -9,7 +9,7 @@ import ( ) // Version is the current tag -var Version = "0.4-beta3" +var Version = "0.5" var log = logrus.New() From e324bdcef2bbaf47a6be9d62e88a04671f824744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 24 Aug 2016 13:17:06 +0200 Subject: [PATCH 34/36] :lipstick: Cleaner way to set the number of workers --- benchmarks_test.go | 35 ++++++++++++------------ components/batch/batcher.go | 7 ++++- components/batch/config.go | 1 + components/httpsender/config.go | 10 +++++++ components/httpsender/httpSender.go | 13 ++++++--- components/httpsender/httpSender_test.go | 5 +++- components/limiter/limiter.go | 5 ++++ examples/http_send.go | 17 ++++++------ integration_test.go | 23 ++++++++++------ pipeline.go | 11 +++----- rbforwarder.go | 6 ++-- utils/composer.go | 1 + 12 files changed, 85 insertions(+), 49 deletions(-) create mode 100644 components/httpsender/config.go diff --git a/benchmarks_test.go b/benchmarks_test.go index fc5a244..da0dcfc 100644 --- a/benchmarks_test.go +++ b/benchmarks_test.go @@ -15,6 +15,11 @@ import ( type Null struct{} +// Workers returns the number of workers +func (null *Null) Workers() int { + return 1 +} + func (null *Null) Spawn(id int) utils.Composer { n := *null return &n @@ -47,7 +52,6 @@ func BenchmarkHTTPBatch100000(b *testing.B) { benchmarkHTTPBatch(10000, b) } func benchmarkQueue(queue int, b *testing.B) { var wg sync.WaitGroup var components []interface{} - var workers []int f := NewRBForwarder(Config{ Retries: 3, @@ -57,9 +61,8 @@ func benchmarkQueue(queue int, b *testing.B) { null := &Null{} components = append(components, null) - workers = append(workers, 1) - f.PushComponents(components, workers) + f.PushComponents(components) f.Run() opts := map[string]interface{}{} @@ -91,7 +94,6 @@ func benchmarkQueue(queue int, b *testing.B) { func benchmarkBatch(batchSize int, b *testing.B) { var wg sync.WaitGroup var components []interface{} - var workers []int f := NewRBForwarder(Config{ Retries: 3, @@ -106,9 +108,8 @@ func benchmarkBatch(batchSize int, b *testing.B) { }, } components = append(components, batch) - workers = append(workers, 1) - f.PushComponents(components, workers) + f.PushComponents(components) f.Run() opts := map[string]interface{}{ @@ -135,6 +136,7 @@ func benchmarkBatch(batchSize int, b *testing.B) { data := fmt.Sprintf("{\"message\": %d}", i) f.Produce([]byte(data), opts, i) } + b.StopTimer() wg.Wait() } @@ -142,7 +144,6 @@ func benchmarkBatch(batchSize int, b *testing.B) { func benchmarkHTTPBatch(batchSize int, b *testing.B) { var wg sync.WaitGroup var components []interface{} - var workers []int f := NewRBForwarder(Config{ Retries: 3, @@ -157,16 +158,16 @@ func benchmarkHTTPBatch(batchSize int, b *testing.B) { }, } components = append(components, batch) - workers = append(workers, 1) sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), + Config: httpsender.Config{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + }, } components = append(components, sender) - workers = append(workers, 1) - f.PushComponents(components, workers) + f.PushComponents(components) f.Run() opts := map[string]interface{}{ @@ -201,7 +202,6 @@ func benchmarkHTTPBatch(batchSize int, b *testing.B) { func benchmarkHTTP(b *testing.B) { var wg sync.WaitGroup var components []interface{} - var workers []int f := NewRBForwarder(Config{ Retries: 3, @@ -210,13 +210,14 @@ func benchmarkHTTP(b *testing.B) { }) sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), + Config: httpsender.Config{ + URL: "http://localhost:8888", + Client: NewTestClient(200, func(r *http.Request) {}), + }, } components = append(components, sender) - workers = append(workers, 1) - f.PushComponents(components, workers) + f.PushComponents(components) f.Run() opts := map[string]interface{}{ diff --git a/components/batch/batcher.go b/components/batch/batcher.go index ebe98c4..c40d4b9 100644 --- a/components/batch/batcher.go +++ b/components/batch/batcher.go @@ -20,7 +20,12 @@ type Batcher struct { done utils.Done } - Config Config // Batcher configuration + Config +} + +// Workers returns the number of workers +func (batcher *Batcher) Workers() int { + return batcher.Config.Workers } // Spawn starts a gorutine that can receive: diff --git a/components/batch/config.go b/components/batch/config.go index 3788c53..b3cecd7 100644 --- a/components/batch/config.go +++ b/components/batch/config.go @@ -2,6 +2,7 @@ package batcher // Config stores the config for a Batcher type Config struct { + Workers int Deflate bool TimeoutMillis uint Limit uint64 diff --git a/components/httpsender/config.go b/components/httpsender/config.go new file mode 100644 index 0000000..dc4158e --- /dev/null +++ b/components/httpsender/config.go @@ -0,0 +1,10 @@ +package httpsender + +import "net/http" + +// Config exposes the configuration for an HTTP Sender +type Config struct { + Workers int + URL string + Client *http.Client +} diff --git a/components/httpsender/httpSender.go b/components/httpsender/httpSender.go index 6ea0f67..8357f50 100644 --- a/components/httpsender/httpSender.go +++ b/components/httpsender/httpSender.go @@ -15,10 +15,15 @@ import ( // to an HTTP endpoint. It's a final component, so it will call Done() instead // of Next() and further components shuld not be added after this component. type HTTPSender struct { - id int - err error - URL string - Client *http.Client + id int + err error + + Config +} + +// Workers returns the number of workers +func (httpsender *HTTPSender) Workers() int { + return httpsender.Config.Workers } // Spawn initializes the HTTP component diff --git a/components/httpsender/httpSender_test.go b/components/httpsender/httpSender_test.go index fbe84cf..7f16fad 100644 --- a/components/httpsender/httpSender_test.go +++ b/components/httpsender/httpSender_test.go @@ -49,7 +49,10 @@ func NewTestClient(code int, cb func(*http.Request)) *http.Client { func TestHTTPSender(t *testing.T) { Convey("Given an HTTP sender with defined URL", t, func() { sender := &HTTPSender{ - URL: "http://example.com", + Config: Config{ + URL: "http://example.com", + Workers: 1, + }, } Convey("When is initialized", func() { diff --git a/components/limiter/limiter.go b/components/limiter/limiter.go index 664519c..5c3ce8c 100644 --- a/components/limiter/limiter.go +++ b/components/limiter/limiter.go @@ -19,6 +19,11 @@ type Limiter struct { clk clock.Clock } +// Workers returns 1 because it should be only one instance of this component +func (l *Limiter) Workers() int { + return 1 +} + // Init initializes the limiter func (l *Limiter) Init(id int) { l.id = id diff --git a/examples/http_send.go b/examples/http_send.go index a699f41..c668e69 100644 --- a/examples/http_send.go +++ b/examples/http_send.go @@ -1,4 +1,4 @@ -// +build examples +// +nobuild examples package main @@ -14,7 +14,6 @@ import ( func main() { var wg sync.WaitGroup var components []interface{} - var workers []int const numMessages = 100000 f := rbforwarder.NewRBForwarder(rbforwarder.Config{ @@ -25,24 +24,26 @@ func main() { batch := &batcher.Batcher{ Config: batcher.Config{ + Workers: 2, TimeoutMillis: 100, Limit: 10000, }, } components = append(components, batch) - workers = append(workers, 5) sender := &httpsender.HTTPSender{ - URL: "http://localhost:8888", + Config: httpsender.Config{ + Workers: 10, + URL: "http://localhost:8888", + }, } components = append(components, sender) - workers = append(workers, 10) - f.PushComponents(components, workers) + f.PushComponents(components) opts := map[string]interface{}{ - "http_endpoint": "librb-http", - "batch_group": "librb-http", + "http_endpoint": "test", + "batch_group": "example", } f.Run() diff --git a/integration_test.go b/integration_test.go index 654987b..abd08b7 100644 --- a/integration_test.go +++ b/integration_test.go @@ -14,6 +14,11 @@ type MockMiddleComponent struct { mock.Mock } +func (c *MockMiddleComponent) Workers() int { + args := c.Called() + return args.Int(0) +} + func (c *MockMiddleComponent) Spawn(id int) utils.Composer { args := c.Called() return args.Get(0).(utils.Composer) @@ -38,6 +43,11 @@ type MockComponent struct { statusCode int } +func (c *MockComponent) Workers() int { + args := c.Called() + return args.Int(0) +} + func (c *MockComponent) Spawn(id int) utils.Composer { args := c.Called() return args.Get(0).(utils.Composer) @@ -69,14 +79,13 @@ func TestRBForwarder(t *testing.T) { QueueSize: numMessages, }) + component.On("Workers").Return(numWorkers) component.On("Spawn").Return(component).Times(numWorkers) var components []interface{} - var instances []int components = append(components, component) - instances = append(instances, numWorkers) - rbforwarder.PushComponents(components, instances) + rbforwarder.PushComponents(components) rbforwarder.Run() //////////////////////////////////////////////////////////////////////////// @@ -264,18 +273,16 @@ func TestRBForwarder(t *testing.T) { for i := 0; i < numWorkers; i++ { component1.On("Spawn").Return(component1) component2.On("Spawn").Return(component2) + component1.On("Workers").Return(numWorkers) + component2.On("Workers").Return(numWorkers) } var components []interface{} - var instances []int components = append(components, component1) components = append(components, component2) - instances = append(instances, numWorkers) - instances = append(instances, numWorkers) - - rbforwarder.PushComponents(components, instances) + rbforwarder.PushComponents(components) rbforwarder.Run() Convey("When a \"Hello World\" message is produced", func() { diff --git a/pipeline.go b/pipeline.go index bc84f68..f537db9 100644 --- a/pipeline.go +++ b/pipeline.go @@ -7,7 +7,6 @@ import ( // Component contains information about a pipeline component type Component struct { - workers int composser utils.Composer pool chan chan *utils.Message } @@ -30,21 +29,19 @@ func newPipeline(input, retry, output chan *utils.Message) *pipeline { } // PushComponent adds a new component to the pipeline -func (p *pipeline) PushComponent(composser utils.Composer, w int) { +func (p *pipeline) PushComponent(composser utils.Composer) { p.components = append(p.components, struct { - workers int composser utils.Composer pool chan chan *utils.Message }{ - workers: w, composser: composser, - pool: make(chan chan *utils.Message, w), + pool: make(chan chan *utils.Message, composser.Workers()), }) } func (p *pipeline) Run() { for index, component := range p.components { - for w := 0; w < component.workers; w++ { + for w := 0; w < component.composser.Workers(); w++ { go func(w, index int, component Component) { component.composser = component.composser.Spawn(w) messages := make(chan *utils.Message) @@ -89,7 +86,7 @@ func (p *pipeline) Run() { // If input channel has been closed, close output channel if !ok { for _, component := range p.components { - for i := 0; i < component.workers; i++ { + for i := 0; i < component.composser.Workers(); i++ { worker := <-component.pool close(worker) } diff --git a/rbforwarder.go b/rbforwarder.go index 9a6fa47..b794b83 100644 --- a/rbforwarder.go +++ b/rbforwarder.go @@ -67,9 +67,9 @@ func (f *RBForwarder) Close() { } // PushComponents adds a new component to the pipeline -func (f *RBForwarder) PushComponents(components []interface{}, w []int) { - for i, component := range components { - f.p.PushComponent(component.(utils.Composer), w[i]) +func (f *RBForwarder) PushComponents(components []interface{}) { + for _, component := range components { + f.p.PushComponent(component.(utils.Composer)) } } diff --git a/utils/composer.go b/utils/composer.go index 424e869..253941f 100644 --- a/utils/composer.go +++ b/utils/composer.go @@ -14,4 +14,5 @@ type Done func(*Message, int, string) type Composer interface { Spawn(int) Composer OnMessage(*Message, Done) + Workers() int } From 2a7334ee84f3cbaa578552aa859fa71606d03b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 24 Aug 2016 13:39:10 +0200 Subject: [PATCH 35/36] :sparkles: Allow set http headers and insecure connections Closes #35 --- benchmarks_test.go | 10 +++++---- components/httpsender/config.go | 8 +++----- components/httpsender/httpSender.go | 32 +++++++++++++++++++++++------ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/benchmarks_test.go b/benchmarks_test.go index da0dcfc..121d17c 100644 --- a/benchmarks_test.go +++ b/benchmarks_test.go @@ -161,13 +161,14 @@ func benchmarkHTTPBatch(batchSize int, b *testing.B) { sender := &httpsender.HTTPSender{ Config: httpsender.Config{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), + URL: "http://localhost:8888", }, } components = append(components, sender) f.PushComponents(components) + sender.Client = NewTestClient(200, func(r *http.Request) {}) + f.Run() opts := map[string]interface{}{ @@ -211,13 +212,14 @@ func benchmarkHTTP(b *testing.B) { sender := &httpsender.HTTPSender{ Config: httpsender.Config{ - URL: "http://localhost:8888", - Client: NewTestClient(200, func(r *http.Request) {}), + URL: "http://localhost:8888", }, } components = append(components, sender) f.PushComponents(components) + sender.Client = NewTestClient(200, func(r *http.Request) {}) + f.Run() opts := map[string]interface{}{ diff --git a/components/httpsender/config.go b/components/httpsender/config.go index dc4158e..534565e 100644 --- a/components/httpsender/config.go +++ b/components/httpsender/config.go @@ -1,10 +1,8 @@ package httpsender -import "net/http" - // Config exposes the configuration for an HTTP Sender type Config struct { - Workers int - URL string - Client *http.Client + Workers int + URL string + Insecure bool } diff --git a/components/httpsender/httpSender.go b/components/httpsender/httpSender.go index 8357f50..faf172f 100644 --- a/components/httpsender/httpSender.go +++ b/components/httpsender/httpSender.go @@ -2,6 +2,7 @@ package httpsender import ( "bytes" + "crypto/tls" "errors" "io" "io/ioutil" @@ -15,8 +16,9 @@ import ( // to an HTTP endpoint. It's a final component, so it will call Done() instead // of Next() and further components shuld not be added after this component. type HTTPSender struct { - id int - err error + id int + err error + Client *http.Client Config } @@ -33,19 +35,24 @@ func (httpsender *HTTPSender) Spawn(id int) utils.Composer { s.id = id if govalidator.IsURL(s.URL) { - if s.Client == nil { - s.Client = &http.Client{} - } + s.Client = new(http.Client) } else { s.err = errors.New("Invalid URL") } + if httpsender.Config.Insecure { + s.Client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + return &s } // OnMessage is called when a new message should be sent via HTTP func (httpsender *HTTPSender) OnMessage(m *utils.Message, done utils.Done) { var u string + var headers map[string]string if httpsender.err != nil { done(m, 2, httpsender.err.Error()) @@ -65,7 +72,20 @@ func (httpsender *HTTPSender) OnMessage(m *utils.Message, done utils.Done) { } buf := bytes.NewBuffer(data) - res, err := httpsender.Client.Post(u, "", buf) + req, err := http.NewRequest("POST", u, buf) + if err != nil { + done(m, 1, "HTTPSender error: "+err.Error()) + return + } + + if h, exists := m.Opts.Get("http_headers"); exists { + headers = h.(map[string]string) + for k, v := range headers { + req.Header.Add(k, v) + } + } + + res, err := httpsender.Client.Do(req) if err != nil { done(m, 1, "HTTPSender error: "+err.Error()) return From 6e6595df34c865d2cb5e16046e4e7f64536e557f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Barrera?= Date: Wed, 24 Aug 2016 14:25:17 +0200 Subject: [PATCH 36/36] :white_check_mark: Increase coverage --- components/batch/batcher_test.go | 9 ++++ components/httpsender/httpSender.go | 18 ++++---- components/httpsender/httpSender_test.go | 52 +++++++++++++++++++++--- components/limiter/limiter_test.go | 8 ++++ 4 files changed, 71 insertions(+), 16 deletions(-) diff --git a/components/batch/batcher_test.go b/components/batch/batcher_test.go index 656a8b8..56ec11a 100644 --- a/components/batch/batcher_test.go +++ b/components/batch/batcher_test.go @@ -28,6 +28,7 @@ func TestBatcher(t *testing.T) { Convey("Given a batcher", t, func() { batcher := &Batcher{ Config: Config{ + Workers: 1, TimeoutMillis: 1000, Limit: 10, MaxPendingBatches: 10, @@ -38,6 +39,14 @@ func TestBatcher(t *testing.T) { b := batcher.Spawn(0).(*Batcher) b.clk = clock.NewMock() + Convey("When the number of workers is requested", func() { + workers := batcher.Workers() + + Convey("Then the number of workers should be correct", func() { + So(workers, ShouldEqual, 1) + }) + }) + Convey("When a message is received with no batch group", func() { m := utils.NewMessage() m.PushPayload([]byte("Hello World")) diff --git a/components/httpsender/httpSender.go b/components/httpsender/httpSender.go index faf172f..a704a6e 100644 --- a/components/httpsender/httpSender.go +++ b/components/httpsender/httpSender.go @@ -36,14 +36,14 @@ func (httpsender *HTTPSender) Spawn(id int) utils.Composer { if govalidator.IsURL(s.URL) { s.Client = new(http.Client) - } else { - s.err = errors.New("Invalid URL") - } - if httpsender.Config.Insecure { - s.Client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + if httpsender.Config.Insecure { + s.Client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } } + } else { + s.err = errors.New("Invalid URL") } return &s @@ -72,11 +72,7 @@ func (httpsender *HTTPSender) OnMessage(m *utils.Message, done utils.Done) { } buf := bytes.NewBuffer(data) - req, err := http.NewRequest("POST", u, buf) - if err != nil { - done(m, 1, "HTTPSender error: "+err.Error()) - return - } + req, _ := http.NewRequest("POST", u, buf) if h, exists := m.Opts.Get("http_headers"); exists { headers = h.(map[string]string) diff --git a/components/httpsender/httpSender_test.go b/components/httpsender/httpSender_test.go index 7f16fad..e4e115f 100644 --- a/components/httpsender/httpSender_test.go +++ b/components/httpsender/httpSender_test.go @@ -50,16 +50,18 @@ func TestHTTPSender(t *testing.T) { Convey("Given an HTTP sender with defined URL", t, func() { sender := &HTTPSender{ Config: Config{ - URL: "http://example.com", - Workers: 1, + Workers: 1, + URL: "http://example.com", + Insecure: true, }, } - Convey("When is initialized", func() { + Convey("When it is initialized", func() { s := sender.Spawn(0).(*HTTPSender) Convey("Then the config should be ok", func() { So(s.Client, ShouldNotBeNil) + So(s.Workers(), ShouldEqual, 1) }) }) @@ -160,6 +162,44 @@ func TestHTTPSender(t *testing.T) { }) }) + Convey("When a message is received with headers", func() { + var url string + var headerValue string + + s := sender.Spawn(0).(*HTTPSender) + m := utils.NewMessage() + m.PushPayload([]byte("Hello World")) + m.Opts.Set("http_headers", map[string]string{ + "key": "value", + }) + + s.Client = NewTestClient(200, func(req *http.Request) { + url = req.URL.String() + headerValue = req.Header.Get("key") + }) + + d := &Doner{ + doneCalled: make(chan struct { + code int + status string + }, 1), + } + d.On("Done", mock.AnythingOfType("*utils.Message"), + mock.AnythingOfType("int"), mock.AnythingOfType("string")) + + s.OnMessage(m, d.Done) + + Convey("Then the message should be sent with headers set", func() { + result := <-d.doneCalled + So(result.status, ShouldEqual, "200 OK") + So(result.code, ShouldBeZeroValue) + So(url, ShouldEqual, "http://example.com/") + So(headerValue, ShouldEqual, "value") + + d.AssertExpectations(t) + }) + }) + Convey("When a message without payload is received", func() { var url string s := sender.Spawn(0).(*HTTPSender) @@ -195,7 +235,7 @@ func TestHTTPSender(t *testing.T) { s := sender.Spawn(0).(*HTTPSender) m := utils.NewMessage() - m.PushPayload([]byte("Hello World")) + m.PushPayload(nil) s.Client = NewTestClient(200, func(req *http.Request) { req.Write(nil) @@ -223,7 +263,9 @@ func TestHTTPSender(t *testing.T) { }) Convey("Given an HTTP sender with invalid URL", t, func() { - sender := &HTTPSender{} + sender := &HTTPSender{ + Config: Config{Insecure: true}, + } s := sender.Spawn(0).(*HTTPSender) Convey("When try to send messages", func() { diff --git a/components/limiter/limiter_test.go b/components/limiter/limiter_test.go index e69d9e7..a641311 100644 --- a/components/limiter/limiter_test.go +++ b/components/limiter/limiter_test.go @@ -31,6 +31,14 @@ func TestHTTPSender(t *testing.T) { } limiter.Init(0) + Convey("When the numer of worker is requested", func() { + workers := limiter.Workers() + + Convey("Should be only one", func() { + So(workers, ShouldEqual, 1) + }) + }) + Convey("When the limit number of messages are reached", func() { clk := limiter.clk.(*clock.Mock) n := Nexter{