Skip to content

parser shifts buffer to reduce large block allocations #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 6, 2016
Merged

parser shifts buffer to reduce large block allocations #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@ const (
var bufferPool internal.BufferPool

type parser struct {
buffer []byte
reader io.Reader
lastRead time.Time
//buffer is a slice of bigBuffer
bigBuffer, buffer []byte
reader io.Reader
lastRead time.Time
}

func newParser(reader io.Reader) *parser {
Expand All @@ -26,7 +27,23 @@ func newParser(reader io.Reader) *parser {

func (p *parser) readMore() (int, error) {
if len(p.buffer) == cap(p.buffer) {
newBuffer := make([]byte, len(p.buffer), len(p.buffer)+defaultBufSize)
var newBuffer []byte
switch {
//initialize the parser
case len(p.bigBuffer) == 0:
p.bigBuffer = make([]byte, defaultBufSize)
newBuffer = p.bigBuffer[0:0]

//shift buffer back to the start of bigBuffer
case 2*len(p.buffer) <= len(p.bigBuffer):
newBuffer = p.bigBuffer[0:len(p.buffer)]

//reallocate big buffer with enough space to shift buffer
default:
p.bigBuffer = make([]byte, 2*len(p.buffer))
newBuffer = p.bigBuffer[0:len(p.buffer)]
}

copy(newBuffer, p.buffer)
p.buffer = newBuffer
}
Expand Down
176 changes: 75 additions & 101 deletions parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)

func BenchmarkParser_ReadMessage(b *testing.B) {
Expand All @@ -17,38 +17,40 @@ func BenchmarkParser_ReadMessage(b *testing.B) {
}
}

func TestParser_JumpLength(t *testing.T) {
stream := "8=FIXT.1.19=11135=D34=449=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=2348=FIXT.1.19=9535=D34=549=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=198"

reader := strings.NewReader(stream)
parser := newParser(reader)
index, err := parser.jumpLength()
type ParserSuite struct {
suite.Suite
*parser
}

if err != nil {
t.Error("Unexpected error: ", err)
}
func TestParserSuite(t *testing.T) {
suite.Run(t, new(ParserSuite))
}

expectedIndex := 111 + 17 - 1
if index != expectedIndex {
t.Error("expected index ", expectedIndex, " got ", index)
}
func (s *ParserSuite) SetupTest() {
s.parser = new(parser)
}

func TestParser_BadLength(t *testing.T) {
func (s *ParserSuite) TestJumpLength() {
stream := "8=FIXT.1.19=11135=D34=449=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=2348=FIXT.1.19=9535=D34=549=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=198"

reader := strings.NewReader(stream)
parser := newParser(reader)
s.reader = strings.NewReader(stream)
index, err := s.parser.jumpLength()
s.Nil(err)

bytes, _ := parser.ReadMessage()
expectedIndex := 111 + 17 - 1
s.Equal(expectedIndex, index)
}

if stream != bytes.String() {
t.Errorf("Expected %v, got %v", stream, bytes.String())
}
func (s *ParserSuite) TestBadLength() {
stream := "8=FIXT.1.19=11135=D34=449=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=2348=FIXT.1.19=9535=D34=549=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=198"

s.reader = strings.NewReader(stream)
bytes, _ := s.parser.ReadMessage()

s.Equal(stream, bytes.String())
}

func TestParser_findStart(t *testing.T) {
func (s *ParserSuite) TestFindStart() {
var testCases = []struct {
stream string
expectError bool
Expand All @@ -59,31 +61,22 @@ func TestParser_findStart(t *testing.T) {
{stream: "hello8=FIX.4.0", expectedStart: 5},
}
for _, tc := range testCases {
reader := strings.NewReader(tc.stream)
parser := newParser(reader)
s.SetupTest()

start, err := parser.findStart()
s.reader = strings.NewReader(tc.stream)

if err != nil {
if !tc.expectError {
t.Error("unxpected error", err)
}
start, err := s.parser.findStart()
if tc.expectError {
s.NotNil(err)
continue
}

if err == nil && tc.expectError {
t.Error("expected error")
}

if start != tc.expectedStart {
t.Errorf("Expected start to be %v, but was %v", tc.expectedStart, start)
}
s.Nil(err)
s.Equal(tc.expectedStart, start)
}

}

func TestParser_ReadEOF(t *testing.T) {

func (s *ParserSuite) TestReadEOF() {
var testCases = []struct {
stream string
}{
Expand All @@ -92,21 +85,20 @@ func TestParser_ReadEOF(t *testing.T) {
}

for _, tc := range testCases {
reader := strings.NewReader(tc.stream)
parser := newParser(reader)
bytes, err := parser.ReadMessage()
assert.Nil(t, bytes)
s.SetupTest()

if err == nil || err.Error() != "EOF" {
t.Error("Expected EOF")
}
s.reader = strings.NewReader(tc.stream)
bytes, err := s.parser.ReadMessage()
s.Nil(bytes)

s.NotNil(err)
s.Equal("EOF", err.Error())
}
}

func TestParser_ReadMessage(t *testing.T) {
func (s *ParserSuite) TestReadMessage() {
stream := "hello8=FIX.4.09=5blah10=1038=FIX.4.09=4foo10=103"
reader := strings.NewReader(stream)
parser := newParser(reader)
s.reader = strings.NewReader(stream)

var testCases = []struct {
expectedBytes string
Expand All @@ -118,68 +110,50 @@ func TestParser_ReadMessage(t *testing.T) {
}

for _, tc := range testCases {
msg, err := parser.ReadMessage()

if err != nil {
t.Error("Unexpected error", err)
}

if tc.expectedBytes != msg.String() {
t.Errorf("Expected '%v' got '%v'", tc.expectedBytes, msg.String())
}

if cap(parser.buffer) != tc.expectedBufferCap {
t.Errorf("Expected capacity %v got %v", tc.expectedBufferCap, cap(parser.buffer))
}

if len(parser.buffer) != tc.expectedBufferLen {
t.Errorf("Expected len %v got %v", tc.expectedBufferLen, len(parser.buffer))
}
msg, err := s.parser.ReadMessage()
s.Nil(err)

s.Equal(tc.expectedBytes, msg.String())
s.Equal(tc.expectedBufferCap, cap(s.parser.buffer))
s.Equal(tc.expectedBufferLen, len(s.parser.buffer))
}
}

func TestParser_ReadMessageGrowBuffer(t *testing.T) {
func (s *ParserSuite) TestReadMessageGrowBuffer() {
stream := "hello8=FIX.4.09=5blah10=1038=FIX.4.09=4foo10=103"

var testCases = []struct {
initialBufCap int
expectedBytes string
expectedBufferLen int
expectedBufferCap int
initialBufCap int
expectedBytes string
expectedBufferLen int
expectedBufferCap int
expectedBigBufferLen int
}{
{initialBufCap: 0, expectedBufferCap: defaultBufSize - 31, expectedBufferLen: len(stream) - 31},
{initialBufCap: 4, expectedBufferCap: defaultBufSize - 27, expectedBufferLen: len(stream) - 31},
{initialBufCap: 8, expectedBufferCap: defaultBufSize - 23, expectedBufferLen: len(stream) - 31},
{initialBufCap: 14, expectedBufferCap: defaultBufSize - 17, expectedBufferLen: len(stream) - 31},
{initialBufCap: 16, expectedBufferCap: defaultBufSize - 15, expectedBufferLen: len(stream) - 31},
{initialBufCap: 23, expectedBufferCap: defaultBufSize - 8, expectedBufferLen: len(stream) - 31},
{initialBufCap: 30, expectedBufferCap: defaultBufSize - 1, expectedBufferLen: len(stream) - 31},
{initialBufCap: 31, expectedBufferCap: 0, expectedBufferLen: 0},
{initialBufCap: 40, expectedBufferCap: 9, expectedBufferLen: 9},
{initialBufCap: 60, expectedBufferCap: 29, expectedBufferLen: 25},
{initialBufCap: 80, expectedBufferCap: 49, expectedBufferLen: 25},
{initialBufCap: 0, expectedBufferCap: defaultBufSize - 31, expectedBufferLen: len(stream) - 31, expectedBigBufferLen: defaultBufSize},
{initialBufCap: 4, expectedBufferCap: 6, expectedBufferLen: 6, expectedBigBufferLen: 32},
{initialBufCap: 8, expectedBufferCap: 6, expectedBufferLen: 6, expectedBigBufferLen: 32},
{initialBufCap: 14, expectedBufferCap: 10, expectedBufferLen: 10, expectedBigBufferLen: 36},
{initialBufCap: 16, expectedBufferCap: 18, expectedBufferLen: 18, expectedBigBufferLen: 44},
{initialBufCap: 23, expectedBufferCap: 10, expectedBufferLen: 10, expectedBigBufferLen: 36},
{initialBufCap: 30, expectedBufferCap: 24, expectedBufferLen: 24, expectedBigBufferLen: 50},
{initialBufCap: 31, expectedBufferCap: 0, expectedBufferLen: 0, expectedBigBufferLen: 31},
{initialBufCap: 40, expectedBufferCap: 9, expectedBufferLen: 9, expectedBigBufferLen: 40},
{initialBufCap: 60, expectedBufferCap: 29, expectedBufferLen: 25, expectedBigBufferLen: 60},
{initialBufCap: 80, expectedBufferCap: 49, expectedBufferLen: 25, expectedBigBufferLen: 80},
}

for _, tc := range testCases {
parser := newParser(strings.NewReader(stream))
parser.buffer = make([]byte, 0, tc.initialBufCap)
msg, err := parser.ReadMessage()

if err != nil {
t.Fatal("unexpected err: ", err)
}

if msg.String() != "8=FIX.4.09=5blah10=103" {
t.Error("Didn't get expected message, got ", msg.String())
}

if cap(parser.buffer) != tc.expectedBufferCap {
t.Errorf("expected cap %v, got %v", tc.expectedBufferCap, cap(parser.buffer))
}

if len(parser.buffer) != tc.expectedBufferLen {
t.Errorf("expected len %v, got %v", tc.expectedBufferLen, len(parser.buffer))
}
s.SetupTest()

s.reader = strings.NewReader(stream)
s.parser.bigBuffer = make([]byte, tc.initialBufCap)
s.parser.buffer = s.parser.bigBuffer[0:0]
msg, err := s.parser.ReadMessage()

s.Nil(err)
s.Equal("8=FIX.4.09=5blah10=103", msg.String())
s.Equal(tc.expectedBigBufferLen, len(s.parser.bigBuffer))
s.Equal(tc.expectedBufferCap, cap(s.parser.buffer))
s.Equal(tc.expectedBufferLen, len(s.parser.buffer))
}
}