From 25fcf931c5222536cad3a1f73fda4bc1922ed043 Mon Sep 17 00:00:00 2001 From: Roman Date: Sun, 2 Jul 2023 14:23:14 +0400 Subject: [PATCH] Fix creation of columns post insert --- .github/workflows/test.yml | 2 +- collection.go | 8 +++++++- column_test.go | 28 ++++++++++++++++++++++++++++ go.mod | 14 +++++++------- go.sum | 34 +++++++++++++++------------------- 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4109668..c57382a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ["1.19"] + go: ["1.19", "1.20"] steps: - name: Set up Go ${{ matrix.go }} uses: actions/setup-go@v3 diff --git a/collection.go b/collection.go index 1b81de3..6e9d2b9 100644 --- a/collection.go +++ b/collection.go @@ -180,7 +180,13 @@ func (c *Collection) CreateColumn(columnName string, column Column) error { return fmt.Errorf("column: unable to create column '%s', already exists", columnName) } - column.Grow(uint32(c.opts.Capacity)) + // Grow the column to the current capacity + capacity := uint32(atomic.LoadUint64(&c.count)) + if c.opts.Capacity > int(capacity) { + capacity = uint32(c.opts.Capacity) + } + + column.Grow(capacity) c.cols.Store(columnName, columnFor(columnName, column)) // If necessary, create a primary key column diff --git a/column_test.go b/column_test.go index cb8bc71..cc8b817 100644 --- a/column_test.go +++ b/column_test.go @@ -734,6 +734,34 @@ func TestNumberMerge(t *testing.T) { }) } +// Tests the case where a column is created after inserting a large enough amount of data +func TestIssue89(t *testing.T) { + coll := NewCollection() + coll.CreateColumn("foo", ForString()) + + // Should be larger than a single chunk + for i := 0; i < 16385; i++ { + coll.Insert(func(row Row) error { + row.SetString("foo", fmt.Sprintf("foo-%d", i)) + return nil + }) + } + + // set up a derived column of data, over initial capacity + coll.CreateColumn("bar", ForString()) + assert.NoError(t, coll.Query(func(txn *Txn) error { + src := txn.String("foo") + dest := txn.String("bar") + return txn.Range(func(_ uint32) { + value, ok := src.Get() + assert.True(t, ok) + dest.Set("bar-" + value[4:]) + }) + })) + + assert.Equal(t, 16385, coll.Count()) +} + func invoke(any interface{}, name string, args ...interface{}) []reflect.Value { inputs := make([]reflect.Value, len(args)) for i := range args { diff --git a/go.mod b/go.mod index e738b32..c134b44 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/kelindar/iostream v1.3.0 github.com/kelindar/simd v1.1.2 github.com/kelindar/smutex v1.0.0 - github.com/klauspost/compress v1.15.12 - github.com/stretchr/testify v1.8.1 + github.com/klauspost/compress v1.16.6 + github.com/stretchr/testify v1.8.4 github.com/tidwall/btree v1.6.0 github.com/zeebo/xxh3 v1.0.2 @@ -17,12 +17,12 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 - github.com/kelindar/async v1.0.0 + github.com/dustin/go-humanize v1.0.1 + github.com/kelindar/async v1.1.0 github.com/kelindar/xxrand v1.0.2 - github.com/klauspost/cpuid/v2 v2.2.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/time v0.2.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/time v0.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d1bb59c..67e31c5 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,10 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/kelindar/async v1.0.0 h1:oJiFAt3fVB/b5zVZKPBU+pP9lR3JVyeox9pYlpdnIK8= -github.com/kelindar/async v1.0.0/go.mod h1:bJRlwaRiqdHi+4dpVDNHdwgyRyk6TxpA21fByLf7hIY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/kelindar/async v1.1.0 h1:uCO6Wn7kuhmRoG9z26+onU2+MZ7vgscRMlFUIAjyPpo= +github.com/kelindar/async v1.1.0/go.mod h1:bJRlwaRiqdHi+4dpVDNHdwgyRyk6TxpA21fByLf7hIY= github.com/kelindar/bitmap v1.4.1 h1:Ih0BWMYXkkZxPMU536DsQKRhdvqFl7tuNjImfLJWC6E= github.com/kelindar/bitmap v1.4.1/go.mod h1:4QyD+TDbfgy8oYB9oC4JzqfudYCYIjhbSP7iLraP+28= github.com/kelindar/intmap v1.1.0 h1:S+YEDvw5FQus5UJDEG+xsLp8il3BTYqBMkkuVVZPMH8= @@ -17,31 +17,27 @@ github.com/kelindar/smutex v1.0.0 h1:+LIZYwPz+v3IWPOse764fNaVQGMVxKV6mbD6OWjQV3o github.com/kelindar/smutex v1.0.0/go.mod h1:nMbCZeAHWCsY9Kt4JqX7ETd+NJeR6Swy9im+Th+qUZQ= github.com/kelindar/xxrand v1.0.2 h1:tODvTkfkYTPUE0W1Tslli7SWng8+Y1hiRI8upDUZIA0= github.com/kelindar/xxrand v1.0.2/go.mod h1:tb7XX0TvlKSIsCqkVUs7GAWdkeab3Ln2vWWxHEADDuA= -github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= -github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/cpuid/v2 v2.2.1 h1:U33DW0aiEj633gHYw3LoDNfkDiYnE5Q8M/TKJn2f2jI= -github.com/klauspost/cpuid/v2 v2.2.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= +github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=