Skip to content

Commit

Permalink
Array support 6 UUID (#307)
Browse files Browse the repository at this point in the history 
* update test data

* initial change for array uuid

* uuid array type support

* update travis timeout setting

* change travis timout behavior

* change travis make process number

* lint change and travis change

* lint comment

* checkin test data and address comment

* lint typo

* add more test cases
  • Loading branch information
@voyager-dw
voyager-dw committed Sep 26, 2019
1 parent 56108d0 commit 10562d5
Show file tree
Hide file tree
Showing 29 changed files with 928 additions and 118 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Expand Up @@ -40,7 +40,7 @@ script:
- while sleep 2m; do echo "=====[ $SECONDS seconds, build still running... ]====="; done &
- make -j 2 travis
- make -j 2 lint
- kill %1
- kill %1
after_success:
# generate coverage output file
- gover . coverage.txt
Expand Down
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Expand Up @@ -188,6 +188,7 @@ add_library(algorithm SHARED
query/hll.cu
query/concurrent_unordered_map.hpp
query/iterator.hpp
query/iterator.cu
query/memory.cu
query/memory.hpp
query/measure_transform.cu
Expand Down
4 changes: 2 additions & 2 deletions memstore/archiving_test.go
View file
Expand Up @@ -272,7 +272,7 @@ var _ = ginkgo.Describe("archiving", func() {

// Old version of archiving store should be purged.
for i, column := range archiveBatch0.Columns {
if i != 3 {
if i != 3 && i != 4 {
Ω(column.(*archiveVectorParty).values).Should(BeNil())
Ω(column.(*archiveVectorParty).nulls).Should(BeNil())
Ω(column.(*archiveVectorParty).counts).Should(BeNil())
Expand All @@ -283,7 +283,7 @@ var _ = ginkgo.Describe("archiving", func() {

// If a batch is partially read, it should not be purged
for i, column := range batch101.Columns {
if i != 3 {
if i != 3 && i != 4 {
Ω(column.(*cLiveVectorParty).GetMode()).ShouldNot(BeEquivalentTo(memCom.AllValuesDefault))
Ω(column.(*cLiveVectorParty).values).ShouldNot(BeNil())
} else {
Expand Down
14 changes: 11 additions & 3 deletions memstore/common/data_type.go
View file
Expand Up @@ -407,6 +407,15 @@ func ConvertToFloat64(value interface{}) (float64, bool) {
return float64(0), false
}

func UUIDFromString(str string) ([2]uint64, error) {
u, err := uuid.FromString(str)
if err != nil {
return [2]uint64{}, err
}
bytes := u.Bytes()
return *(*[2]uint64)(unsafe.Pointer(&bytes[0])), nil
}

// ConvertToUUID convert input into uuid type ([2]uint64) at best effort
func ConvertToUUID(value interface{}) ([2]uint64, bool) {
switch v := value.(type) {
Expand All @@ -418,12 +427,11 @@ func ConvertToUUID(value interface{}) ([2]uint64, bool) {
}
return [2]uint64{}, false
case string:
u, err := uuid.FromString(string(v))
val, err := UUIDFromString(string(v))
if err != nil {
return [2]uint64{}, false
}
bytes := u.Bytes()
return *(*[2]uint64)(unsafe.Pointer(&bytes[0])), true
return val, true
}

return [2]uint64{}, false
Expand Down
34 changes: 28 additions & 6 deletions query/aql_compiler.go
View file
Expand Up @@ -766,14 +766,22 @@ func (qc *AQLQueryContext) Rewrite(expression expr.Expr) expr.Expr {
e.RHS = expr.Cast(e.RHS, highestType)
}

if rhs != nil && lhs.DataType == memCom.GeoPoint {
if rhs != nil && e.LHS.Type() == expr.GeoPoint {
if val, err := memCom.GeoPointFromString(rhs.Val); err != nil {
qc.Error = err
} else {
e.RHS = &expr.GeopointLiteral{
Val: val,
}
}
} else if rhs != nil && e.LHS.Type() == expr.UUID {
if val, err := memCom.UUIDFromString(rhs.Val); err != nil {
qc.Error = err
} else {
e.RHS = &expr.UUIDLiteral{
Val: val,
}
}
}
case expr.IN:
return qc.expandINop(e)
Expand Down Expand Up @@ -1100,13 +1108,27 @@ func (qc *AQLQueryContext) Rewrite(expression expr.Expr) expr.Expr {
nil, "array function %s argument type mismatch", e.Name)
break
}
if val, err := memCom.GeoPointFromString(strLiteral.Val); err != nil {
val, err := memCom.GeoPointFromString(strLiteral.Val)
if err != nil {
qc.Error = err
break
} else {
literalExpr = &expr.GeopointLiteral{
Val: val,
}
}
literalExpr = &expr.GeopointLiteral{
Val: val,
}
case memCom.UUID:
strLiteral, ok := secondArg.(*expr.StringLiteral)
if !ok {
qc.Error = utils.StackError(nil, "array function %s needs uuid string literal", e.Name)
break
}
val, err := memCom.UUIDFromString(strLiteral.Val)
if err != nil {
qc.Error = err
break
}
literalExpr = &expr.UUIDLiteral{
Val: val,
}
case memCom.Uint8, memCom.Uint16, memCom.Uint32, memCom.Int8, memCom.Int16, memCom.Int32:
ok := false
Expand Down
221 changes: 220 additions & 1 deletion query/aql_compiler_test.go
View file
Expand Up @@ -3607,13 +3607,15 @@ var _ = ginkgo.Describe("AQL compiler", func() {
memCom.ArrayBool,
memCom.ArrayGeoPoint,
memCom.ArraySmallEnum,
memCom.ArrayUUID,
},
ColumnIDs: map[string]int{
"request_at": 0,
"int_array_field": 1,
"bool_array_field": 2,
"geopoint_array_field": 3,
"enum_array_field": 4,
"uuid_array_field": 5,
},
Schema: metaCom.Table{
Name: "table1",
Expand All @@ -3624,6 +3626,7 @@ var _ = ginkgo.Describe("AQL compiler", func() {
{Name: "bool_array_field", Type: metaCom.ArrayBool},
{Name: "geopoint_array_field", Type: metaCom.ArrayGeoPoint},
{Name: "enum_array_field", Type: metaCom.ArraySmallEnum},
{Name: "uuid_array_field", Type: metaCom.ArrayUUID},
},
},
EnumDicts: map[string]memCom.EnumDict{
Expand Down Expand Up @@ -3689,7 +3692,7 @@ var _ = ginkgo.Describe("AQL compiler", func() {
ExprType: expr.Unsigned,
}))

// contains works as dimension and filter
// contains works as dimension and filter for geopint
qc.Query = &queryCom.AQLQuery{
Table: "table1",
Measures: []queryCom.Measure{
Expand Down Expand Up @@ -3732,6 +3735,113 @@ var _ = ginkgo.Describe("AQL compiler", func() {
ExprType: expr.Boolean,
}))

// not work for invalid geopint string literal
qc.Query = &queryCom.AQLQuery{
Table: "table1",
Measures: []queryCom.Measure{
{Expr: "1"},
},
Dimensions: []queryCom.Dimension{
{Expr: "contains(enum_array_field, 'foo')"},
},
Filters: []string{
"request_at >= 1540399140000",
"request_at < 1540399320000",
"contains(geopoint_array_field, '12345')",
},
}
qc.parseExprs()
qc.resolveTypes()
Ω(qc.Error).ShouldNot(BeNil())

// not work for non-string literal
qc.Query = &queryCom.AQLQuery{
Table: "table1",
Measures: []queryCom.Measure{
{Expr: "1"},
},
Dimensions: []queryCom.Dimension{
{Expr: "contains(enum_array_field, 'foo')"},
},
Filters: []string{
"request_at >= 1540399140000",
"request_at < 1540399320000",
"contains(geopoint_array_field, 12345)",
},
}
qc.parseExprs()
qc.resolveTypes()
Ω(qc.Error).ShouldNot(BeNil())

// contains works as dimension and filter for uuid
qc.Query = &queryCom.AQLQuery{
Table: "table1",
Measures: []queryCom.Measure{
{Expr: "1"},
},
Dimensions: []queryCom.Dimension{
{Expr: "contains(enum_array_field, 'foo')"},
},
Filters: []string{
"request_at >= 1540399140000",
"request_at < 1540399320000",
"contains(uuid_array_field, '11223344-1122-1122-1122-112233445566')",
},
}
qc.parseExprs()
qc.resolveTypes()
qc.processFilters()
qc.processDimensions()
Ω(qc.Error).Should(BeNil())
Ω(qc.Query.FiltersParsed[0]).Should(Equal(&expr.BinaryExpr{
Op: expr.ARRAY_CONTAINS,
ExprType: expr.Boolean,
LHS: &expr.VarRef{
Val: "uuid_array_field",
DataType: memCom.ArrayUUID,
ColumnID: 5,
},
RHS: &expr.UUIDLiteral{Val: [2]uint64{2454780729447621137, 7373874951294624273}},
}))

// not work for invalid string literal uuid
qc.Query = &queryCom.AQLQuery{
Table: "table1",
Measures: []queryCom.Measure{
{Expr: "1"},
},
Dimensions: []queryCom.Dimension{
{Expr: "contains(enum_array_field, 'foo')"},
},
Filters: []string{
"request_at >= 1540399140000",
"request_at < 1540399320000",
"contains(uuid_array_field, '12345')",
},
}
qc.parseExprs()
qc.resolveTypes()
Ω(qc.Error).ShouldNot(BeNil())

// not work for non-string literal uuid
qc.Query = &queryCom.AQLQuery{
Table: "table1",
Measures: []queryCom.Measure{
{Expr: "1"},
},
Dimensions: []queryCom.Dimension{
{Expr: "contains(enum_array_field, 'foo')"},
},
Filters: []string{
"request_at >= 1540399140000",
"request_at < 1540399320000",
"contains(uuid_array_field, 12345)",
},
}
qc.parseExprs()
qc.resolveTypes()
Ω(qc.Error).ShouldNot(BeNil())

// element_at works as dimension and filter
qc.Query = &queryCom.AQLQuery{
Table: "table1",
Expand Down Expand Up @@ -3775,4 +3885,113 @@ var _ = ginkgo.Describe("AQL compiler", func() {
ExprType: expr.Unsigned,
}))
})

ginkgo.It("parses uuid expressions", func() {
q := &queryCom.AQLQuery{
Table: "trips",
Measures: []queryCom.Measure{
{
Expr: "count(*)",
},
},
}

qc := &AQLQueryContext{
TableIDByAlias: map[string]int{
"trips": 0,
},
TableScanners: []*TableScanner{
{
Schema: &memCom.TableSchema{
ValueTypeByColumn: []memCom.DataType{
memCom.Uint16,
memCom.UUID,
},
ColumnIDs: map[string]int{
"city_id": 0,
"trip_uuid": 1,
},
Schema: metaCom.Table{
Columns: []metaCom.Column{
{Name: "city_id", Type: metaCom.Uint16},
{Name: "trip_uuid", Type: metaCom.UUID},
},
},
},
},
},
}
qc.Query = q
qc.parseExprs()
Ω(qc.Error).Should(BeNil())
Ω(q.Measures[0].ExprParsed).Should(Equal(&expr.Call{
Name: "count",
Args: []expr.Expr{
&expr.Wildcard{},
},
}))

qc.Query = &queryCom.AQLQuery{
Table: "trips",
Measures: []queryCom.Measure{
{
Expr: "count(*)",
},
},
Filters: []string{
"trip_uuid = '11223344-1122-1122-1122-112233445566'",
},
}
qc.parseExprs()
Ω(qc.Error).Should(BeNil())
Ω(qc.Query.Measures[0].ExprParsed).Should(Equal(&expr.Call{
Name: "count",
Args: []expr.Expr{
&expr.Wildcard{},
},
}))
Ω(qc.Query.FiltersParsed[0]).Should(Equal(&expr.BinaryExpr{
Op: expr.EQ,
LHS: &expr.VarRef{Val: "trip_uuid"},
RHS: &expr.StringLiteral{Val: "11223344-1122-1122-1122-112233445566"},
}))

qc.resolveTypes()

Ω(qc.Query.FiltersParsed[0]).Should(Equal(&expr.BinaryExpr{
Op: expr.EQ,
LHS: &expr.VarRef{Val: "trip_uuid", ColumnID: 1, TableID: 0, ExprType: expr.UUID, DataType: memCom.UUID},
RHS: &expr.UUIDLiteral{Val: [2]uint64{2454780729447621137, 7373874951294624273}},
ExprType: expr.Boolean,
}))

// invalid uuid string
qc.Query = &queryCom.AQLQuery{
Table: "trips",
Measures: []queryCom.Measure{
{
Expr: "count(*)",
},
},
Filters: []string{
"trip_uuid = '11223344-1122-1122-1122-1122334455'",
},
}
qc.parseExprs()
Ω(qc.Error).Should(BeNil())
Ω(qc.Query.Measures[0].ExprParsed).Should(Equal(&expr.Call{
Name: "count",
Args: []expr.Expr{
&expr.Wildcard{},
},
}))
Ω(qc.Query.FiltersParsed[0]).Should(Equal(&expr.BinaryExpr{
Op: expr.EQ,
LHS: &expr.VarRef{Val: "trip_uuid"},
RHS: &expr.StringLiteral{Val: "11223344-1122-1122-1122-1122334455"},
}))

qc.resolveTypes()
Ω(qc.Error).ShouldNot(BeNil())
})
})

0 comments on commit 10562d5

Please sign in to comment.