From f35d1dc9c8c8d0277ea03e9e89ad94cef6b0160e Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Fri, 17 Feb 2023 11:35:51 -0600 Subject: [PATCH 1/2] added support for time quantum inserts with explicit timestamps --- sql3/errors.go | 10 ++++- sql3/planner/expression.go | 49 +++++++++++------------ sql3/planner/expressiontypes.go | 19 ++++----- sql3/planner/opinsert.go | 56 ++++++++++++++++++++++++++- sql3/test/defs/defs_date_functions.go | 6 --- sql3/test/defs/defs_timequantum.go | 25 +++++++++++- 6 files changed, 118 insertions(+), 47 deletions(-) diff --git a/sql3/errors.go b/sql3/errors.go index f04d1e401..5052863d5 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -109,7 +109,8 @@ const ( // insert errors - ErrInsertValueOutOfRange errors.Code = "ErrInsertValueOutOfRange" + ErrInsertValueOutOfRange errors.Code = "ErrInsertValueOutOfRange" + ErrUnexpectedTimeQuantumTupleLength errors.Code = "ErrUnexpectedTimeQuantumTupleLength" // bulk insert errors @@ -706,6 +707,13 @@ func NewErrInsertValueOutOfRange(line, col int, columnName string, rowNumber int ) } +func NewErrUnexpectedTimeQuantumTupleLength(line, col int, columnName string, rowNumber int, badValue []interface{}, length int) error { + return errors.New( + ErrInsertValueOutOfRange, + fmt.Sprintf("[%d:%d] inserting value into column '%s', row %d, value '%v' out of range", line, col, columnName, rowNumber, badValue), + ) +} + // bulk insert func NewErrReadingDatasource(line, col int, dataSource string, errorText string) error { diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 7ad6314de..292e0716d 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -41,6 +41,14 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType, return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) } return pql.NewDecimal(val*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil + + case *parser.DataTypeTimestamp: + val, ok := value.(int64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) + } + tm := time.Unix(val, 0).UTC() + return tm, nil } case *parser.DataTypeID: @@ -57,6 +65,14 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType, return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) } return pql.NewDecimal(int64(val)*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil + + case *parser.DataTypeTimestamp: + val, ok := value.(int64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected value type '%T'", value) + } + tm := time.Unix(val, 0).UTC() + return tm, nil } case *parser.DataTypeDecimal: @@ -2435,34 +2451,15 @@ func newExprTupleLiteralPlanExpression(members []types.PlanExpression, dataType } func (n *exprTupleLiteralPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) { - timestampEval, err := n.members[0].Evaluate(currentRow) - if err != nil { - return nil, err - } - - // if it is a string, do a coercion - if val, ok := timestampEval.(string); ok { - if tm, err := timestampFromString(val); err != nil { - return nil, sql3.NewErrInvalidTypeCoercion(0, 0, val, n.members[0].Type().TypeDescription()) - } else { - timestampEval = tm + result := make([]interface{}, len(n.members)) + for i, m := range n.members { + v, err := m.Evaluate(currentRow) + if err != nil { + return nil, err } + result[i] = v } - - setEval, err := n.members[1].Evaluate(currentRow) - if err != nil { - return nil, err - } - - // nil if anything is nil - if timestampEval == nil || setEval == nil { - return nil, nil - } - - return []interface{}{ - timestampEval, - setEval, - }, nil + return result, nil } func (n *exprTupleLiteralPlanExpression) Type() parser.ExprDataType { diff --git a/sql3/planner/expressiontypes.go b/sql3/planner/expressiontypes.go index cab4add69..43ff26f3c 100644 --- a/sql3/planner/expressiontypes.go +++ b/sql3/planner/expressiontypes.go @@ -290,12 +290,8 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par if len(source.Members) != 2 { return false } - _, ok := source.Members[0].(*parser.DataTypeTimestamp) - if !ok { - _, ok = source.Members[0].(*parser.DataTypeString) - if !ok { - return false - } + if !typesAreAssignmentCompatible(parser.NewDataTypeTimestamp(), source.Members[0]) { + return false } _, ok = source.Members[1].(*parser.DataTypeStringSet) if !ok { @@ -324,12 +320,8 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par if len(source.Members) != 2 { return false } - _, ok := source.Members[0].(*parser.DataTypeTimestamp) - if !ok { - _, ok = source.Members[0].(*parser.DataTypeString) - if !ok { - return false - } + if !typesAreAssignmentCompatible(parser.NewDataTypeTimestamp(), source.Members[0]) { + return false } _, ok = source.Members[1].(*parser.DataTypeIDSet) if !ok { @@ -354,6 +346,9 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par switch sourceType.(type) { case *parser.DataTypeTimestamp: return true + case *parser.DataTypeInt: + //could be a int convertable to a date + return true case *parser.DataTypeString: //could be a string parseable as a date return true diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index b5f7dee22..e2914c61a 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -13,6 +13,7 @@ import ( "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" "github.com/pkg/errors" ) @@ -253,9 +254,9 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { } case pilosa.FieldTypeTime: - row.Time = qbatchTime switch v := eval.(type) { case []int64: + row.Time = qbatchTime uint64s := make([]uint64, len(v)) for i := range v { if v[i] < 0 { @@ -264,6 +265,59 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { uint64s[i] = uint64(v[i]) } row.Values[posVals[idx]] = uint64s + + case []interface{}: + // it's a tuple, check length + if len(v) != 2 { + return nil, sql3.NewErrUnexpectedTimeQuantumTupleLength(0, 0, columnName, rowNumber+1, v, len(v)) + } + + tupleType, ok := iv.Type().(*parser.DataTypeTuple) + if !ok { + return nil, sql3.NewErrInternalf("unexpected tuple type '%T'", v[0]) + } + + // first member must be a timestamp or coercable as one + cval, err := coerceValue(tupleType.Members[0], parser.NewDataTypeTimestamp(), v[0], parser.Pos{Line: 0, Column: 0}) + if err != nil { + return nil, err + } + tval, ok := cval.(time.Time) + if !ok { + return nil, sql3.NewErrInternalf("unexpected tuple time value type '%T'", v[0]) + } + var qrowTime fbbatch.QuantizedTime + qrowTime.Set(tval) + row.Time = qrowTime + + // second member must be a set of the correct type + targetCol := i.targetColumns[idx] + + switch targetCol.Type().(type) { + case *parser.DataTypeStringSetQuantum: + sval, ok := v[1].([]string) + if !ok { + return nil, sql3.NewErrInternalf("string set type expected '%T'", v[1]) + } + row.Values[posVals[idx]] = sval + + case *parser.DataTypeIDSetQuantum: + sval, ok := v[1].([]int64) + if !ok { + return nil, sql3.NewErrInternalf("id set type expected '%T'", v[1]) + } + uint64s := make([]uint64, len(sval)) + for i := range sval { + if sval[i] < 0 { + return nil, sql3.NewErrInternalf("converting negative slice value to uint64: %d", sval[i]) + } + uint64s[i] = uint64(sval[i]) + } + row.Values[posVals[idx]] = uint64s + + default: + return nil, sql3.NewErrInternalf("unexpected set type '%T'", targetCol.Type()) + } default: row.Values[posVals[idx]] = eval } diff --git a/sql3/test/defs/defs_date_functions.go b/sql3/test/defs/defs_date_functions.go index cd35c5099..74383589f 100644 --- a/sql3/test/defs/defs_date_functions.go +++ b/sql3/test/defs/defs_date_functions.go @@ -28,12 +28,6 @@ var datePartTests = TableTest{ ), ExpErr: "an expression of type 'int' cannot be passed to a parameter of type 'string'", }, - { - SQLs: sqls( - "select datepart('1', 2)", - ), - ExpErr: "an expression of type 'int' cannot be passed to a parameter of type 'timestamp'", - }, { SQLs: sqls( "select datepart('1', current_timestamp)", diff --git a/sql3/test/defs/defs_timequantum.go b/sql3/test/defs/defs_timequantum.go index 5aa898e05..6945f5ad6 100644 --- a/sql3/test/defs/defs_timequantum.go +++ b/sql3/test/defs/defs_timequantum.go @@ -7,13 +7,36 @@ var timeQuantumInsertTest = TableTest{ srcHdrs( srcHdr("_id", fldTypeID), srcHdr("i1", fldTypeInt, "min 0", "max 1000"), + srcHdr("ss1", fldTypeStringSet, "timequantum 'YMD'"), srcHdr("ids1", fldTypeIDSet, "timequantum 'YMD'"), ), ), SQLTests: []SQLTest{ { SQLs: sqls( - "insert into time_quantum_insert (_id, i1, ids1) values (1, 1, [1])", + "insert into time_quantum_insert (_id, i1, ss1, ids1) values (1, 1, ['1'], [1])", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "insert into time_quantum_insert (_id, i1, ss1, ids1) values (1, 1, {['1']}, {[1]})", + ), + ExpErr: "an expression of type 'tuple(stringset)' cannot be assigned to type 'stringset'", + }, + { + SQLs: sqls( + "insert into time_quantum_insert (_id, i1, ss1, ids1) values (1, 1, {1676649734, ['1']}, {1676649734, [1]})", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "insert into time_quantum_insert (_id, i1, ss1, ids1) values (1, 1, {'2022-01-01T00:00:00Z', ['1']}, {'2022-01-01T00:00:00Z', [1]})", ), ExpHdrs: hdrs(), ExpRows: rows(), From a6b5086f4b220c775135d8393305d75e53d95dfb Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Fri, 17 Feb 2023 11:40:58 -0600 Subject: [PATCH 2/2] fixed copy pasta --- sql3/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql3/errors.go b/sql3/errors.go index 5052863d5..62354ddf8 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -709,7 +709,7 @@ func NewErrInsertValueOutOfRange(line, col int, columnName string, rowNumber int func NewErrUnexpectedTimeQuantumTupleLength(line, col int, columnName string, rowNumber int, badValue []interface{}, length int) error { return errors.New( - ErrInsertValueOutOfRange, + ErrUnexpectedTimeQuantumTupleLength, fmt.Sprintf("[%d:%d] inserting value into column '%s', row %d, value '%v' out of range", line, col, columnName, rowNumber, badValue), ) }