plan,executor: calculate generated columns in CRUD.

ddl/ddl_api.go

@@ -595,7 +595,7 @@ func (d *ddl) buildTableInfo(tableName model.CIStr, cols []*table.Column, constr
 					return nil, errKeyColumnDoesNotExits.Gen("key column %s doesn't exist in table", key.Column.Name)
 				}
 				// Virtual columns cannot be used in primary key.
-				if len(col.GeneratedExprString) != 0 && !col.GeneratedStored {
+				if col.IsGenerated() && !col.GeneratedStored {
 					return nil, errUnsupportedOnGeneratedColumn.GenByArgs("Defining a virtual generated column as primary key")
 				}
 			}

ddl/generated_column.go

@@ -108,7 +108,7 @@ func checkModifyGeneratedColumn(originCols []*table.Column, oldCol, newCol *tabl
 	var stored = [2]bool{false, false}
 	var cols = [2]*table.Column{oldCol, newCol}
 	for i, col := range cols {
-		if len(col.GeneratedExprString) == 0 || col.GeneratedStored {
+		if !col.IsGenerated() || col.GeneratedStored {
 			stored[i] = true
 		}
 	}
@@ -122,10 +122,10 @@ func checkModifyGeneratedColumn(originCols []*table.Column, oldCol, newCol *tabl
 		if column == oldCol {
 			colName2Generation[newCol.Name.L] = columnGenerationInDDL{
 				position:    i,
-				generated:   len(newCol.GeneratedExprString) != 0,
+				generated:   newCol.IsGenerated(),
 				dependences: newCol.Dependences,
 			}
-		} else if len(column.GeneratedExprString) == -1 {
+		} else if !column.IsGenerated() {
 			colName2Generation[column.Name.L] = columnGenerationInDDL{
 				position:  i,
 				generated: false,

executor/builder.go

@@ -271,10 +271,12 @@ func (b *executorBuilder) buildSet(v *plan.Set) Executor {
 
 func (b *executorBuilder) buildInsert(v *plan.Insert) Executor {
 	ivs := &InsertValues{
-		ctx:     b.ctx,
-		Columns: v.Columns,
-		Lists:   v.Lists,
-		Setlist: v.Setlist,
+		ctx:        b.ctx,
+		Columns:    v.Columns,
+		Lists:      v.Lists,
+		Setlist:    v.Setlist,
+		GenColumns: v.GenCols.Columns,
+		GenExprs:   v.GenCols.Exprs,
 	}
 	if len(v.Children()) > 0 {
 		ivs.SelectExec = b.build(v.Children()[0])
@@ -285,7 +287,7 @@ func (b *executorBuilder) buildInsert(v *plan.Insert) Executor {
 	}
 	insert := &InsertExec{
 		InsertValues: ivs,
-		OnDuplicate:  v.OnDuplicate,
+		OnDuplicate:  append(v.OnDuplicate, v.GenCols.OnDuplicates...),
 		Priority:     v.Priority,
 		Ignore:       v.Ignore,
 	}
@@ -298,7 +300,13 @@ func (b *executorBuilder) buildLoadData(v *plan.LoadData) Executor {
 		b.err = errors.Errorf("Can not get table %d", v.Table.TableInfo.ID)
 		return nil
 	}
-	insertVal := &InsertValues{ctx: b.ctx, Table: tbl, Columns: v.Columns}
+	insertVal := &InsertValues{
+		ctx:        b.ctx,
+		Table:      tbl,
+		Columns:    v.Columns,
+		GenColumns: v.GenCols.Columns,
+		GenExprs:   v.GenCols.Exprs,
+	}
 	tableCols := tbl.Cols()
 	columns, err := insertVal.getColumns(tableCols)
 	if err != nil {

executor/executor_test.go

@@ -1213,6 +1213,123 @@ func (s *testSuite) TestGeneratedColumnWrite(c *C) {
 	}
 }
 
+// TestGeneratedColumnRead tests select generated columns from table.
+// They should be calculated from their generation expressions.
+func (s *testSuite) TestGeneratedColumnRead(c *C) {
+	defer func() {
+		s.cleanEnv(c)
+		testleak.AfterTest(c)()
+	}()
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec(`CREATE TABLE test_gc_read(a int primary key, b int, c int as (a+b), d int as (a*b) stored)`)
+
+	// Insert only column a and b, leave c and d be calculated from them.
+	tk.MustExec(`INSERT INTO test_gc_read (a, b) VALUES (0,null),(1,2),(3,4)`)
+	result := tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`)
+	result.Check(testkit.Rows(`0 <nil> <nil> <nil>`, `1 2 3 2`, `3 4 7 12`))
+
+	tk.MustExec(`INSERT INTO test_gc_read SET a = 5, b = 10`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`)
+	result.Check(testkit.Rows(`0 <nil> <nil> <nil>`, `1 2 3 2`, `3 4 7 12`, `5 10 15 50`))
+
+	tk.MustExec(`REPLACE INTO test_gc_read (a, b) VALUES (5, 6)`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`)
+	result.Check(testkit.Rows(`0 <nil> <nil> <nil>`, `1 2 3 2`, `3 4 7 12`, `5 6 11 30`))
+
+	tk.MustExec(`INSERT INTO test_gc_read (a, b) VALUES (5, 8) ON DUPLICATE KEY UPDATE b = 9`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`)
+	result.Check(testkit.Rows(`0 <nil> <nil> <nil>`, `1 2 3 2`, `3 4 7 12`, `5 9 14 45`))
+
+	// Test select only-generated-column-without-dependences.
+	result = tk.MustQuery(`SELECT c, d FROM test_gc_read`)
+	result.Check(testkit.Rows(`<nil> <nil>`, `3 2`, `7 12`, `14 45`))
+
+	// Test order of on duplicate key update list.
+	tk.MustExec(`INSERT INTO test_gc_read (a, b) VALUES (5, 8) ON DUPLICATE KEY UPDATE a = 6, b = a`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`)
+	result.Check(testkit.Rows(`0 <nil> <nil> <nil>`, `1 2 3 2`, `3 4 7 12`, `6 6 12 36`))
+
+	tk.MustExec(`INSERT INTO test_gc_read (a, b) VALUES (6, 8) ON DUPLICATE KEY UPDATE b = 8, a = b`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`)
+	result.Check(testkit.Rows(`0 <nil> <nil> <nil>`, `1 2 3 2`, `3 4 7 12`, `8 8 16 64`))
+
+	// Test where-conditions on virtual/stored generated columns.
+	result = tk.MustQuery(`SELECT * FROM test_gc_read WHERE c = 7`)
+	result.Check(testkit.Rows(`3 4 7 12`))
+
+	result = tk.MustQuery(`SELECT * FROM test_gc_read WHERE d = 64`)
+	result.Check(testkit.Rows(`8 8 16 64`))
+
+	// Test update where-conditions on virtual/generated columns.
+	tk.MustExec(`UPDATE test_gc_read SET a = a + 100 WHERE c = 7`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read WHERE c = 107`)
+	result.Check(testkit.Rows(`103 4 107 412`))
+
+	// Test update where-conditions on virtual/generated columns.
+	tk.MustExec(`UPDATE test_gc_read m SET m.a = m.a + 100 WHERE c = 107`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read WHERE c = 207`)
+	result.Check(testkit.Rows(`203 4 207 812`))
+
+	tk.MustExec(`UPDATE test_gc_read SET a = a - 200 WHERE d = 812`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read WHERE d = 12`)
+	result.Check(testkit.Rows(`3 4 7 12`))
+
+	// Test on-conditions on virtual/stored generated columns.
+	tk.MustExec(`CREATE TABLE test_gc_help(a int primary key, b int, c int, d int)`)
+	tk.MustExec(`INSERT INTO test_gc_help(a, b, c, d) SELECT * FROM test_gc_read`)
+
+	result = tk.MustQuery(`SELECT t1.* FROM test_gc_read t1 JOIN test_gc_help t2 ON t1.c = t2.c ORDER BY t1.a`)
+	result.Check(testkit.Rows(`1 2 3 2`, `3 4 7 12`, `8 8 16 64`))
+
+	result = tk.MustQuery(`SELECT t1.* FROM test_gc_read t1 JOIN test_gc_help t2 ON t1.d = t2.d ORDER BY t1.a`)
+	result.Check(testkit.Rows(`1 2 3 2`, `3 4 7 12`, `8 8 16 64`))
+
+	// Test generated column in subqueries.
+	result = tk.MustQuery(`SELECT * FROM test_gc_read t WHERE t.a not in (SELECT t.a FROM test_gc_read t where t.c > 5)`)
+	result.Check(testkit.Rows(`0 <nil> <nil> <nil>`, `1 2 3 2`))
+
+	result = tk.MustQuery(`SELECT * FROM test_gc_read t WHERE t.c in (SELECT t.c FROM test_gc_read t where t.c > 5)`)
+	result.Check(testkit.Rows(`3 4 7 12`, `8 8 16 64`))
+
+	result = tk.MustQuery(`SELECT tt.b FROM test_gc_read tt WHERE tt.a = (SELECT max(t.a) FROM test_gc_read t WHERE t.c = tt.c)`)
+	result.Check(testkit.Rows(`2`, `4`, `8`))
+
+	// Test aggregation on virtual/stored generated columns.
+	result = tk.MustQuery(`SELECT c, sum(a) aa, max(d) dd FROM test_gc_read GROUP BY c ORDER BY aa`)
+	result.Check(testkit.Rows(`<nil> 0 <nil>`, `3 1 2`, `7 3 12`, `16 8 64`))
+
+	result = tk.MustQuery(`SELECT a, sum(c), sum(d) FROM test_gc_read GROUP BY a ORDER BY a`)
+	result.Check(testkit.Rows(`0 <nil> <nil>`, `1 3 2`, `3 7 12`, `8 16 64`))
+
+	// Test multi-update on generated columns.
+	tk.MustExec(`UPDATE test_gc_read m, test_gc_read n SET m.a = m.a + 10, n.a = n.a + 10`)
+	result = tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`)
+	result.Check(testkit.Rows(`10 <nil> <nil> <nil>`, `11 2 13 22`, `13 4 17 52`, `18 8 26 144`))
+
+	// Test not null generated columns.
+	tk.MustExec(`CREATE TABLE test_gc_read_1(a int primary key, b int, c int as (a+b) not null, d int as (a*b) stored)`)
+	tk.MustExec(`CREATE TABLE test_gc_read_2(a int primary key, b int, c int as (a+b), d int as (a*b) stored not null)`)
+	tests := []struct {
+		stmt string
+		err  int
+	}{
+		// Can't insert these records, because generated columns are not null.
+		{`insert into test_gc_read_1(a, b) values (1, null)`, mysql.ErrBadNull},
+		{`insert into test_gc_read_2(a, b) values (1, null)`, mysql.ErrBadNull},
+	}
+	for _, tt := range tests {
+		_, err := tk.Exec(tt.stmt)
+		if tt.err != 0 {
+			c.Assert(err, NotNil)
+			terr := errors.Trace(err).(*errors.Err).Cause().(*terror.Error)
+			c.Assert(terr.Code(), Equals, terror.ErrCode(tt.err))
+		} else {
+			c.Assert(err, IsNil)
+		}
+	}
+}
+
 func (s *testSuite) TestToPBExpr(c *C) {
 	defer func() {
 		s.cleanEnv(c)

executor/show.go

@@ -397,7 +397,7 @@ func (e *ShowExec) fetchShowCreateTable() error {
 	var pkCol *table.Column
 	for i, col := range tb.Cols() {
 		buf.WriteString(fmt.Sprintf("  `%s` %s", col.Name.O, col.GetTypeDesc()))
-		if len(col.GeneratedExprString) != 0 {
+		if col.IsGenerated() {
 			// It's a generated column.
 			buf.WriteString(fmt.Sprintf(" GENERATED ALWAYS AS (%s)", col.GeneratedExprString))
 			if col.GeneratedStored {

executor/write.go

@@ -638,6 +638,9 @@ type InsertValues struct {
 	Lists     [][]expression.Expression
 	Setlist   []*expression.Assignment
 	IsPrepare bool
+
+	GenColumns []*ast.ColumnName
+	GenExprs   []expression.Expression
 }
 
 // InsertExec represents an insert executor.
@@ -763,30 +766,32 @@ func (e *InsertValues) getColumns(tableCols []*table.Column) ([]*table.Column, e
 		for _, v := range e.Setlist {
 			columns = append(columns, v.Col.ColName.O)
 		}
-
+		for _, v := range e.GenColumns {
+			columns = append(columns, v.Name.O)
+		}
 		cols, err = table.FindCols(tableCols, columns)
 		if err != nil {
 			return nil, errors.Errorf("INSERT INTO %s: %s", e.Table.Meta().Name.O, err)
 		}
-
 		if len(cols) == 0 {
 			return nil, errors.Errorf("INSERT INTO %s: empty column", e.Table.Meta().Name.O)
 		}
-	} else {
+	} else if len(e.Columns) > 0 {
 		// Process `name` type column.
 		columns := make([]string, 0, len(e.Columns))
 		for _, v := range e.Columns {
 			columns = append(columns, v.Name.O)
 		}
+		for _, v := range e.GenColumns {
+			columns = append(columns, v.Name.O)
+		}
 		cols, err = table.FindCols(tableCols, columns)
 		if err != nil {
 			return nil, errors.Errorf("INSERT INTO %s: %s", e.Table.Meta().Name.O, err)
 		}
-
-		// If cols are empty, use all columns instead.
-		if len(cols) == 0 {
-			cols = tableCols
-		}
+	} else {
+		// If e.Columns are empty, use all columns instead.
+		cols = tableCols
 	}
 
 	// Check column whether is specified only once.
@@ -812,7 +817,7 @@ func (e *InsertValues) fillValueList() error {
 	return nil
 }
 
-func (e *InsertValues) checkValueCount(insertValueCount, valueCount, num int, cols []*table.Column) error {
+func (e *InsertValues) checkValueCount(insertValueCount, valueCount, genColsCount int, num int, cols []*table.Column) error {
 	// TODO: This check should be done in plan builder.
 	if insertValueCount != valueCount {
 		// "insert into t values (), ()" is valid.
@@ -825,8 +830,18 @@ func (e *InsertValues) checkValueCount(insertValueCount, valueCount, num int, co
 	if valueCount == 0 && len(e.Columns) > 0 {
 		// "insert into t (c1) values ()" is not valid.
 		return ErrWrongValueCountOnRow.GenByArgs(num + 1)
-	} else if valueCount > 0 && valueCount != len(cols) {
-		return ErrWrongValueCountOnRow.GenByArgs(num + 1)
+	} else if valueCount > 0 {
+		explicitSetLen := 0
+		if len(e.Columns) != 0 {
+			explicitSetLen = len(e.Columns)
+		} else {
+			explicitSetLen = len(e.Setlist)
+		}
+		if explicitSetLen > 0 && valueCount+genColsCount != len(cols) {
+			return ErrWrongValueCountOnRow.GenByArgs(num + 1)
+		} else if explicitSetLen == 0 && valueCount != len(cols) {
+			return ErrWrongValueCountOnRow.GenByArgs(num + 1)
+		}
 	}
 	return nil
 }
@@ -840,7 +855,7 @@ func (e *InsertValues) getRows(cols []*table.Column) (rows [][]types.Datum, err
 	rows = make([][]types.Datum, len(e.Lists))
 	length := len(e.Lists[0])
 	for i, list := range e.Lists {
-		if err = e.checkValueCount(length, len(list), i, cols); err != nil {
+		if err = e.checkValueCount(length, len(list), len(e.GenColumns), i, cols); err != nil {
 			return nil, errors.Trace(err)
 		}
 		e.currRow = int64(i)
@@ -903,6 +918,15 @@ func (e *InsertValues) fillRowData(cols []*table.Column, vals []types.Datum, ign
 	if err = table.CastValues(e.ctx, row, cols, ignoreErr); err != nil {
 		return nil, errors.Trace(err)
 	}
+	for i, expr := range e.GenExprs {
+		var val types.Datum
+		val, err = expr.Eval(row)
+		if err = e.filterErr(err, ignoreErr); err != nil {
+			return nil, errors.Trace(err)
+		}
+		offset := cols[len(vals)+i].Offset
+		row[offset] = val
+	}
 	if err = table.CheckNotNull(e.Table.Cols(), row); err != nil {
 		return nil, errors.Trace(err)
 	}
@@ -934,7 +958,9 @@ func (e *InsertValues) initDefaultValues(row []types.Datum, hasValue []bool, ign
 			needDefaultValue = true
 			// TODO: Append Warning ErrColumnCantNull.
 		}
-		if mysql.HasAutoIncrementFlag(c.Flag) {
+		if mysql.HasAutoIncrementFlag(c.Flag) || c.IsGenerated() {
+			// Just leave generated column as null. It will be calculated later
+			// but before we check whether the column can be null or not.
 			needDefaultValue = false
 		}
 		if needDefaultValue {

expression/schema.go

@@ -74,7 +74,17 @@ func (s *Schema) Clone() *Schema {
 	for id, cols := range s.TblID2Handle {
 		schema.TblID2Handle[id] = make([]*Column, 0, len(cols))
 		for _, col := range cols {
-			schema.TblID2Handle[id] = append(schema.TblID2Handle[id], col.Clone().(*Column))
+			var inColumns = false
+			for i, colInColumns := range s.Columns {
+				if col == colInColumns {
+					schema.TblID2Handle[id] = append(schema.TblID2Handle[id], schema.Columns[i])
+					inColumns = true
+					break
+				}
+			}
+			if !inColumns {
+				schema.TblID2Handle[id] = append(schema.TblID2Handle[id], col.Clone().(*Column))
+			}
 		}
 	}
 	return schema

model/model.go

@@ -78,6 +78,11 @@ func (c *ColumnInfo) Clone() *ColumnInfo {
 	return &nc
 }
 
+// IsGenerated returns true if the column is generated column.
+func (c *ColumnInfo) IsGenerated() bool {
+	return len(c.GeneratedExprString) != 0
+}
+
 // ExtraHandleID is the column ID of column which we need to append to schema to occupy the handle's position
 // for use of execution phase.
 const ExtraHandleID = -1