Adding random query generation for endtoend testing of the Gen4 plann…

…er (#13260)

go/test/endtoend/utils/cmp.go

@@ -233,6 +233,8 @@ func (mcmp *MySQLCompare) ExecWithColumnCompare(query string) *sqltypes.Result {
 //   - MySQL and Vitess did not find an error, but their results are matching
 //
 // The result set and error produced by Vitess are returned to the caller.
+// If the Vitess and MySQL error are both nil, but the results do not match,
+// the mismatched results are instead returned as an error, as well as the Vitess result set
 func (mcmp *MySQLCompare) ExecAllowAndCompareError(query string) (*sqltypes.Result, error) {
 	mcmp.t.Helper()
 	vtQr, vtErr := mcmp.VtConn.ExecuteFetch(query, 1000, true)
@@ -242,7 +244,7 @@ func (mcmp *MySQLCompare) ExecAllowAndCompareError(query string) (*sqltypes.Resu
 	// Since we allow errors, we don't want to compare results if one of the client failed.
 	// Vitess and MySQL should always be agreeing whether the query returns an error or not.
 	if vtErr == nil && mysqlErr == nil {
-		compareVitessAndMySQLResults(mcmp.t, query, mcmp.VtConn, vtQr, mysqlQr, false)
+		vtErr = compareVitessAndMySQLResults(mcmp.t, query, mcmp.VtConn, vtQr, mysqlQr, false)
 	}
 	return vtQr, vtErr
 }

go/test/endtoend/utils/mysql.go

@@ -18,6 +18,7 @@ package utils
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -154,24 +155,27 @@ func prepareMySQLWithSchema(params mysql.ConnParams, sql string) error {
 	return nil
 }
 
-func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn, vtQr, mysqlQr *sqltypes.Result, compareColumns bool) {
+func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn, vtQr, mysqlQr *sqltypes.Result, compareColumns bool) error {
 	if vtQr == nil && mysqlQr == nil {
-		return
+		return nil
 	}
 	if vtQr == nil {
 		t.Error("Vitess result is 'nil' while MySQL's is not.")
-		return
+		return errors.New("Vitess result is 'nil' while MySQL's is not.\n")
 	}
 	if mysqlQr == nil {
 		t.Error("MySQL result is 'nil' while Vitess' is not.")
-		return
+		return errors.New("MySQL result is 'nil' while Vitess' is not.\n")
 	}
+
+	var errStr string
 	if compareColumns {
 		vtColCount := len(vtQr.Fields)
 		myColCount := len(mysqlQr.Fields)
 		if vtColCount > 0 && myColCount > 0 {
 			if vtColCount != myColCount {
 				t.Errorf("column count does not match: %d vs %d", vtColCount, myColCount)
+				errStr += fmt.Sprintf("column count does not match: %d vs %d\n", vtColCount, myColCount)
 			}
 
 			var vtCols []string
@@ -180,26 +184,27 @@ func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn
 				vtCols = append(vtCols, vtField.Name)
 				myCols = append(myCols, mysqlQr.Fields[i].Name)
 			}
-			assert.Equal(t, myCols, vtCols, "column names do not match - the expected values are what mysql produced")
+			if !assert.Equal(t, myCols, vtCols, "column names do not match - the expected values are what mysql produced") {
+				errStr += "column names do not match - the expected values are what mysql produced\n"
+				errStr += fmt.Sprintf("Not equal: \nexpected: %v\nactual: %v\n", myCols, vtCols)
+			}
 		}
 	}
 	stmt, err := sqlparser.Parse(query)
 	if err != nil {
 		t.Error(err)
-		return
+		return err
 	}
 	orderBy := false
 	if selStmt, isSelStmt := stmt.(sqlparser.SelectStatement); isSelStmt {
 		orderBy = selStmt.GetOrderBy() != nil
 	}
 
-	if orderBy && sqltypes.ResultsEqual([]sqltypes.Result{*vtQr}, []sqltypes.Result{*mysqlQr}) {
-		return
-	} else if sqltypes.ResultsEqualUnordered([]sqltypes.Result{*vtQr}, []sqltypes.Result{*mysqlQr}) {
-		return
+	if (orderBy && sqltypes.ResultsEqual([]sqltypes.Result{*vtQr}, []sqltypes.Result{*mysqlQr})) || sqltypes.ResultsEqualUnordered([]sqltypes.Result{*vtQr}, []sqltypes.Result{*mysqlQr}) {
+		return nil
 	}
 
-	errStr := "Query (" + query + ") results mismatched.\nVitess Results:\n"
+	errStr += "Query (" + query + ") results mismatched.\nVitess Results:\n"
 	for _, row := range vtQr.Rows {
 		errStr += fmt.Sprintf("%s\n", row)
 	}
@@ -212,6 +217,7 @@ func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn
 		errStr += fmt.Sprintf("query plan: \n%s\n", qr.Rows[0][0].ToString())
 	}
 	t.Error(errStr)
+	return errors.New(errStr)
 }
 
 func compareVitessAndMySQLErrors(t *testing.T, vtErr, mysqlErr error) {