feat: UUID backend changes (#815)

* feat: uuid changes

* changes

* linting

* test cases

* fixes

tests

tests

tests

test fix

* fixes

* changes

* changes

* changes

* removing-ui-changes

* comment changes

* session file compatibility

* fix tests

common/constants/constants.go

@@ -101,6 +101,8 @@ const (
 	// Metadata table names
 	SMT_JOB_TABLE      string = "SMT_JOB"
 	SMT_RESOURCE_TABLE string = "SMT_RESOURCE"
+	// Auto Generated Keys
+	UUID string = "UUID"
 	// Default gcs path of the Dataflow template.
 	DEFAULT_TEMPLATE_PATH string = "gs://dataflow-templates/latest/flex/Cloud_Datastream_to_Spanner"
 )

internal/convert.go

@@ -223,27 +223,26 @@ type MonitoringResources struct {
 type ShardResources struct {
 	DatastreamResources DatastreamResources
 	PubsubResources     PubsubResources
-	DataflowResources	DataflowResources
-	GcsResources		GcsResources
+	DataflowResources   DataflowResources
+	GcsResources        GcsResources
 	MonitoringResources MonitoringResources
 }
 
-
 // Stores information related to the streaming migration process.
 type streamingStats struct {
-	Streaming                     bool                        // Flag for confirmation of streaming migration.
-	TotalRecords                  map[string]map[string]int64 // Tablewise count of records received for processing, broken down by record type i.e. INSERT, MODIFY & REMOVE.
-	BadRecords                    map[string]map[string]int64 // Tablewise count of records not converted successfully, broken down by record type.
-	DroppedRecords                map[string]map[string]int64 // Tablewise count of records successfully converted but failed to written on Spanner, broken down by record type.
-	SampleBadRecords              []string                    // Records that generated errors during conversion.
-	SampleBadWrites               []string                    // Records that faced errors while writing to Cloud Spanner.
-	DatastreamResources           DatastreamResources
-	DataflowResources             DataflowResources
-	PubsubResources               PubsubResources
-	GcsResources                  GcsResources
-	MonitoringResources           MonitoringResources
-	ShardToShardResourcesMap      map[string]ShardResources
-	AggMonitoringResources        MonitoringResources
+	Streaming                bool                        // Flag for confirmation of streaming migration.
+	TotalRecords             map[string]map[string]int64 // Tablewise count of records received for processing, broken down by record type i.e. INSERT, MODIFY & REMOVE.
+	BadRecords               map[string]map[string]int64 // Tablewise count of records not converted successfully, broken down by record type.
+	DroppedRecords           map[string]map[string]int64 // Tablewise count of records successfully converted but failed to written on Spanner, broken down by record type.
+	SampleBadRecords         []string                    // Records that generated errors during conversion.
+	SampleBadWrites          []string                    // Records that faced errors while writing to Cloud Spanner.
+	DatastreamResources      DatastreamResources
+	DataflowResources        DataflowResources
+	PubsubResources          PubsubResources
+	GcsResources             GcsResources
+	MonitoringResources      MonitoringResources
+	ShardToShardResourcesMap map[string]ShardResources
+	AggMonitoringResources   MonitoringResources
 }
 
 type PubsubCfg struct {
@@ -439,7 +438,7 @@ func (conv *Conv) AddShardIdColumn() {
 			colName := conv.buildColumnNameWithBase(t, ShardIdColumn)
 			columnId := GenerateColumnId()
 			ct.ColIds = append(ct.ColIds, columnId)
-			ct.ColDefs[columnId] = ddl.ColumnDef{Name: colName, Id: columnId, T: ddl.Type{Name: ddl.String, Len: 50}, NotNull: false}
+			ct.ColDefs[columnId] = ddl.ColumnDef{Name: colName, Id: columnId, T: ddl.Type{Name: ddl.String, Len: 50}, NotNull: false, AutoGen: ddl.AutoGenCol{Name: "", GenerationType: ""}}
 			ct.ShardIdColumn = columnId
 			conv.SpSchema[t] = ct
 			var issues []SchemaIssue
@@ -475,7 +474,7 @@ func (conv *Conv) AddPrimaryKeys() {
 				k := conv.buildColumnNameWithBase(t, SyntheticPrimaryKey)
 				columnId := GenerateColumnId()
 				ct.ColIds = append(ct.ColIds, columnId)
-				ct.ColDefs[columnId] = ddl.ColumnDef{Name: k, Id: columnId, T: ddl.Type{Name: ddl.String, Len: 50}}
+				ct.ColDefs[columnId] = ddl.ColumnDef{Name: k, Id: columnId, T: ddl.Type{Name: ddl.String, Len: 50}, AutoGen: ddl.AutoGenCol{Name: "", GenerationType: ""}}
 				ct.PrimaryKeys = []ddl.IndexKey{{ColId: columnId, Order: 1}}
 				conv.SyntheticPKeys[t] = SyntheticPKey{columnId, 0}
 				addMissingPrimaryKeyWarning(ct.Id, columnId, conv, MissingPrimaryKey)

schema/schema.go

@@ -47,11 +47,11 @@ type Table struct {
 // Column represents a database column.
 // TODO: add support for foreign keys.
 type Column struct {
-	Name    string
-	Type    Type
-	NotNull bool
-	Ignored Ignored
-	Id      string
+	Name    	string
+	Type    	Type
+	NotNull 	bool
+	Ignored 	Ignored
+	Id      	string
 }
 
 // ForeignKey represents a foreign key.
@@ -119,6 +119,12 @@ type Ignored struct {
 	AutoIncrement bool
 }
 
+type AutoGenCol struct {
+	Name string
+	// Type of autogenerated column, example, pre-defined(uuid) or user-defined(sequence) 
+	GenerationType string
+}
+
 // Print converts ty to a string suitable for printing.
 func (ty Type) Print() string {
 	s := ty.Name

sources/common/toddl.go

@@ -131,6 +131,10 @@ func (ss *SchemaToSpannerImpl) SchemaToSpannerDDLHelper(conv *internal.Conv, tod
 			NotNull: isNotNull,
 			Comment: "From: " + quoteIfNeeded(srcCol.Name) + " " + srcCol.Type.Print(),
 			Id:      srcColId,
+			AutoGen: ddl.AutoGenCol{
+				Name: "",
+				GenerationType: "",
+			},
 		}
 		if !checkIfColumnIsPartOfPK(srcColId, srcTable.PrimaryKeys) {
 			totalNonKeyColumnSize += getColumnSize(ty.Name, ty.Len)

spanner/ddl/ast.go

@@ -179,6 +179,7 @@ type ColumnDef struct {
 	NotNull bool
 	Comment string
 	Id      string
+	AutoGen AutoGenCol
 }
 
 // Config controls how AST nodes are printed (aka unparsed).
@@ -231,8 +232,10 @@ func (cd ColumnDef) PrintColumnDef(c Config) (string, string) {
 	var s string
 	if c.SpDialect == constants.DIALECT_POSTGRESQL {
 		s = fmt.Sprintf("%s %s", c.quote(cd.Name), cd.T.PGPrintColumnDefType())
+		s += cd.AutoGen.PGPrintAutoGenCol()
 	} else {
 		s = fmt.Sprintf("%s %s", c.quote(cd.Name), cd.T.PrintColumnDefType())
+		s += cd.AutoGen.PrintAutoGenCol()
 	}
 	if cd.NotNull {
 		s += " NOT NULL"
@@ -375,6 +378,26 @@ type CreateIndex struct {
 	// interleaving clauses yet, so we omit them for now.
 }
 
+type AutoGenCol struct {
+	Name string
+	// Type of autogenerated column, example, pre-defined(uuid) or user-defined(sequence)
+	GenerationType string
+}
+
+func (agc AutoGenCol) PrintAutoGenCol() string {
+	if agc.Name == constants.UUID && agc.GenerationType == "Pre-defined" {
+		return " DEFAULT (GENERATE_UUID())"
+	}
+	return ""
+}
+
+func (agc AutoGenCol) PGPrintAutoGenCol() string {
+	if agc.Name == constants.UUID && agc.GenerationType == "Pre-defined" {
+		return " DEFAULT (spanner.generate_uuid())"
+	}
+	return ""
+}
+
 // PrintCreateIndex unparses a CREATE INDEX statement.
 func (ci CreateIndex) PrintCreateIndex(ct CreateTable, c Config) string {
 	var keys []string
@@ -400,7 +423,7 @@ func (ci CreateIndex) PrintCreateIndex(ct CreateTable, c Config) string {
 	if ci.StoredColumnIds != nil {
 		storedColumns := []string{}
 		for _, colId := range ci.StoredColumnIds {
-			if (!isStoredColumnKeyPartOfPrimaryKey(ct, colId)) {
+			if !isStoredColumnKeyPartOfPrimaryKey(ct, colId) {
 				storedColumns = append(storedColumns, c.quote(ct.ColDefs[colId].Name))
 			}
 		}

spanner/ddl/ast_test.go

@@ -408,6 +408,32 @@ func TestPrintForeignKeyAlterTable(t *testing.T) {
 	}
 }
 
+func TestPrintAutoGenCol(t *testing.T) {
+	tests := []struct {
+		agc      AutoGenCol
+		expected string
+	}{
+		{AutoGenCol{Name: constants.UUID, GenerationType: "Pre-defined"}, " DEFAULT (GENERATE_UUID())"},
+		{AutoGenCol{GenerationType: "", Name: ""}, ""},
+	}
+	for _, tc := range tests {
+		assert.Equal(t, tc.expected, tc.agc.PrintAutoGenCol())
+	}
+}
+
+func TestPGPrintAutoGenCol(t *testing.T) {
+	tests := []struct {
+		agc      AutoGenCol
+		expected string
+	}{
+		{AutoGenCol{Name: constants.UUID, GenerationType: "Pre-defined"}, " DEFAULT (spanner.generate_uuid())"},
+		{AutoGenCol{GenerationType: "", Name: ""}, ""},
+	}
+	for _, tc := range tests {
+		assert.Equal(t, tc.expected, tc.agc.PGPrintAutoGenCol())
+	}
+}
+
 func TestGetDDL(t *testing.T) {
 	s := Schema{
 		"t1": CreateTable{

ui/src/app/components/object-detail/object-detail.component.html

@@ -169,8 +169,7 @@ <h3 class="title">
                       EDIT
                     </button>
                   </div>
-                  <!--[TODO:khajanchi] Remove the predicate && false once transformations is in place-->
-                  <div *ngIf="!isEditMode && !currentObject.isDeleted && false">
+                  <div *ngIf="!isEditMode && !currentObject.isDeleted">
                     <button mat-stroked-button color="primary" (click)="addNewColumn()"
                       *ngIf="currentObject!.isSpannerNode">
                       <mat-icon class="edit-icon">edit</mat-icon>

ui/src/app/services/fetch/fetch.service.ts

@@ -173,6 +173,10 @@ export class FetchService {
     return this.http.get(`${this.url}/typemap`)
   }
 
+  getAutoGenMap() {
+    return this.http.get(`${this.url}/autoGenMap`)
+  }
+
   getSpannerDefaultTypeMap() {
     return this.http.get(`${this.url}/spannerDefaultTypeMap`)
   }

webv2/api/schema.go

@@ -42,6 +42,8 @@ var postgresTypeMap = make(map[string][]types.TypeIssue)
 var sqlserverTypeMap = make(map[string][]types.TypeIssue)
 var oracleTypeMap = make(map[string][]types.TypeIssue)
 
+var autoGenMap = make(map[string][]types.AutoGen)
+
 func init() {
 	sessionState := session.GetSessionState()
 	utilities.InitObjectId()
@@ -335,6 +337,23 @@ func GetTypeMap(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(filteredTypeMap)
 }
 
+func GetAutoGenMap(w http.ResponseWriter, r *http.Request) {
+
+	sessionState := session.GetSessionState()
+	if sessionState.Conv == nil || sessionState.Driver == "" {
+		http.Error(w, fmt.Sprintf("Schema is not converted or Driver is not configured properly. Please retry converting the database to Spanner."), http.StatusNotFound)
+		return
+	}
+	sessionState.Conv.ConvLock.Lock()
+	defer sessionState.Conv.ConvLock.Unlock()
+	switch sessionState.Driver {
+	case constants.MYSQL:
+		initializeAutoGenMap()
+	}
+	w.WriteHeader(http.StatusOK)
+	json.NewEncoder(w).Encode(autoGenMap)
+}
+
 // GetTableWithErrors checks the errors in the spanner schema
 // and returns a list of tables with errors
 func GetTableWithErrors(w http.ResponseWriter, r *http.Request) {
@@ -1534,3 +1553,48 @@ func addShardIdToForeignKeyPerTable(isAddedAtFirst bool, table ddl.CreateTable)
 		sessionState.Conv.SpSchema[table.Id].ForeignKeys[i] = fk
 	}
 }
+
+func initializeAutoGenMap() {
+	sessionState := session.GetSessionState()
+	autoGenMap = make(map[string][]types.AutoGen)
+	switch sessionState.Conv.SpDialect {
+	case constants.DIALECT_POSTGRESQL:
+		makePostgresDialectAutoGenMap()
+		return
+	default:
+		makeGoogleSqlDialectAutoGenMap()
+		return
+	}
+}
+
+func makePostgresDialectAutoGenMap() {
+	for _, srcTypeName := range []string{ddl.Bool, ddl.Date, ddl.Float64, ddl.Int64, ddl.PGBytea, ddl.PGFloat8, ddl.PGInt8, ddl.PGJSONB, ddl.PGTimestamptz, ddl.PGVarchar, ddl.Numeric} {
+		autoGenMap[srcTypeName] = []types.AutoGen{
+			{
+				Name: "",
+				GenerationType: "",
+			},
+		}
+	}
+	autoGenMap[ddl.PGVarchar] = append(autoGenMap[ddl.PGVarchar],
+		types.AutoGen{
+			Name: "UUID",
+			GenerationType: "Pre-defined",
+		})
+}
+
+func makeGoogleSqlDialectAutoGenMap() {
+	for _, srcTypeName := range []string{ddl.Bool, ddl.Bytes, ddl.Date, ddl.Float64, ddl.Int64, ddl.String, ddl.Timestamp, ddl.Numeric, ddl.JSON} {
+		autoGenMap[srcTypeName] = []types.AutoGen{
+			{
+				Name: "",
+				GenerationType: "",
+			},
+		}
+	}
+	autoGenMap[ddl.String] = append(autoGenMap[ddl.String],
+		types.AutoGen{
+			Name: "UUID",
+			GenerationType: "Pre-defined",
+		})
+}