diff --git a/README.md b/README.md index b2e9fa9..383285c 100644 --- a/README.md +++ b/README.md @@ -130,9 +130,9 @@ For more examples how to use the tool, please check out the [wiki](https://githu # Known Issues -1. If you have a unique index on a foreign key column then there are chance the constraint creation would fail, since mockd doesn't pick up unique value for foriegn key value it picks up random values from the reference table. -2. CHECK constraints isn't supported , so recreating check constraints would fail, use `custom` subcommand to control the data being inserted -3. On Greenplum Datbase tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints +1. If you have a composite unique index where one column is part of foreign key column then there are chances the constraint creation would fail. +2. Fixing CHECK constraints isn't supported due to complexity, so recreating check constraints would fail, use `custom` subcommand to control the data being inserted +3. On Greenplum Database partition tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints 4. Custom data types are not supported, use `custom` sub command to control the data for that custom data types # Developers / Collaboration diff --git a/cmd.go b/cmd.go index f166a77..db7fa6c 100644 --- a/cmd.go +++ b/cmd.go @@ -132,6 +132,10 @@ var tablesCmd = &cobra.Command{ if IsStringEmpty(cmdOptions.Tab.SchemaName) { Fatalf("Cannot have the schema name empty, please check the arguments") } + // If number of columns is greater than the limit, then error out + if cmdOptions.Tab.MaxColumns > 1600 { + Fatalf("Postgres cannot have more than 1600 columns, check the arguments") + } }, PostRun: func(cmd *cobra.Command, args []string) { Info("Successfully completed running the table sub command") diff --git a/constraintsBackup.go b/constraintsBackup.go index 65f7d9f..3fc54ed 100644 --- a/constraintsBackup.go +++ b/constraintsBackup.go @@ -103,9 +103,9 @@ func RemoveConstraints(table string) { for _, c := range constraints { // Generate the DROP DDL command if c.Constrainttype == "index" { // if the constraint is a index - statement = fmt.Sprintf("DROP INDEX %s CASCADE;", c.Constraintname) + statement = fmt.Sprintf("DROP INDEX \"%s\" CASCADE;", c.Constraintname) } else { // if the constraint is a constraint - statement = fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT %s CASCADE;", table, c.Constraintname) + statement = fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT \"%s\" CASCADE;", table, c.Constraintname) } // Execute the statement diff --git a/constraintsRestore.go b/constraintsRestore.go index 99a0925..1319f56 100644 --- a/constraintsRestore.go +++ b/constraintsRestore.go @@ -32,7 +32,7 @@ func FixConstraints() { case v == "UNIQUE": // Run the same logic as primary key fixPKey(con) //case v == "CHECK": // TODO: Its hard to predict the check constraint ATM - // fixCheck(db, con) + // fixCheck(db, con) case v == "FOREIGN": fixFKey(con) } @@ -196,12 +196,17 @@ func recreateAllConstraints() { _, err := ExecuteDB(content) if err != nil && !IgnoreErrorString(fmt.Sprintf("%s", err)) { Debugf("Error creating constraint %s, err: %v", content, err) - err = WriteToFile(failedConstraintsFile, content+"\n") - if err != nil { - Fatalf("Error when saving the failed restore to file %s, err %v", - failedConstraintsFile, err) + // Try an attempt to recreate constraint again after deleting the + // violating row + successOrFailure := deleteViolatingPkOrUkConstriants(content) + if !successOrFailure { // didn't succeed, ask the user to fix it manually + err = WriteToFile(failedConstraintsFile, content+"\n") + if err != nil { + Fatalf("Error when saving the failed restore to file %s, err %v", + failedConstraintsFile, err) + } + AnyError = true } - AnyError = true } bar.Add(1) } @@ -210,7 +215,39 @@ func recreateAllConstraints() { } if AnyError { - Warnf("There have been issue creating few constraints, all the "+ - "constraints that failed has been saved on to file: %s", failedConstraintsFile) + Warnf("There have been issue creating few constraints and would need manual cleanup at your end, "+ + "all the constraints that failed has been saved on to file: %s", failedConstraintsFile) } } + +// we tried to fix the primary key violation, but due to the nature +// of how we fix the constraints like PK (or UK) followed by FK , there +// are chances that we might inject duplicate keys again, for eg.s if +// there is a PK ( or UK ) on a FK reference table. so the aim here +// is, we will delete the rows that violates it and hoping that it will help in +// recreating the constraints. Yes we will loose that row at least that help to +// recreate constraints ( fingers crossed :) ) +func deleteViolatingPkOrUkConstriants(con string) bool { + Debugf("Attempting to run the constraint command %s second time, after deleting violating rows", con) + // does the DDL contain PK or UK keyword then do the following + // rest send them back for user to fix it. + if isSubStringAvailableOnString(con, "ADD CONSTRAINT.*PRIMARY KEY|ADD CONSTRAINT.*UNIQUE") { + column, _ := ColExtractor(con, `\(.*?\)`) + table, _ := ColExtractor(con, `ALTER TABLE(.*)ADD CONSTRAINT`) + table = strings.Trim(strings.Trim(table, "ALTER TABLE"), "ADD CONSTRAINT") + column = strings.Trim(column, "()") + err := deleteViolatingConstraintKeys(table, column) + if err != nil { // we failed to delete the the constraint violation rows + Debugf("Error when deleting rows from the constraint violation rows: %v", err) + return false + } + _, err = ExecuteDB(con) // retry to create the constraint again + if err != nil { // we failed to recreate the constraint + Debugf("Error when 2nd attempt to recreate constraint: %v", err) + return false + } + // successfully cleaned it up + return true + } + return false +} diff --git a/helpers.go b/helpers.go index 7cd0655..cebc585 100644 --- a/helpers.go +++ b/helpers.go @@ -209,6 +209,15 @@ func BracketsExists(dt string) bool { } } +// Does the string contain the substring +func isSubStringAvailableOnString(s string, criteria string) bool { + var re = regexp.MustCompile(fmt.Sprintf("%s", criteria)) + if re.MatchString(s) { + return true + } + return false +} + // Built a method to find if the values exits with a slice func StringContains(item string, slice []string) bool { set := make(map[string]struct{}, len(slice)) diff --git a/sql.go b/sql.go index 2f3b485..7c4f5db 100644 --- a/sql.go +++ b/sql.go @@ -244,9 +244,11 @@ func GetPGConstraintDDL(conntype string) []DBConstraints { Debugf("Extracting the DDL of the %s constraints", conntype) var result []DBConstraints query := ` -SELECT n.nspname - || '.' - || c.relname tablename, +SELECT '"' + || n.nspname + || '"."' + || c.relname + || '"' tablename, con.conname constraintname, pg_catalog.Pg_get_constraintdef(con.oid, true) constraintKey FROM pg_catalog.pg_class c, @@ -277,10 +279,12 @@ func GetPGIndexDDL() []DBIndex { Debugf("Extracting the unique indexes") var result []DBIndex query := ` -SELECT schemaname - || '.' - || tablename tablename, - indexdef indexdef +SELECT '"' + || schemaname + || '"."' + || tablename + || '"' tablename, + indexdef indexdef FROM pg_indexes WHERE schemaname IN (SELECT nspname FROM pg_namespace @@ -546,3 +550,25 @@ WHERE %[2]s = '%[6]s' Fatalf("Error when updating the foreign key for table %s, err: %v", key.Table, err) } } + +// Delete the violating key +func deleteViolatingConstraintKeys(tab string, column string) error { + Debugf("Deleting the rows the table that violates the constraints: %s:(%s)", tab, column ) + query := ` +DELETE +FROM %[1]s +WHERE ( + %[2]s) IN + ( + SELECT %[2]s + FROM %[1]s + GROUP BY %[2]s + HAVING count(*) > 1); +` + query = fmt.Sprintf(query, tab, column) + _, err := ExecuteDB(query) + if err != nil { + return err + } + return nil +} diff --git a/worker.go b/worker.go index dbbae22..0440738 100644 --- a/worker.go +++ b/worker.go @@ -180,7 +180,7 @@ func GenerateTableName(tab, schema string) string { // Throw warning if there is skipped tables func skipTablesWarning() { if len(skippedTab) > 0 { - Warnf("These tables are skipped since these datatypes are not supported by %s: %s", + Warnf("These tables are skipped since these data types are not supported by %s: %s", programName, strings.Join(skippedTab, ",")) } }