Add support for MySQL Cluster (NDB)

As defined in Issue #20

.build/COMPATIBLE_VERSION

@@ -1 +1 @@
-1.21.0
+1.23.0

.build/VERSION

@@ -1 +1 @@
-1.22.0
+1.23.0

.build/create-version-source-file.go

@@ -71,7 +71,8 @@ func main() {
 		"CompatibleVersionDate": compatibleVersionDate,
 		"Timestamp":             time.Now().Format("2006-01-02 15:04"),
 	}
-	versionCode := common.TemplateFill(template, data)
+	versionCode, err := common.SafeTemplateFill("create-version-source-file", template, data)
+	common.ErrCheckExitf(err, 1, "error filling version code template %s", err)
 
 	err = common.WriteString(versionCode, versionDestFile)
 	common.ErrCheckExitf(err, 1, "error writing version code file %s", versionDestFile)

abbreviations/abbreviations.go

@@ -146,7 +146,9 @@ func LoadAbbreviations() {
 			for _, item := range abbreviations[arg] {
 				if item != "" {
 					// Replaces possible vars with their value
-					item = common.TemplateFill(item, variables)
+
+					item, err = common.SafeTemplateFill("loadAbbreviations func", item, variables)
+					common.ErrCheckExitf(err, 1, "error filling abbreviations template: %s", err)
 					// adds the replacement items to the new argument list
 					replacement += " " + item
 					newArgs = append(newArgs, item)

cmd/admin.go

@@ -102,7 +102,8 @@ func preserveSandbox(sandboxDir, sandboxName string) {
 		"NoClearCmd":   noClearCmd,
 	}
 	template = common.TrimmedLines(template)
-	newClearMessage := common.TemplateFill(template, data)
+	newClearMessage, err := common.SafeTemplateFill("preserveSandbox func", template, data)
+	common.ErrCheckExitf(err, 1, "error filling preservation file: %s", err)
 	err = common.WriteString(newClearMessage, clear)
 	if err != nil {
 		common.Exitf(1, "%+v", err)

cmd/unpack.go

@@ -32,12 +32,12 @@ import (
 func detectTarballFlavor(tarballName string) string {
 	flavor := ""
 	flavorsRegexps := map[string]string{
-		common.MySQLFlavor:         `mysql`,
 		common.PerconaServerFlavor: `Percona-Server`,
 		common.MariaDbFlavor:       `mariadb`,
-		common.NDBFlavor:           `mysql-cluster`,
+		common.NdbFlavor:           `mysql-cluster`,
 		common.TiDbFlavor:          `tidb`,
 		common.PxcFlavor:           `Percona-XtraDB-Cluster`,
+		common.MySQLFlavor:         `mysql`,
 	}
 
 	for key, value := range flavorsRegexps {

common/capabilities.go

@@ -45,7 +45,7 @@ const (
 	MySQLFlavor         = "mysql"
 	PerconaServerFlavor = "percona"
 	MariaDbFlavor       = "mariadb"
-	NDBFlavor           = "ndb"
+	NdbFlavor           = "ndb"
 	PxcFlavor           = "pxc"
 	TiDbFlavor          = "tidb"
 
@@ -68,6 +68,7 @@ const (
 	NativeAuth       = "nativeAuth"
 	DataDict         = "datadict"
 	XtradbCluster    = "xtradbCluster"
+	NdbCluster       = "ndbCluster"
 	RootAuth         = "rootAuth"
 )
 
@@ -179,15 +180,17 @@ var FlavorCompositionList = []FlavorIndicator{
 		},
 		flavor: PxcFlavor,
 	},
-	//{
-	//	AllNeeded: true,
-	//	elements: []ElementPath{
-	//		{"bin", "ndbd"},
-	//		{"bin", "ndb_mgm"},
-	//		{"bin", "ndb_mgmd"},
-	//	},
-	//	flavor: NDBFlavor,
-	//},
+	{
+		AllNeeded: true,
+		elements: []ElementPath{
+			{"bin", "ndbd"},
+			{"bin", "ndb_mgm"},
+			{"bin", "ndb_mgmd"},
+			{"bin", "ndbd"},
+			{"bin", "ndbmtd"},
+		},
+		flavor: NdbFlavor,
+	},
 	{
 		AllNeeded: false,
 		elements: []ElementPath{
@@ -241,11 +244,19 @@ var TiDBCapabilities = Capabilities{
 		// No capabilities so far
 	},
 }
-var NDBCapabilities = Capabilities{
-	Flavor:      NDBFlavor,
+var NdbCapabilities = Capabilities{
+	Flavor:      NdbFlavor,
 	Description: "MySQL NDB Cluster",
-	Features:    FeatureList{
-		// No capabilities so far
+	Features: FeatureList{
+		DynVariables: MySQLCapabilities.Features[DynVariables],
+		Initialize:   MySQLCapabilities.Features[Initialize],
+		CreateUser:   MySQLCapabilities.Features[CreateUser],
+		Roles:        MySQLCapabilities.Features[Roles],
+		SetPersist:   MySQLCapabilities.Features[SetPersist],
+		NdbCluster: {
+			Description: "MySQL NDB Cluster",
+			Since:       globals.MinimumNdbClusterVersion,
+		},
 	},
 }
 
@@ -285,7 +296,7 @@ var AllCapabilities = map[string]Capabilities{
 	PerconaServerFlavor: PerconaCapabilities,
 	MariaDbFlavor:       MariadbCapabilities,
 	TiDbFlavor:          TiDBCapabilities,
-	NDBFlavor:           NDBCapabilities,
+	NdbFlavor:           NdbCapabilities,
 	PxcFlavor:           PxcCapabilities,
 }
 

common/tprintf.go

@@ -17,7 +17,9 @@ package common
 
 import (
 	"bytes"
+	"fmt"
 	"github.com/datacharmer/dbdeployer/globals"
+	"reflect"
 	"regexp"
 	"text/template"
 	"time"
@@ -42,6 +44,7 @@ func TrimmedLines(s string) string {
 // TemplateFill passed template string is formatted using its operands and returns the resulting string.
 // Spaces are added between operands when neither is a string.
 // Based on code from https://play.golang.org/p/COHKlB2RML
+// DEPRECATED: replaced by SafeTemplateFill
 func TemplateFill(tmpl string, data StringMap) string {
 
 	// Adds timestamp and version info
@@ -66,3 +69,78 @@ func TemplateFill(tmpl string, data StringMap) string {
 	// Returns the populated template
 	return buf.String()
 }
+
+// Returns true if a StringMap (or an inner StringMap) contains a given key
+func hasKey(sm StringMap, wantedKey string) bool {
+	for key, value := range sm {
+		if key == wantedKey {
+			return true
+		}
+		valType := reflect.TypeOf(value)
+		if valType == reflect.TypeOf(StringMap{}) {
+			innerSm := value.(StringMap)
+			return hasKey(innerSm, wantedKey)
+		}
+		if valType == reflect.TypeOf([]StringMap{}) {
+			innerSm := value.([]StringMap)
+			for _, ism := range innerSm {
+				if hasKey(ism, wantedKey) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// SafeTemplateFill passed template string is formatted using its operands and returns the resulting string.
+// It checks that the data was safely initialized
+func SafeTemplateFill(template_name, tmpl string, data StringMap) (string, error) {
+
+	// Adds timestamp, version info, and empty engine clause if one was not provided
+	timestamp := time.Now()
+	_, engineClauseExists := data["EngineClause"]
+	_, timeStampExists := data["DateTime"]
+	_, versionExists := data["AppVersion"]
+	if !timeStampExists {
+		data["DateTime"] = timestamp.Format(time.UnixDate)
+	}
+	if !versionExists {
+		data["AppVersion"] = VersionDef
+	}
+	if !engineClauseExists {
+		data["EngineClause"] = ""
+	}
+
+	// Checks that all data was initialized
+	// This check is especially useful when introducing new templates
+
+	/**/
+	// First, we get all variables in the pattern {{.VarName}}
+	reTemplateVar := regexp.MustCompile(`\{\{\.([^{]+)\}\}`)
+	varList := reTemplateVar.FindAllStringSubmatch(tmpl, -1)
+	if len(varList) > 0 {
+		for _, capture := range varList {
+			// For each variable in the template text, we look whether it is
+			// in the map
+			if !hasKey(data, capture[1]) {
+				//fmt.Printf("### >>> %#v<<<\n", data)
+				return globals.EmptyString,
+					fmt.Errorf("data field '%s' (intended for template '%s') was not initialized ",
+						capture[1], template_name)
+			}
+		}
+	}
+	/**/
+	// Creates a template
+	processTemplate := template.Must(template.New("tmp").Parse(tmpl))
+	buf := &bytes.Buffer{}
+
+	// If an error occurs, returns an empty string
+	if err := processTemplate.Execute(buf, data); err != nil {
+		return globals.EmptyString, err
+	}
+
+	// Returns the populated template
+	return buf.String(), nil
+}

common/tprintf_test.go

@@ -49,8 +49,8 @@ func TestTemplateFill(t *testing.T) {
 		},
 	}
 	for _, td := range dataCollection {
-		result := TemplateFill(template, td.data)
-		if result == td.expected {
+		result, err := SafeTemplateFill("tmpl1", template, td.data)
+		if err == nil && result == td.expected {
 			t.Logf("ok - string formatted as expected: '%s'\n", result)
 		} else {
 			t.Logf("not ok - Expected %s - found %s\n", td.expected, result)
@@ -60,8 +60,14 @@ func TestTemplateFill(t *testing.T) {
 	template = `{{.DateTime}}`
 	// The DateTime field is auto generated
 	data := StringMap{}
-	result := TemplateFill(template, data)
+	result, err := SafeTemplateFill("tmpl2", template, data)
+	compare.OkIsNil("filling template 2", err, t)
 	//             Sun Oct    7  07: 42: 24 CEST 2018
 	reExpected := `\w+ \w+\s+\d+ \d+:\d+:\d+ \w+ \d+`
 	compare.OkMatchesString("Timestamp", result, reExpected, t)
+	template = `{{.NoSuchVar}} {{.SuchVarExists}}`
+	data = StringMap{"SuchVarExists": "something"}
+	result, err = SafeTemplateFill("tmpl3", template, data)
+	compare.OkIsNotNil("filling template 3", err, t)
+	compare.OkMatchesString("filling template string", err.Error(), `NoSuchVar`, t)
 }

common/version.go

@@ -16,11 +16,11 @@
 package common
 
 // This file was generated during build. Do not edit.
-// Build time: 2019-03-07 08:32
+// Build time: 2019-03-13 07:08
 
-var VersionDef string = "1.22.0" // 2019-03-07
+var VersionDef string = "1.23.0" // 2019-03-13
 
 // Compatible version is the version used to mark compatible archives (templates, configuration).
 // It is usually major.minor.0, except when we are at version 0.x, when
 // every revision may bring incompatibility
-var CompatibleVersion string = "1.21.0" // 2019-03-05
+var CompatibleVersion string = "1.23.0" // 2019-03-13

defaults/defaults.go

@@ -41,8 +41,9 @@ type DbdeployerDefaults struct {
 	AllMastersReplicationBasePort int `json:"all-masters-replication-base-port"`
 	MultipleBasePort              int `json:"multiple-base-port"`
 	// GaleraBasePort                 int    `json:"galera-base-port"`
-	PxcBasePort int `json:"pxc-base-port"`
-	// NdbBasePort                    int    `json:"ndb-base-port"`
+	PxcBasePort       int    `json:"pxc-base-port"`
+	NdbBasePort       int    `json:"ndb-base-port"`
+	NdbClusterPort    int    `json:"ndb-cluster-port"`
 	GroupPortDelta    int    `json:"group-port-delta"`
 	MysqlXPortDelta   int    `json:"mysqlx-port-delta"`
 	MasterName        string `json:"master-name"`
@@ -62,7 +63,7 @@ type DbdeployerDefaults struct {
 	RemoteIndexFile   string `json:"remote-index-file"`
 	// GaleraPrefix                   string `json:"galera-prefix"`
 	PxcPrefix string `json:"pxc-prefix"`
-	// NdbPrefix                      string `json:"ndb-prefix"`
+	NdbPrefix string `json:"ndb-prefix"`
 	Timestamp string `json:"timestamp"`
 }
 
@@ -101,7 +102,8 @@ var (
 		MultipleBasePort:              16000,
 		PxcBasePort:                   18000,
 		// GaleraBasePort:                17000,
-		// NdbBasePort:                   19000,
+		NdbBasePort:       19000,
+		NdbClusterPort:    20000,
 		GroupPortDelta:    125,
 		MysqlXPortDelta:   10000,
 		MasterName:        "master",
@@ -120,7 +122,7 @@ var (
 		RemoteRepository:  "https://raw.githubusercontent.com/datacharmer/mysql-docker-minimal/master/dbdata",
 		RemoteIndexFile:   "available.json",
 		// GaleraPrefix:                  "galera_msb_",
-		// NdbPrefix:                     "ndb_msb_",
+		NdbPrefix: "ndb_msb_",
 		PxcPrefix: "pxc_msb_",
 		Timestamp: time.Now().Format(time.UnixDate),
 	}
@@ -210,7 +212,8 @@ func ValidateDefaults(nd DbdeployerDefaults) bool {
 		checkInt("all-masters-base-port", nd.AllMastersReplicationBasePort, minPortValue, maxPortValue) &&
 		// checkInt("galera-base-port", nd.GaleraBasePort, minPortValue, maxPortValue) &&
 		checkInt("pxc-base-port", nd.PxcBasePort, minPortValue, maxPortValue) &&
-		// checkInt("ndb-base-port", nd.NdbBasePort, minPortValue, maxPortValue) &&
+		checkInt("ndb-base-port", nd.NdbBasePort, minPortValue, maxPortValue) &&
+		checkInt("ndb-cluster-port", nd.NdbClusterPort, minPortValue, maxPortValue) &&
 		checkInt("group-port-delta", nd.GroupPortDelta, 101, 299) &&
 		checkInt("mysqlx-port-delta", nd.MysqlXPortDelta, 2000, 15000)
 	if !allInts {
@@ -222,7 +225,8 @@ func ValidateDefaults(nd DbdeployerDefaults) bool {
 		nd.MultipleBasePort != nd.MasterSlaveBasePort &&
 		nd.MultipleBasePort != nd.FanInReplicationBasePort &&
 		nd.MultipleBasePort != nd.AllMastersReplicationBasePort &&
-		// nd.MultipleBasePort != nd.NdbBasePort &&
+		nd.MultipleBasePort != nd.NdbBasePort &&
+		nd.MultipleBasePort != nd.NdbClusterPort &&
 		// nd.MultipleBasePort != nd.GaleraBasePort &&
 		nd.MultipleBasePort != nd.PxcBasePort &&
 		nd.MultiplePrefix != nd.GroupSpPrefix &&
@@ -232,7 +236,7 @@ func ValidateDefaults(nd DbdeployerDefaults) bool {
 		nd.MultiplePrefix != nd.FanInPrefix &&
 		nd.MultiplePrefix != nd.AllMastersPrefix &&
 		nd.MasterAbbr != nd.SlaveAbbr &&
-		// nd.MultiplePrefix != nd.NdbPrefix &&
+		nd.MultiplePrefix != nd.NdbPrefix &&
 		// nd.MultiplePrefix != nd.GaleraPrefix &&
 		nd.MultiplePrefix != nd.PxcPrefix &&
 		nd.SandboxHome != nd.SandboxBinary
@@ -253,7 +257,7 @@ func ValidateDefaults(nd DbdeployerDefaults) bool {
 		nd.MultiplePrefix != "" &&
 		nd.PxcPrefix != "" &&
 		// nd.GaleraPrefix != "" &&
-		//nd.NdbPrefix != "" &&
+		nd.NdbPrefix != "" &&
 		nd.SandboxHome != "" &&
 		nd.SandboxBinary != "" &&
 		nd.RemoteIndexFile != "" &&
@@ -327,8 +331,10 @@ func UpdateDefaults(label, value string, storeDefaults bool) {
 		newDefaults.FanInReplicationBasePort = common.Atoi(value)
 	case "all-masters-base-port":
 		newDefaults.AllMastersReplicationBasePort = common.Atoi(value)
-	// case "ndb-base-port":
-	//	 new_defaults.NdbBasePort = common.Atoi(value)
+	case "ndb-base-port":
+		newDefaults.NdbBasePort = common.Atoi(value)
+	case "ndb-cluster-port":
+		newDefaults.NdbClusterPort = common.Atoi(value)
 	// case "galera-base-port":
 	//	 new_defaults.GaleraBasePort = common.Atoi(value)
 	case "pxc-base-port":
@@ -371,8 +377,8 @@ func UpdateDefaults(label, value string, storeDefaults bool) {
 	// 	new_defaults.GaleraPrefix = value
 	case "pxc-prefix":
 		newDefaults.PxcPrefix = value
-	// case "ndb-prefix":
-	// 	new_defaults.NdbPrefix = value
+	case "ndb-prefix":
+		newDefaults.NdbPrefix = value
 	default:
 		common.Exitf(1, "unrecognized label %s", label)
 	}