Implement flavor and capabilities support

Implement Issue #49 Add support for --client-from=X
Implement Issue #50 Change evaluation of version to include a flavor
Fix Issue #51 unpack command fails when tarball name doesn't include a version
Implement Issue #52 unpack command should create a FLAVOR file for the extracted tarball
Add command "dbdeployer admin capabilities [flavor [version]]"

.build/COMPATIBLE_VERSION

@@ -1 +1 @@
-1.17.0
+1.18.0

.build/VERSION

@@ -1 +1 @@
-1.17.1
+1.18.0

README.md

@@ -3,7 +3,7 @@
 [DBdeployer](https://github.com/datacharmer/dbdeployer) is a tool that deploys MySQL database servers easily.
 This is a port of [MySQL-Sandbox](https://github.com/datacharmer/mysql-sandbox), originally written in Perl, and re-designed from the ground up in [Go](https://golang.org). See the [features comparison](https://github.com/datacharmer/dbdeployer/blob/master/docs/features.md) for more detail.
 
-Documentation updated for version 1.17.0 (15-Jan-2019 20:16 UTC)
+Documentation updated for version 1.18.0 (09-Feb-2019 14:28 UTC)
 
 [![Build Status](https://travis-ci.org/datacharmer/dbdeployer.svg "Travis CI status")](https://travis-ci.org/datacharmer/dbdeployer)
 
@@ -15,7 +15,7 @@ Get the one for your O.S. from [dbdeployer releases](https://github.com/datachar
 
 For example:
 
-    $ VERSION=v1.17.0
+    $ VERSION=v1.18.0
     $ OS=linux
     $ origin=https://github.com/datacharmer/dbdeployer/releases/tag/$VERSION
     $ wget $origin/dbdeployer-$VERSION.$OS.tar.gz
@@ -50,7 +50,7 @@ For example:
 The program doesn't have any dependencies. Everything is included in the binary. Calling *dbdeployer* without arguments or with ``--help`` will show the main help screen.
 
     $ dbdeployer --version
-    dbdeployer version 1.17.0
+    dbdeployer version 1.18.0
 
 
     $ dbdeployer -h
@@ -115,6 +115,7 @@ If you don't have any tarballs installed in your system, you should first ``unpa
     	
 
     Flags:
+          --flavor string           Defines the tarball flavor (MySQL, NDB, Percona Server, etc)
       -h, --help                    help for unpack
           --prefix string           Prefix for the final expanded directory
           --shell                   Unpack a shell tarball into the corresponding server directory
@@ -141,6 +142,7 @@ The easiest command is ``deploy single``, which installs a single sandbox.
           --base-port int                 Overrides default base-port (for multiple sandboxes)
           --binary-version string         Specifies the version when the basedir directory name does not contain it (i.e. it is not x.x.xx)
           --bind-address string           defines the database bind-address  (default "127.0.0.1")
+          --client-from string            Where to get the client binaries from
           --concurrent                    Runs multiple sandbox deployments concurrently
           --custom-mysqld string          Uses an alternative mysqld (must be in the same directory as regular mysqld)
       -p, --db-password string            database password (default "msandbox")
@@ -150,6 +152,7 @@ The easiest command is ``deploy single``, which installs a single sandbox.
           --enable-general-log            Enables general log for the sandbox (MySQL 5.1+)
           --enable-mysqlx                 Enables MySQLX plugin (5.7.12+)
           --expose-dd-tables              In MySQL 8.0+ shows data dictionary tables
+          --flavor string                 Defines the tarball flavor (MySQL, NDB, Percona Server, etc)
           --force                         If a destination sandbox already exists, it will be overwritten
           --gtid                          enables GTID
       -h, --help                          help for deploy
@@ -958,18 +961,18 @@ Should you need to compile your own binaries for dbdeployer, follow these steps:
 1. Make sure you have go 1.10+ installed in your system, and that the ``$GOPATH`` variable is set.
 2. Run ``go get -u github.com/datacharmer/dbdeployer``.  This will import all the code that is needed to build dbdeployer.
 3. Change directory to ``$GOPATH/src/github.com/datacharmer/dbdeployer``.
-4. Run ``./scripts/build.sh {linux|OSX} 1.17.0``
-5. If you need the docs enabled binaries (see the section "Generating additional documentation") run ``MKDOCS=1 ./scripts/build.sh {linux|OSX} 1.17.0``
+4. Run ``./scripts/build.sh {linux|OSX} 1.18.0``
+5. If you need the docs enabled binaries (see the section "Generating additional documentation") run ``MKDOCS=1 ./scripts/build.sh {linux|OSX} 1.18.0``
 
 ## Generating additional documentation
 
 Between this file and [the API API list](https://github.com/datacharmer/dbdeployer/blob/master/docs/API/API-1.1.md), you have all the existing documentation for dbdeployer.
 Should you need additional formats, though, dbdeployer is able to generate them on-the-fly. Tou will need the docs-enabled binaries: in the distribution list, you will find:
 
-* dbdeployer-1.17.0-docs.linux.tar.gz
-* dbdeployer-1.17.0-docs.osx.tar.gz
-* dbdeployer-1.17.0.linux.tar.gz
-* dbdeployer-1.17.0.osx.tar.gz
+* dbdeployer-1.18.0-docs.linux.tar.gz
+* dbdeployer-1.18.0-docs.osx.tar.gz
+* dbdeployer-1.18.0.linux.tar.gz
+* dbdeployer-1.18.0.osx.tar.gz
 
 The executables containing ``-docs`` in their name have the same capabilities of the regular ones, but in addition they can run the *hidden* command ``tree``, with alias ``docs``.
 

cmd/admin.go

@@ -1,5 +1,5 @@
 // DBDeployer - The MySQL Sandbox
-// Copyright © 2006-2018 Giuseppe Maxia
+// Copyright © 2006-2019 Giuseppe Maxia
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@ import (
 	"github.com/spf13/cobra"
 	"os"
 	"path"
+	"sort"
 )
 
 func unPreserveSandbox(sandboxDir, sandboxName string) {
@@ -220,7 +221,7 @@ func upgradeSandbox(sandboxDir, oldSandbox, newSandbox string) error {
 	if oldMajor == 10 || newMajor == 10 {
 		common.Exit(1, "upgrade from and to MariaDB is not supported")
 	}
-	greaterThanNewVersion, err := common.GreaterOrEqualVersion(oldSbdesc.Version, newVersionList)
+	greaterThanNewVersion, err := common.GreaterOrEqualVersionList(oldVersionList, newVersionList)
 	common.ErrCheckExitf(err, 1, globals.ErrWhileComparingVersions)
 	if greaterThanNewVersion {
 		common.Exitf(1, "version %s must be greater than %s", newUpgradeVersion, oldUpgradeVersion)
@@ -300,6 +301,46 @@ func runUpgradeSandbox(cmd *cobra.Command, args []string) {
 	}
 }
 
+func showCapabilities(cmd *cobra.Command, args []string) {
+	flavor := ""
+	version := ""
+	if len(args) > 0 {
+		flavor = args[0]
+	}
+	if len(args) > 1 {
+		version = args[1]
+	}
+	for fl, capability := range common.AllCapabilities {
+		if flavor == "" || (flavor != "" && flavor == fl) {
+			var features = make(common.FeatureList)
+			var featureNames []string
+			for featureName, cap := range capability.Features {
+
+				if version != "" {
+					can, _ := common.HasCapability(flavor, featureName, version)
+					if !can {
+						continue
+					}
+				}
+				featureNames = append(featureNames, featureName)
+				features[featureName] = cap
+			}
+			fmt.Printf("## %s\n", fl)
+			sort.Strings(featureNames)
+			for _, fn := range featureNames {
+				cap := features[fn]
+				minVers := common.IntSliceToDottedString(cap.Since)
+				untilVers := ""
+				if cap.Until != nil {
+					untilVers = "until " + common.IntSliceToDottedString(cap.Until)
+				}
+				fmt.Printf("%-25s: %-45s : since %s  %s\n", fn, cap.Description, minVers, untilVers)
+			}
+			fmt.Println()
+		}
+	}
+}
+
 var (
 	adminCmd = &cobra.Command{
 		Use:     "admin",
@@ -334,11 +375,25 @@ The data directory of the old sandbox will be moved to the new one.`,
 		Example: "dbdeployer admin upgrade msb_8_0_11 msb_8_0_12",
 		Run:     runUpgradeSandbox,
 	}
+	adminCapabilitiesCmd = &cobra.Command{
+		Use:   "capabilities [flavor [version]]",
+		Short: "Shows capabilities of a given flavor [and optionally version]",
+		Long: `Shows the capabilities of all flavors. 
+If a flavor is specified, only the capabilities of that flavor are shown.
+If also a version is specified, we show what that version supports`,
+		Example: `dbdeployer admin capabilities
+dbdeployer admin capabilities mysql
+dbdeployer admin capabilities mysql 5.7.11
+dbdeployer admin capabilities mysql 5.7.13
+`,
+		Run: showCapabilities,
+	}
 )
 
 func init() {
 	rootCmd.AddCommand(adminCmd)
 	adminCmd.AddCommand(adminLockCmd)
 	adminCmd.AddCommand(adminUnlockCmd)
 	adminCmd.AddCommand(adminUpgradeCmd)
+	adminCmd.AddCommand(adminCapabilitiesCmd)
 }

cmd/deploy.go

@@ -1,5 +1,5 @@
 // DBDeployer - The MySQL Sandbox
-// Copyright © 2006-2018 Giuseppe Maxia
+// Copyright © 2006-2019 Giuseppe Maxia
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -83,4 +83,6 @@ func init() {
 	setPflag(deployCmd, globals.UseTemplateLabel, "", "", "", "[template_name:file_name] Replace existing template with one from file", true)
 	setPflag(deployCmd, globals.SandboxDirectoryLabel, "", "", "", "Changes the default sandbox directory", false)
 	setPflag(deployCmd, globals.HistoryDirLabel, "", "", "", "Where to store mysql client history (default: in sandbox directory)", false)
+	setPflag(deployCmd, globals.FlavorLabel, "","", "", "Defines the tarball flavor (MySQL, NDB, Percona Server, etc)", false)
+	setPflag(deployCmd, globals.ClientFromLabel, "","", "", "Where to get the client binaries from", false)
 }

cmd/replication.go

@@ -52,7 +52,9 @@ func replicationSandbox(cmd *cobra.Command, args []string) {
 		}
 		// 5.5.1
 
-		isMinimumSync, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumSemiSyncVersion)
+
+		// isMinimumSync, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumSemiSyncVersion)
+		isMinimumSync, err := common.HasCapability(sd.Flavor, common.SemiSynch,sd.Version)
 		common.ErrCheckExitf(err, 1, globals.ErrWhileComparingVersions)
 		if isMinimumSync {
 			sd.SemiSyncOptions = sandbox.SingleTemplates["semisync_master_options"].Contents

cmd/single.go

@@ -113,6 +113,30 @@ func checkForRootValue(value, label, defaultVal string) {
 	}
 }
 
+
+// Gets the Database flavor
+// If none is found, defaults to MySQL
+func getFlavor(userDefinedFlavor, basedir string) string {
+	flavor := userDefinedFlavor
+	flavorFile := path.Join(basedir, globals.FlavorFileName)
+	if common.FileExists(flavorFile) {
+		flavorText, err := common.SlurpAsString(flavorFile)
+		if err != nil {
+			common.Exitf(1, "error reading flavor file %s: %s", flavorFile, err)
+		}
+		flavorText = strings.TrimSpace(flavorText)
+		if userDefinedFlavor != "" && userDefinedFlavor != flavorText {
+			common.Exitf(1, "user defined flavor %s doesn't match found flavor %s", userDefinedFlavor, flavorText)
+		}
+		flavor = flavorText
+	}
+	// TODO Flavor detection based on tarball contents
+	if flavor == "" {
+		flavor = common.MySQLFlavor
+	}
+	return flavor
+}
+
 func fillSandboxDdefinition(cmd *cobra.Command, args []string) (sandbox.SandboxDef, error) {
 	var sd sandbox.SandboxDef
 
@@ -205,6 +229,16 @@ func fillSandboxDdefinition(cmd *cobra.Command, args []string) (sandbox.SandboxD
 		common.Exitf(1, "basedir '%s' not found", sd.Basedir)
 	}
 
+	sd.ClientBasedir, _ = flags.GetString(globals.ClientFromLabel)
+	if sd.ClientBasedir == "" {
+		sd.ClientBasedir = sd.Basedir
+	} else {
+		clientBasedir := path.Join(basedir, sd.ClientBasedir)
+		if !common.DirExists(clientBasedir) {
+			common.Exitf(1, globals.ErrDirectoryNotFound, clientBasedir)
+		}
+		sd.ClientBasedir = clientBasedir
+	}
 	err = common.CheckTarballOperatingSystem(sd.Basedir)
 	common.ErrCheckExitf(err, 1, "incorrect tarball detected")
 
@@ -241,7 +275,9 @@ func fillSandboxDdefinition(cmd *cobra.Command, args []string) (sandbox.SandboxD
 	sd.DbUser, _ = flags.GetString(globals.DbUserLabel)
 	sd.DbPassword, _ = flags.GetString(globals.DbPasswordLabel)
 	sd.RplUser, _ = flags.GetString(globals.RplUserLabel)
+	sd.Flavor, _ = flags.GetString(globals.FlavorLabel)
 
+	sd.Flavor = getFlavor(sd.Flavor, sd.Basedir)
 	checkForRootValue(sd.DbUser, globals.DbUserLabel, globals.DbUserValue)
 	checkForRootValue(sd.RplUser, globals.RplUserLabel, globals.RplUserValue)
 
@@ -287,13 +323,15 @@ func fillSandboxDdefinition(cmd *cobra.Command, args []string) (sandbox.SandboxD
 	if gtid {
 		templateName := "gtid_options_56"
 		// 5.7.0
-		isEnhancedGtid, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumEnhancedGtidVersion)
+		// isEnhancedGtid, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumEnhancedGtidVersion)
+		isEnhancedGtid, err := common.HasCapability(sd.Flavor, common.EnhancedGTID, sd.Version)
 		common.ErrCheckExitf(err, 1, globals.ErrWhileComparingVersions)
 		if isEnhancedGtid {
 			templateName = "gtid_options_57"
 		}
 		// 5.6.9
-		isMinimumGtid, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumGtidVersion)
+		//isMinimumGtid, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumGtidVersion)
+		isMinimumGtid, err := common.HasCapability(sd.Flavor, common.GTID, sd.Version)
 		common.ErrCheckExitf(err, 1, globals.ErrWhileComparingVersions)
 		if isMinimumGtid {
 			sd.GtidOptions = sandbox.SingleTemplates[templateName].Contents
@@ -307,7 +345,8 @@ func fillSandboxDdefinition(cmd *cobra.Command, args []string) (sandbox.SandboxD
 	if replCrashSafe && sd.ReplCrashSafeOptions == "" {
 		// 5.6.2
 
-		isMinimumCrashSafe, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumCrashSafeVersion)
+		// isMinimumCrashSafe, err := common.GreaterOrEqualVersion(sd.Version, globals.MinimumCrashSafeVersion)
+		isMinimumCrashSafe, err := common.HasCapability(sd.Flavor, common.CrashSafe, sd.Version)
 		common.ErrCheckExitf(err, 1, globals.ErrWhileComparingVersions)
 		if isMinimumCrashSafe {
 			sd.ReplCrashSafeOptions = sandbox.SingleTemplates["repl_crash_safe_options"].Contents

cmd/templates.go

@@ -262,7 +262,11 @@ func importTemplates(cmd *cobra.Command, args []string) {
 	if err != nil {
 		common.Exitf(1, "error converting compatible version %s from file %s\n", common.CompatibleVersion, versionFile)
 	}
-	compatibleTemplate, err := common.GreaterOrEqualVersion(templateVersion, compatibleVersionList)
+	templateVersionList, err := common.VersionToList(templateVersion)
+	if err != nil {
+		common.Exitf(1, "error converting version %s from file %s\n", templateVersion, versionFile)
+	}
+	compatibleTemplate, err := common.GreaterOrEqualVersionList(templateVersionList, compatibleVersionList)
 	common.ErrCheckExitf(err, 1, globals.ErrWhileComparingVersions)
 	if !compatibleTemplate {
 		common.Exitf(1, "templates are for version %s. The minimum compatible version is %s", templateVersion, common.CompatibleVersion)

cmd/unpack.go

@@ -1,5 +1,5 @@
 // DBDeployer - The MySQL Sandbox
-// Copyright © 2006-2018 Giuseppe Maxia
+// Copyright © 2006-2019 Giuseppe Maxia
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -28,6 +28,25 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func detectFlavor(tarballName string) string {
+	flavor := ""
+	flavorsRegexps := map[string]string{
+		common.MySQLFlavor : `mysql`,
+		common.PerconaServerFlavor : `Percona-Server`,
+		common.MariaDbFlavor : `mariadb`,
+		common.NDBFlavor : `mysql-cluster`,
+		common.TiDbFlavor : `tidb`,
+	}
+
+	for key, value := range flavorsRegexps {
+		re := regexp.MustCompile(value)
+		if re.MatchString(tarballName) {
+			return key
+		}
+	}
+	return flavor
+}
+
 func unpackTarball(cmd *cobra.Command, args []string) {
 	flags := cmd.Flags()
 	Basedir, err := getAbsolutePathFromFlag(cmd, "sandbox-binary")
@@ -41,7 +60,11 @@ func unpackTarball(cmd *cobra.Command, args []string) {
 	tarball := args[0]
 	reVersion := regexp.MustCompile(`(\d+\.\d+\.\d+)`)
 	verList := reVersion.FindAllStringSubmatch(tarball, -1)
-	detectedVersion := verList[0][0]
+
+	detectedVersion := ""
+	if verList != nil {
+		detectedVersion = verList[0][0]
+	}
 	// common.CondPrintf(">> %#v %s\n",verList, detected_version)
 
 	isShell, _ := flags.GetBool(globals.ShellLabel)
@@ -51,6 +74,13 @@ func unpackTarball(cmd *cobra.Command, args []string) {
 			"unpack: Option --target-server can only be used with --shell")
 	}
 
+	flavor, _ := flags.GetString(globals.FlavorLabel)
+	if flavor == "" {
+		flavor = detectFlavor(tarball)
+		if flavor == "" {
+			common.Exitf(1, "No flavor detected in %s. Please use --%s", tarball, globals.FlavorLabel)
+		}
+	}
 	Version, _ := flags.GetString(globals.UnpackVersionLabel)
 	if Version == "" {
 		Version = detectedVersion
@@ -109,6 +139,8 @@ func unpackTarball(cmd *cobra.Command, args []string) {
 		err = os.Rename(finalName, destination)
 		common.ErrCheckExitf(err, 1, "%s", err)
 	}
+	err = common.WriteString(flavor, path.Join(destination, globals.FlavorFileName))
+	common.ErrCheckExitf(err, 1, "error writing %s in %s", globals.FlavorFileName, destination)
 }
 
 // unpackCmd represents the unpack command
@@ -145,4 +177,5 @@ func init() {
 	unpackCmd.PersistentFlags().String(globals.PrefixLabel, "", "Prefix for the final expanded directory")
 	unpackCmd.PersistentFlags().Bool(globals.ShellLabel, false, "Unpack a shell tarball into the corresponding server directory")
 	unpackCmd.PersistentFlags().String(globals.TargetServerLabel, "", "Uses a different server to unpack a shell tarball")
+	unpackCmd.PersistentFlags().String(globals.FlavorLabel, "", "Defines the tarball flavor (MySQL, NDB, Percona Server, etc)")
 }