Update the run hook to skip the pg-helper for non bootstrap node

components/applications-service/habitat/hooks/run

@@ -10,10 +10,11 @@ exec 2>&1
 # Postgres Database Management
 # We do this here because init hooks block the hab supervisor
 DBNAME="{{cfg.storage.database}}"
-
-pg-helper ensure-service-database "$DBNAME"
-pg-helper create-extension "$DBNAME" "pgcrypto"
-pg-helper fix-permissions "$DBNAME"
+if [ ! -f /hab/.skip_migration ]; then
+  pg-helper ensure-service-database "$DBNAME"
+  pg-helper create-extension "$DBNAME" "pgcrypto"
+  pg-helper fix-permissions "$DBNAME"
+fi
 
 # cleanup old migration files
 rm -rf "{{pkg.svc_static_path}}/schema"

components/authn-service/habitat/hooks/run

@@ -6,8 +6,11 @@ exec 2>&1
 # Call the script to block until user accepts the MLSA via the package's config
 {{pkgPathFor "chef/mlsa"}}/bin/accept {{cfg.mlsa.accept}}
 
-pg-helper ensure-service-database "{{cfg.storage.database}}"
+if [ ! -f /hab/.skip_migration ]; then
+  pg-helper ensure-service-database "{{cfg.storage.database}}"
+
+  pg-helper create-extension "{{cfg.storage.database}}" "pgcrypto"
+fi
 
-pg-helper create-extension "{{cfg.storage.database}}" "pgcrypto"
 
 exec authn-service serve {{pkg.svc_config_path}}/config.yml

components/authz-service/habitat/hooks/run

@@ -6,11 +6,12 @@ exec 2>&1
 # Call the script to block until user accepts the MLSA via the package's config
 {{pkgPathFor "chef/mlsa"}}/bin/accept {{cfg.mlsa.accept}}
 
-pg-helper ensure-service-database "{{cfg.storage.database}}"
-
-pg-helper create-extension "{{cfg.storage.database}}" "pgcrypto"
-pg-helper create-extension "{{cfg.storage.database}}" "uuid-ossp"
+if [ ! -f /hab/.skip_migration ]; then
+  pg-helper ensure-service-database "{{cfg.storage.database}}"
 
+  pg-helper create-extension "{{cfg.storage.database}}" "pgcrypto"
+  pg-helper create-extension "{{cfg.storage.database}}" "uuid-ossp"
+fi
 # cleanup old migration files
 rm -rf {{pkg.svc_static_path}}/{migrations,data-migrations}
 

components/automate-backend-deployment/habitat/plan.sh

@@ -26,7 +26,10 @@ pkg_deps=(
   core/busybox-static  
   chef/automate-ha-cluster-ctl
 )
-
+#core/aws-cli ( core/aws-cli/1.21.11/20231020110846 core/aws-cli/1.21.11/20220817123642 )
+   # core/openssl11 ( core/openssl11/1.1.1w/20231020105352 core/openssl11/1.1.1k/20220311131131 )
+   # core/python ( core/python/3.10.0/20231020105702 core/python/3.10.0/20220817121853 )
+   #
 pkg_build_deps=(
   core/gcc
 )

components/automate-builder-api/habitat/hooks/run

@@ -18,9 +18,11 @@ render-template config.toml "{{pkg.svc_var_path}}/etc/config.toml" \
 
 export DBNAME="{{cfg.datastore.database}}"
 export USERNAME="{{cfg.datastore.user}}"
-pg-helper ensure-service-database "$DBNAME"
-pg-helper fix-permissions "$USERNAME"
-pg-helper alter-role --superuser=true "$USERNAME"
+if [ ! -f /hab/.skip_migration ]; then
+  pg-helper ensure-service-database "$DBNAME"
+  pg-helper fix-permissions "$USERNAME"
+  pg-helper alter-role --superuser=true "$USERNAME"
+fi
 
 if ! ls {{pkg.svc_data_path}}/keys/bldr-*; then
   echo "generating bldr key"

components/automate-cli/cmd/chef-automate/automateClusterCtlUtil.go

@@ -273,7 +273,7 @@ func moveAirgapFrontendBundlesOnlyToTransferDir(airgapMetadata airgap.UnpackMeta
 	return nil
 }
 
-func getVersion(airgapBundle string) (string, error) {
+func GetVersion(airgapBundle string) (string, error) {
 	_, manifestBytes, err := airgap.GetMetadata(airgapBundle)
 	if err != nil {
 		return "", status.Annotate(err, status.AirgapUnpackInstallBundleError)
@@ -304,7 +304,7 @@ func moveAirgapBackendBundlesOnlyToTransferDir(airgapMetadata airgap.UnpackMetad
 	return nil
 }
 func getFrontendBundleName(airgapPath string) (string, error) {
-	version, err := getVersion(airgapPath)
+	version, err := GetVersion(airgapPath)
 	if err != nil {
 		return "", err
 	}

components/automate-cli/cmd/chef-automate/ha_node_add.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"os"
+	"strings"
 
 	"github.com/chef/automate/components/automate-cli/pkg/docs"
 	"github.com/chef/automate/lib/io/fileutils"
@@ -52,10 +55,90 @@ func runAddNodeHACmd(addDeleteNodeHACmdFlags *AddDeleteNodeHACmdFlags) func(c *c
 		if err != nil {
 			return err
 		}
+		// 1. get the version of the automate from the bootstrap_automate
+		// 2. get the bundle name from /hab/a2_deploy_workspace/terraform/a2ha_aib_fe.auto.tfvars
+		// 3. check bundle existing here /hab/a2_deploy_workspace/terraform/transfer_files/*.aib
+		// 1 and 2 should be same then we have to proceed, other wise we have to give instruction how to proceed
+		// we can test the procedure in case 1 and 2 are different
+		/*
+			frontend_aib_dest_file = "/var/tmp/frontend-4.10.21.aib"
+			frontend_aib_local_file = "frontend-4.10.21.aib"
+		*/
+		if !preCheckForAddNode() {
+			// we should not reach here
+			return fmt.Errorf("airgap bundle is missing")
+		}
 		return nodeAdder.Execute(c, args)
 	}
 }
 
+const filePath = "/hab/a2_deploy_workspace/terraform/a2ha_aib_fe.auto.tfvars"
+
+func isFileExist(filePath string) bool {
+	_, err := os.Stat(filePath)
+	if err == nil {
+		return true
+	} else if errors.Is(err, fs.ErrNotExist) {
+		writer.Error("file not exists " + filePath + " " + err.Error())
+	}
+	return false
+}
+
+func getAirgapBundleTransferFileVersion(filePath string) (string, error) {
+	fileContent, err := os.ReadFile(filePath)
+	if err != nil {
+		writer.Error("fail to read the file" + filePath + " " + err.Error())
+		return "", err
+	}
+
+	lines := strings.Split(string(fileContent), "\n")
+	bundleName := ""
+	for _, line := range lines {
+		parts := strings.Split(line, "=")
+		if len(parts) != 2 {
+			continue
+		}
+		key := strings.TrimSpace(parts[0])
+		value := strings.TrimSpace(parts[1])
+		switch key {
+		case "frontend_aib_local_file":
+			bundleName = strings.Trim(value, "\"")
+		}
+	}
+	bundleFilePath := "/hab/a2_deploy_workspace/terraform/transfer_files/" + bundleName
+	if isFileExist(bundleFilePath) {
+		airgapbundleVersion, err := GetVersion(bundleFilePath)
+		if err != nil {
+			writer.Error("fail to GetVersion" + bundleFilePath + " " + err.Error())
+			return "", err
+		}
+		return airgapbundleVersion, err
+	}
+	return "", err
+}
+
+func preCheckForAddNode() bool {
+	if isFileExist(filePath) {
+		airgapBundleVersion, err1 := getAirgapBundleTransferFileVersion(filePath)
+		if err1 != nil {
+			// Not able to get the version still we are proceding
+			writer.Println("not able to get the version from the transfer file " + airgapBundleVersion + err1.Error())
+		}
+		installedVersion, err := GetMinimunBuildVersionFromFrontEndServer()
+		if err != nil {
+			// Not able to get the version still we are proceding
+			writer.Println("not able to get the version from the frontend node " + installedVersion + err.Error())
+		}
+		if airgapBundleVersion == installedVersion {
+			// go-ahead with add-node : success case
+			return true
+		}
+		writer.Println("Airgapped Bundle version : " + airgapBundleVersion)
+		writer.Println("Installed Bundle version : " + installedVersion)
+		return false
+	}
+	return false
+}
 func haAddNodeFactory(addDeleteNodeHACmdFlags *AddDeleteNodeHACmdFlags, deployerType string) (HAModifyAndDeploy, error) {
 	if addDeleteNodeHACmdFlags.onPremMode && addDeleteNodeHACmdFlags.awsMode {
 		return nil, errors.New("Cannot use both --onprem-mode and --aws-mode together. Provide only one at a time")

components/automate-cli/cmd/chef-automate/summary.go

@@ -27,7 +27,7 @@ const (
 	curlHeaderFlag              = "--header"
 	curlAuthorization           = "'Authorization: Bearer %s'"
 	initialServiceState         = "down"
-	initialServicePid           = ""
+	initialServicePid           = "0"
 	initialHealth               = "ERROR"
 	initialFormattedDuration    = "0d 0h 0m 0s"
 	initialRole                 = "Unknown"
@@ -478,7 +478,12 @@ func (ss *Summary) getBEDefaultServiceDetails(output string) (string, string, st
 	}
 	memeberId := defaultServiceDetails["sys"].(map[string]interface{})["member_id"].(string)
 	serviceState := defaultServiceDetails["process"].(map[string]interface{})["state"].(string)
-	servicePid := fmt.Sprintf("%d", int(defaultServiceDetails["process"].(map[string]interface{})["pid"].(float64)))
+	var servicePid string
+	if defaultServiceDetails["process"].(map[string]interface{})["pid"] == nil {
+		servicePid = "0"
+	} else {
+		servicePid = fmt.Sprintf("%d", int(defaultServiceDetails["process"].(map[string]interface{})["pid"].(float64)))
+	}
 	startingTime := defaultServiceDetails["process"].(map[string]interface{})["state_entered"].(float64)
 	startingTime = float64(nowFunc().UTC().Unix()) - startingTime
 

components/automate-cli/cmd/chef-automate/upgrade.go

@@ -386,6 +386,12 @@ func restartDeploymentService() error {
 }
 
 func runAutomateHAFlow(args []string, offlineMode bool) error {
+	// First, get the minimum version of the automate from all the FE node
+	// we cannot depands up on /hab/a2_deploy_workspace/terraform/a2ha_aib_fe.auto.tfvars for upgrade
+	// In case of upgrade break/fails in-between then the above file do the block us to trigger
+	// the subsequent upgrade
+	// We have version_check_for_addnode in provision.sh.tpl, this will block the upgrade
+	// if version is same on any FE
 	if !upgradeRunCmdFlags.skipVerify {
 		err := executeConfigVerifyAndPromptConfirmationOnError("")
 		if err != nil {
@@ -410,6 +416,17 @@ func runAutomateHAFlow(args []string, offlineMode bool) error {
 		}
 	}
 	modeOfDeployment := getModeOfDeployment()
+	// get the Installed Minimum version
+	installedVersion, err := GetMinimunBuildVersionFromFrontEndServer()
+	if err != nil {
+		// Not able to get the version still we are proceding
+		writer.Println("not able to get the version from the frontend node " + err.Error())
+	}
+	airgapbundleVersion, _ := GetVersion(upgradeRunCmdFlags.airgap)
+	if !CompareSemverVersion(installedVersion, airgapbundleVersion) {
+		return errors.New("cannot downgrade the cluster")
+	}
+
 	if modeOfDeployment == EXISTING_INFRA_MODE {
 
 		infra, err := getAutomateHAInfraDetails()
@@ -432,7 +449,6 @@ func runAutomateHAFlow(args []string, offlineMode bool) error {
 		if err != nil {
 			return err
 		}
-
 		finalTemplate := renderSettingsToA2HARBFile(existingNodesA2harbTemplate, result, DEPLOY)
 		writeToA2HARBFile(finalTemplate, initConfigHabA2HAPathFlag.a2haDirPath+"a2ha.rb")
 		writer.Println("a2ha.rb has regenerated...")

components/automate-cli/cmd/chef-automate/version.go

@@ -24,6 +24,7 @@ import (
 	"github.com/chef/automate/lib/stringutils"
 	"github.com/chef/automate/lib/version"
 	"github.com/fatih/color"
+	semver "github.com/hashicorp/go-version"
 )
 
 var versionCmd = &cobra.Command{
@@ -56,6 +57,28 @@ var VersionCommandFlags = struct {
 	isPostgresql bool
 }{}
 
+type semverVersion []string
+
+func (s semverVersion) Len() int {
+	return len(s)
+}
+
+func (s semverVersion) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s semverVersion) Less(i, j int) bool {
+	v1, err := semver.NewVersion(s[i])
+	if err != nil {
+		logrus.Errorf("Error while getting HA Infra details :: %s", err)
+	}
+	v2, err := semver.NewVersion(s[j])
+	if err != nil {
+		logrus.Errorf("Error while getting HA Infra details :: %s", err)
+	}
+	return v1.LessThan(v2)
+}
+
 func runVersionCmd(cmd *cobra.Command, args []string) error {
 	writer.Printf("Version: %s\n", "2")
 	// Check for bastion
@@ -335,6 +358,38 @@ func getBastionVersion() error {
 	return nil
 }
 
+// getFrontEndVersion : this func use to run the command on all the FE and return the array as a output
+func getFrontEndVersion(automateIps []string, infra *AutomateHAInfraDetails, cmdExecuter RemoteCmdExecutor) (map[string]string, error) {
+	automateCmd := A2VERSIONCMD
+	if VersionCommandFlags.verbose {
+		automateCmd = A2VERSIONVERBOSE
+	}
+	nodeMap := &NodeTypeAndCmd{
+		Automate: &Cmd{CmdInputs: &CmdInputs{NodeType: false}},
+		Frontend: &Cmd{CmdInputs: &CmdInputs{
+			Cmd:                      automateCmd,
+			NodeIps:                  automateIps,
+			NodeType:                 true,
+			SkipPrintOutput:          true,
+			HideSSHConnectionMessage: true}},
+		ChefServer: &Cmd{CmdInputs: &CmdInputs{NodeType: false}},
+		Postgresql: &Cmd{CmdInputs: &CmdInputs{NodeType: false}},
+		Opensearch: &Cmd{CmdInputs: &CmdInputs{NodeType: false}},
+		Infra:      infra,
+	}
+
+	cmdresult, err := cmdExecuter.ExecuteWithNodeMap(nodeMap)
+	if err != nil {
+		logrus.Error("ERROR", err)
+		return nil, err
+	}
+	versionMap := make(map[string]string)
+	for ip, result := range cmdresult {
+		versionMap[ip] = result[0].Output
+	}
+	return versionMap, nil
+}
+
 func getChefAutomateVersion(automateIps []string, infra *AutomateHAInfraDetails, cmdExecuter RemoteCmdExecutor) (map[string]string, error) {
 	automateCmd := A2VERSIONCMD
 	if VersionCommandFlags.verbose {
@@ -355,7 +410,6 @@ func getChefAutomateVersion(automateIps []string, infra *AutomateHAInfraDetails,
 	}
 
 	cmdresult, err := cmdExecuter.ExecuteWithNodeMap(nodeMap)
-
 	if err != nil {
 		logrus.Error("ERROR", err)
 		return nil, err
@@ -755,3 +809,60 @@ func extractVersion(input string, pattern string) (string, error) {
 	version := match[1]
 	return version, nil
 }
+
+// Below function get the chef-automate version from all the node and find the minimum version
+// In case of upgrade break in between, then re-trigger of upgrade required the minimum version
+// to check for other node required to upgrade
+func GetMinimunBuildVersionFromFrontEndServer() (string, error) {
+	infra, err := getAutomateHAInfraDetails()
+	if err != nil {
+		logrus.Errorf("Error while getting HA Infra details :: %s", err)
+		return "", err
+	}
+	automateIps, chefServerIps, _, _, errList := getIPAddressesFromFlagOrInfra(infra)
+	if errList != nil && errList.Len() > 0 {
+		logrus.Errorf("Error while getting IP addresses :: %s", getSingleErrorFromList(errList))
+		return "", getSingleErrorFromList(errList)
+	}
+
+	frontEnd := append(automateIps, chefServerIps...)
+	sshUtil := NewSSHUtil(&SSHConfig{})
+	cmdExecutor := NewRemoteCmdExecutorWithoutNodeMap(sshUtil, writer)
+
+	if len(frontEnd) != 0 {
+		versions, err := getFrontEndVersion(frontEnd, infra, cmdExecutor)
+		logrus.Debug("map of versions :", versions)
+		if err != nil {
+			logrus.Errorf("Error while getting Automate Version :: %s", err)
+			return "", err
+		}
+		return getMinimumVersion(versions), nil
+	}
+	return "", nil
+}
+
+func getMinimumVersion(mVersions map[string]string) string {
+	minVer := "100.0.0" // need to handle this
+	for key, value := range mVersions {
+		version, _ := extractVersion(value, VERSIONREGEX)
+		if len(version) < 1 {
+			logrus.Debug(key, version)
+			continue
+		}
+		if CompareSemverVersion(version, minVer) {
+			minVer = version
+		}
+
+	}
+	logrus.Debug(minVer)
+	return minVer
+}
+
+// return true when second > first
+// return false when second <= first
+// CompareSemverVersion
+func CompareSemverVersion(first, second string) bool {
+	v1, _ := semver.NewVersion(first)
+	v2, _ := semver.NewVersion(second)
+	return v1.LessThan(v2)
+}