Integration tests for gp configure, start, stop and status commands

This pull request adds integration tests for the gp command, which was
introduced in Initial project spine PR.

Code structure: Integration test files have been organized under the go-
tools/test/integration directory. Within this directory, dedicated
subdirectories have been created for each gp sub-command, including
configure, start, status, and stop. These subdirectories contain
distinct files for positive and negative test cases, as well as a file
for setup and teardown functions and global variables.

Also introduced a new make target named integration-test to run the
integration tests.

To run tests specifically for one of the commands, such as "gp start,"
you can use the "go test" command within the package directory (go-
tools/test/integration/start). You also have the flexibility to execute
an individual test case or testfunction by employing the "go test"
command along with various options, including the "-run" flag.
Here's an example:

to run tests in the "TestSingleHostStartSuccess" function:
go test -v -run TestSingleHostStartSuccess

to run a specific test case within the "TestSingleHostStartSuccess":
go test -v -run TestSingleHostStartSuccess/start_hub_and_agents_successfully

Furthermore, a test helper package named "testutils" has been included.
This package contains a collection of functions for general test
assertions and verifications.

Testing done: To run these integration tests, a dedicated job has been
added to the existing pipeline through our pipeline.yml file. This job
facilitates the execution of both single-host and multi-host tests,
running them in parallel stages.

Co-authored-by: Sruthi C P <sruthip2@vmware.com>
Co-authored-by: Nihal Jain <jnihal@vmware.com>

gpMgmt/bin/go-tools/Makefile

@@ -10,7 +10,7 @@ MAC_ENV := env GOOS=darwin GOARCH=amd64
 
 .PHONY: unit test
 TEST_PACKAGES := ./...
-unit: TEST_PACKAGES := $(shell go list ./...)
+unit: TEST_PACKAGES := $(shell go list ./... | grep -v ./test/integration)
 unit:
 	go test -v -count=1 $(TEST_PACKAGES) --cover -coverprofile=coverage.out;
 	@cat coverage.out | \
@@ -100,3 +100,9 @@ clean:
 
 uninstall:
 	rm -rf $(GPHOME)/bin/gp
+
+.PHONY: integration-test
+TEST_PACKAGES := ./test/integration/...
+integration-test: TEST_PACKAGES := $(shell go list ./test/integration/...)
+integration-test:
+	go test -v -count=1 $(TEST_PACKAGES) -timeout 0 -p 1 -args -hostfile=$(FILE)

gpMgmt/bin/go-tools/ci/pipelines/pipeline.yml

@@ -160,7 +160,9 @@ groups:
     jobs:
       - lint
       - build
-      - run_tests
+      - unit_tests
+      - integration_tests
+
 
 jobs:
   - name: lint
@@ -217,18 +219,15 @@ jobs:
           params:
             file: gp_binary/gp.tgz
 
-  # TODO: Create seperate jobs for unit tests and integration tests.
-  - name: run_tests
+  # TODO: Convert this to use a docker container
+  - name: unit_tests
     plan:
     - in_parallel:
         steps:
         - do:
-          - get: gp_binary
-            resource: gp_binary
-            passed: [build]
-            trigger: true
           - get: gpdb_binary
           - get: gpdb_src
+            trigger: true
           - get: ccp_src
           - get: ccp-image
           - get: terraform.d
@@ -240,24 +239,101 @@ jobs:
               vars:
                 <<: *ccp_default_vars
                 instance_type: n1-standard-2
-                number_of_nodes: 4
+                number_of_nodes: 1
           - task: gen_cluster
             file: ccp_src/ci/tasks/gen_cluster.yml
             params:
               <<: *ccp_gen_cluster_default_params
-          - task: run_tests
+          - task: unit_tests
             image: ccp-image
             config:
               platform: linux
               inputs:
                 - name: gpdb_src
                 - name: ccp_src
                 - name: cluster_env_files
-                - name: gp_binary
               run:
-                path: gpdb_src/gpMgmt/bin/go-tools/ci/scripts/run_tests.bash
+                path: bash
+                args:
+                  - -c
+                  - |
+                    set -eux -o pipefail
+                    ccp_src/scripts/setup_ssh_to_cluster.sh
+                    ssh -n cdw "
+                        set -eux -o pipefail
+                        cd /home/gpadmin/gpdb_src/gpMgmt/bin/go-tools
+                        make unit
+                    "
 
             on_success:
               <<: *ccp_destroy
             ensure:
               <<: *set_failed
+
+  - name: integration_tests
+    plan:
+      - get: gp_binary
+        resource: gp_binary
+        passed: [ build ]
+        trigger: true
+      - get: gpdb_binary
+      - get: gpdb_src
+      - get: ccp_src
+      - get: ccp-image
+      - get: terraform.d
+        params:
+          unpack: true
+      - in_parallel:
+          steps:
+            - do:
+                - put: terraform
+                  params:
+                    <<: *ccp_default_params
+                    vars:
+                      <<: *ccp_default_vars
+                      instance_type: n1-standard-2
+                      number_of_nodes: 1
+                - task: gen_cluster
+                  file: ccp_src/ci/tasks/gen_cluster.yml
+                  params:
+                    <<: *ccp_gen_cluster_default_params
+                - task: single_host_tests
+                  config:
+                    platform: linux
+                    inputs:
+                      - name: gpdb_src
+                      - name: ccp_src
+                      - name: cluster_env_files
+                      - name: gp_binary
+                      - name: ccp-image
+                    run:
+                      path: gpdb_src/gpMgmt/bin/go-tools/ci/scripts/run_integration_tests.bash
+                  image: ccp-image
+            - do:
+                - put: terraform
+                  params:
+                    <<: *ccp_default_params
+                    vars:
+                      <<: *ccp_default_vars
+                      instance_type: n1-standard-2
+                      number_of_nodes: 4
+                - task: gen_cluster
+                  file: ccp_src/ci/tasks/gen_cluster.yml
+                  params:
+                    <<: *ccp_gen_cluster_default_params
+                - task: multi_host_tests
+                  config:
+                    platform: linux
+                    inputs:
+                      - name: gpdb_src
+                      - name: ccp_src
+                      - name: cluster_env_files
+                      - name: gp_binary
+                      - name: ccp-image
+                    run:
+                      path: gpdb_src/gpMgmt/bin/go-tools/ci/scripts/run_integration_tests.bash
+                  image: ccp-image
+                  on_success:
+                    <<: *ccp_destroy
+                  ensure:
+                    <<: *set_failed

gpMgmt/bin/go-tools/ci/scripts/run_integration_tests.bash

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eux -o pipefail
+
+ccp_src/scripts/setup_ssh_to_cluster.sh
+
+scp cluster_env_files/hostfile_all cdw:/tmp
+tar -xzf gp_binary/gp.tgz
+scp gp cdw:/home/gpadmin/
+
+ssh -n cdw "
+    set -eux -o pipefail
+
+    export PATH=/usr/local/go/bin:\$PATH
+    source /usr/local/greenplum-db-devel/greenplum_path.sh
+
+    chmod +x gp
+    gpsync -f /tmp/hostfile_all gp =:/usr/local/greenplum-db-devel/bin/gp
+    cd /home/gpadmin/gpdb_src/gpMgmt/bin/go-tools
+    ./ci/scripts/generate_ssl_cert_multi_host.bash
+
+    make integration-test FILE=/tmp/hostfile_all
+"

gpMgmt/bin/go-tools/cli/commands.go

@@ -59,19 +59,36 @@ func InitializeCommand(cmd *cobra.Command, args []string) error {
 		return err
 	}
 	hubLogDir = Conf.LogDir
-	InitializeLogger(cmd, args)
+
+	err = InitializeLogger(cmd, args)
+	if err != nil {
+		return err
+	}
 
 	return nil
 }
 
-func InitializeLogger(cmd *cobra.Command, args []string) {
+func InitializeLogger(cmd *cobra.Command, args []string) error {
 	// CommandPath lists the names of the called command and all of its parent commands, so this
 	// turns e.g. "gp stop hub" into "gp_stop_hub" to generate a unique log file name for each command.
 	logName := strings.ReplaceAll(cmd.CommandPath(), " ", "_")
+
+	// Create the log directory if it does not exist, otherwise gplog panics
+	if _, err := os.Stat(hubLogDir); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(hubLogDir, 0755)
+			if err != nil {
+				return err
+			}
+		}
+
+		return err
+	}
+
 	gplog.SetLogFileNameFunc(func(program string, logdir string) string {
-		// TODO: Check if this path exists or not, otherwise gplog panics
 		return filepath.Join(hubLogDir, fmt.Sprintf("%s.log", logName))
 	})
+
 	gplog.InitializeLogging(logName, "")
 
 	timeFormat := time.Now().Format("2006-01-02 15:04:05.000000")
@@ -82,6 +99,8 @@ func InitializeLogger(cmd *cobra.Command, args []string) {
 	if Verbose {
 		gplog.SetVerbosity(gplog.LOGDEBUG)
 	}
+
+	return nil
 }
 
 func ConnectToHubFunc(conf *hub.Config) (idl.HubClient, error) {

gpMgmt/bin/go-tools/cli/hub.go

@@ -60,7 +60,7 @@ func configureCmd() *cobra.Command {
 	configureCmd := &cobra.Command{
 		Use:    "configure",
 		Short:  "Configure gp as a systemd daemon",
-		PreRun: InitializeLogger,
+		PreRunE: InitializeLogger,
 		RunE:   RunConfigure,
 	}
 
@@ -83,10 +83,22 @@ func configureCmd() *cobra.Command {
 	configureCmd.Flags().StringVar(&hostfilePath, "hostfile", "", `Path to file containing a list of segment hostnames`)
 	configureCmd.MarkFlagsMutuallyExclusive("host", "hostfile")
 
-	viper.BindPFlag("gphome", configureCmd.Flags().Lookup("gphome")) //nolint
+	requiredFlags := []string{
+		"ca-certificate",
+		"ca-key",
+		"server-certificate",
+		"server-key",
+	}
+	for _, flag := range requiredFlags {
+		configureCmd.MarkFlagRequired(flag) // nolint
+	}
+
+	viper.BindPFlag("gphome", configureCmd.Flags().Lookup("gphome")) // nolint
 	gphome = viper.GetString("gphome")
+
 	return configureCmd
 }
+
 func RunConfigure(cmd *cobra.Command, args []string) (err error) {
 	if gphome == "" {
 		return fmt.Errorf("not a valid gphome found\n")
@@ -104,6 +116,10 @@ func RunConfigure(cmd *cobra.Command, args []string) (err error) {
 	if !cmd.Flags().Lookup("host").Changed && !cmd.Flags().Lookup("hostfile").Changed {
 		return errors.New("at least one hostname must be provided using either --host or --hostfile")
 	}
+
+	if agentPort == hubPort {
+		return errors.New("hub port and agent port must be different")
+	}
 
 	// Convert file/directory paths to absolute path before writing to gp.Conf file
 	err = resolveAbsolutePaths(cmd)