K8SPG-718 patroni metrics to pmm (#1209)

* K8SPG-718 patroni metrics to pmm

* remove hardcoded versions, the tests will fail but they will pass again once we bump the version officially to 2.8

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: gkech

e2e-tests/tests/monitoring-pmm3/05-check-qan.yaml

@@ -11,5 +11,6 @@ commands:
       instance=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/instance-set=instance1 -o 'jsonpath={.items[].metadata.name}')
 
       get_metric_values node_boot_time_seconds ${NAMESPACE}-${instance} ${token}
+      get_metric_values patroni_postgres_running ${NAMESPACE}-${instance} ${token}
       get_qan20_values_pmm3 ${NAMESPACE}-${instance} ${token}
     timeout: 240

percona/pmm/pmm.go

@@ -224,7 +224,7 @@ func sidecarContainerV2(pgc *v2.PerconaPGCluster) corev1.Container {
 			},
 			{
 				Name:  "PMM_AGENT_PRERUN_SCRIPT",
-				Value: agentPrerunScript(pgc.Spec.PMM.QuerySource, pgc),
+				Value: agentPrerunScript(pgc, false),
 			},
 			{
 				Name:  "PMM_AGENT_PATHS_TEMPDIR",
@@ -407,7 +407,7 @@ func sidecarContainerV3(pgc *v2.PerconaPGCluster) corev1.Container {
 			},
 			{
 				Name:  "PMM_AGENT_PRERUN_SCRIPT",
-				Value: agentPrerunScript(pgc.Spec.PMM.QuerySource, pgc),
+				Value: agentPrerunScript(pgc, true),
 			},
 			{
 				Name:  "PMM_AGENT_PATHS_TEMPDIR",
@@ -419,7 +419,7 @@ func sidecarContainerV3(pgc *v2.PerconaPGCluster) corev1.Container {
 	return container
 }
 
-func agentPrerunScript(querySource v2.PMMQuerySource, pgc *v2.PerconaPGCluster) string {
+func agentPrerunScript(pgc *v2.PerconaPGCluster, isPMM3 bool) string {
 	wait := "pmm-admin status --wait=10s"
 	annotate := "pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted'"
 
@@ -435,7 +435,7 @@ func agentPrerunScript(querySource v2.PMMQuerySource, pgc *v2.PerconaPGCluster)
 		"--skip-connection-check",
 		"--metrics-mode=push",
 		"--service-name=$(PMM_AGENT_SETUP_NODE_NAME)",
-		fmt.Sprintf("--query-source=%s", querySource),
+		fmt.Sprintf("--query-source=%s", pgc.Spec.PMM.QuerySource),
 	}
 
 	if pgc.CompareVersion("2.7.0") >= 0 {
@@ -449,5 +449,11 @@ func agentPrerunScript(querySource v2.PMMQuerySource, pgc *v2.PerconaPGCluster)
 	}
 	addService := fmt.Sprintf("pmm-admin add postgresql %s", strings.Join(addServiceArgs, " "))
 
+	if pgc.CompareVersion("2.8.0") >= 0 && isPMM3 {
+		addPatroniMetrics := fmt.Sprintf(
+			"pmm-admin add external --scheme=https --listen-port=%d --tls-skip-verify --service-name=$(PMM_AGENT_SETUP_NODE_NAME)-patroni-external", pgc.Spec.Patroni.GetPort())
+		return fmt.Sprintf("%s; %s; %s; %s", wait, addService, addPatroniMetrics, annotate)
+	}
+
 	return wait + "; " + addService + "; " + annotate
 }

percona/pmm/pmm_test.go

@@ -321,7 +321,7 @@ func TestSidecarContainerV3(t *testing.T) {
 		"DB_TYPE":                       "postgresql",
 		"DB_USER":                       v2.UserMonitoring,
 		"DB_PASS":                       "", // secret reference is asserted separately
-		"PMM_AGENT_PRERUN_SCRIPT":       "pmm-admin status --wait=10s; pmm-admin add postgresql --username=$(DB_USER) --password='$(DB_PASS)' --host=127.0.0.1 --port=5432 --tls-cert-file=/pgconf/tls/tls.crt --tls-key-file=/pgconf/tls/tls.key --tls-ca-file=/pgconf/tls/ca.crt --tls-skip-verify --skip-connection-check --metrics-mode=push --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --query-source= --cluster=test-cluster --environment=dev-postgres; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted'",
+		"PMM_AGENT_PRERUN_SCRIPT":       "pmm-admin status --wait=10s; pmm-admin add postgresql --username=$(DB_USER) --password='$(DB_PASS)' --host=127.0.0.1 --port=5432 --tls-cert-file=/pgconf/tls/tls.crt --tls-key-file=/pgconf/tls/tls.key --tls-ca-file=/pgconf/tls/ca.crt --tls-skip-verify --skip-connection-check --metrics-mode=push --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --query-source= --cluster=test-cluster --environment=dev-postgres; pmm-admin add external --scheme=https --listen-port=8008 --tls-skip-verify --service-name=$(PMM_AGENT_SETUP_NODE_NAME)-patroni-external; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted'",
 		"PMM_AGENT_PATHS_TEMPDIR":       "/tmp/pmm",
 	}
 

pkg/apis/postgres-operator.crunchydata.com/v1beta1/patroni_types.go

@@ -83,6 +83,9 @@ type PatroniSwitchover struct {
 const (
 	PatroniSwitchoverTypeFailover   = "Failover"
 	PatroniSwitchoverTypeSwitchover = "Switchover"
+
+	// K8SPG-718
+	patroniDefaultPort = int32(8008)
 )
 
 // Default sets the default values for certain Patroni configuration attributes,
@@ -105,6 +108,15 @@ func (s *PatroniSpec) Default() {
 	}
 }
 
+// GetPort returns the patroni port. Added as part of K8SPG-718.
+func (s *PatroniSpec) GetPort() int32 {
+	patroniPort := patroniDefaultPort
+	if s != nil && s.Port != nil {
+		patroniPort = *s.Port
+	}
+	return patroniPort
+}
+
 type PatroniStatus struct {
 
 	// - "database_system_identifier" of https://github.com/zalando/patroni/blob/v2.0.1/docs/rest_api.rst#monitoring-endpoint