Skip to content

Commit

Permalink
Merge pull request #889 from sanchezl/point-to-point-names
Browse files Browse the repository at this point in the history 
add target endpoint description to check name
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot committed Jun 26, 2020
2 parents f73bebb + 2c7c22c commit 0c87c5b
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 33 deletions.
20 changes: 11 additions & 9 deletions pkg/cmd/checkendpoints/controller/connection_checker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net"
"regexp"
"sync"
"time"

Expand Down Expand Up @@ -110,7 +111,7 @@ func (c *connectionChecker) updateStatus(ctx context.Context) {
// checkEndpoint performs the check and manages the PodNetworkConnectivityCheck.Status changes that result.
func (c *connectionChecker) checkEndpoint(ctx context.Context, check *operatorcontrolplanev1alpha1.PodNetworkConnectivityCheck) {
latencyInfo, err := getTCPConnectLatency(ctx, check.Spec.TargetEndpoint)
statusUpdates := manageStatusLogs(check.Spec.TargetEndpoint, err, latencyInfo)
statusUpdates := manageStatusLogs(check, err, latencyInfo)
if len(statusUpdates) > 0 {
statusUpdates = append(statusUpdates, manageStatusOutage(c.recorder))
}
Expand Down Expand Up @@ -145,16 +146,17 @@ func isDNSError(err error) bool {

// manageStatusLogs returns a status update function that updates the PodNetworkConnectivityCheck.Status's
// Successes/Failures logs reflect the results of the check.
func manageStatusLogs(address string, checkErr error, latency *trace.LatencyInfo) []v1alpha1helpers.UpdateStatusFunc {
func manageStatusLogs(check *operatorcontrolplanev1alpha1.PodNetworkConnectivityCheck, checkErr error, latency *trace.LatencyInfo) []v1alpha1helpers.UpdateStatusFunc {
var statusUpdates []v1alpha1helpers.UpdateStatusFunc
host, _, _ := net.SplitHostPort(address)
description := regexp.MustCompile(".*-to-").ReplaceAllString(check.Name, "")
host, _, _ := net.SplitHostPort(check.Spec.TargetEndpoint)
if isDNSError(checkErr) {
klog.V(2).Infof("%7s | %-15s | %10s | Failure looking up host %s: %v", "Failure", "DNSError", latency.DNS, host, checkErr)
return append(statusUpdates, v1alpha1helpers.AddFailureLogEntry(operatorcontrolplanev1alpha1.LogEntry{
Start: metav1.NewTime(latency.DNSStart),
Success: false,
Reason: operatorcontrolplanev1alpha1.LogEntryReasonDNSError,
Message: fmt.Sprintf("Failure looking up host %s: %v", host, checkErr),
Message: fmt.Sprintf("%s: failure looking up host %s: %v", description, host, checkErr),
Latency: metav1.Duration{Duration: latency.DNS},
}))
}
Expand All @@ -164,26 +166,26 @@ func manageStatusLogs(address string, checkErr error, latency *trace.LatencyInfo
Start: metav1.NewTime(latency.DNSStart),
Success: true,
Reason: operatorcontrolplanev1alpha1.LogEntryReasonDNSResolve,
Message: fmt.Sprintf("Resolved host name %s successfully", host),
Message: fmt.Sprintf("%s: resolved host name %s successfully", description, host),
Latency: metav1.Duration{Duration: latency.DNS},
}))
}
if checkErr != nil {
klog.V(2).Infof("%7s | %-15s | %10s | Failed to establish a TCP connection to %s: %v", "Failure", "TCPConnectError", latency.Connect, address, checkErr)
klog.V(2).Infof("%7s | %-15s | %10s | Failed to establish a TCP connection to %s: %v", "Failure", "TCPConnectError", latency.Connect, check.Spec.TargetEndpoint, checkErr)
return append(statusUpdates, v1alpha1helpers.AddFailureLogEntry(operatorcontrolplanev1alpha1.LogEntry{
Start: metav1.NewTime(latency.ConnectStart),
Success: false,
Reason: operatorcontrolplanev1alpha1.LogEntryReasonTCPConnectError,
Message: fmt.Sprintf("Failed to establish a TCP connection to %s: %v", address, checkErr),
Message: fmt.Sprintf("%s: failed to establish a TCP connection to %s: %v", description, check.Spec.TargetEndpoint, checkErr),
Latency: metav1.Duration{Duration: latency.Connect},
}))
}
klog.V(2).Infof("%7s | %-15s | %10s | TCP connection to %v succeeded", "Success", "TCPConnect", latency.Connect, address)
klog.V(2).Infof("%7s | %-15s | %10s | TCP connection to %v succeeded", "Success", "TCPConnect", latency.Connect, check.Spec.TargetEndpoint)
return append(statusUpdates, v1alpha1helpers.AddSuccessLogEntry(operatorcontrolplanev1alpha1.LogEntry{
Start: metav1.NewTime(latency.ConnectStart),
Success: true,
Reason: operatorcontrolplanev1alpha1.LogEntryReasonTCPConnect,
Message: fmt.Sprintf("TCP connection to %s succeeded", address),
Message: fmt.Sprintf("%s: tcp connection to %s succeeded", description, check.Spec.TargetEndpoint),
Latency: metav1.Duration{Duration: latency.Connect},
}))
}
Expand Down
17 changes: 12 additions & 5 deletions pkg/cmd/checkendpoints/controller/connection_checker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,14 @@ func TestManageStatusLogs(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
status := tc.initial
updateStatusFuncs := manageStatusLogs("host:port", tc.err, tc.trace)
updateStatusFuncs := manageStatusLogs(&v1alpha1.PodNetworkConnectivityCheck{
ObjectMeta: metav1.ObjectMeta{
Name: "test-to-target-endpoint",
},
Spec: v1alpha1.PodNetworkConnectivityCheckSpec{
TargetEndpoint: "host:port",
},
}, tc.err, tc.trace)
for _, updateStatusFunc := range updateStatusFuncs {
updateStatusFunc(status)
}
Expand Down Expand Up @@ -238,19 +245,19 @@ func podNetworkConnectivityCheckStatus(options ...func(status *v1alpha1.PodNetwo
}

func withTCPConnectErrorEntry(start int, options ...func(entry *v1alpha1.LogEntry)) func(status *v1alpha1.PodNetworkConnectivityCheckStatus) {
return withFailureEntry(start, v1alpha1.LogEntryReasonTCPConnectError, "Failed to establish a TCP connection to host:port: connect tcp: test error", options...)
return withFailureEntry(start, v1alpha1.LogEntryReasonTCPConnectError, "target-endpoint: failed to establish a TCP connection to host:port: connect tcp: test error", options...)
}

func withDNSErrorEntry(start int, options ...func(entry *v1alpha1.LogEntry)) func(status *v1alpha1.PodNetworkConnectivityCheckStatus) {
return withFailureEntry(start, v1alpha1.LogEntryReasonDNSError, "Failure looking up host host: connect tcp: lookup host: test error", options...)
return withFailureEntry(start, v1alpha1.LogEntryReasonDNSError, "target-endpoint: failure looking up host host: connect tcp: lookup host: test error", options...)
}

func withDNSResolveEntry(start int, options ...func(entry *v1alpha1.LogEntry)) func(status *v1alpha1.PodNetworkConnectivityCheckStatus) {
return withSuccessEntry(start, v1alpha1.LogEntryReasonDNSResolve, "Resolved host name host successfully", options...)
return withSuccessEntry(start, v1alpha1.LogEntryReasonDNSResolve, "target-endpoint: resolved host name host successfully", options...)
}

func withTCPConnectEntry(start int, options ...func(entry *v1alpha1.LogEntry)) func(status *v1alpha1.PodNetworkConnectivityCheckStatus) {
return withSuccessEntry(start, v1alpha1.LogEntryReasonTCPConnect, "TCP connection to host:port succeeded", options...)
return withSuccessEntry(start, v1alpha1.LogEntryReasonTCPConnect, "target-endpoint: tcp connection to host:port succeeded", options...)
}

func withSuccessEntry(start int, reason, message string, options ...func(entry *v1alpha1.LogEntry)) func(status *v1alpha1.PodNetworkConnectivityCheckStatus) {
Expand Down
68 changes: 49 additions & 19 deletions pkg/operator/connectivitycheckcontroller/connectivity_check_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"net/url"
"regexp"
"strings"

"github.com/ghodss/yaml"
operatorv1 "github.com/openshift/api/operator/v1"
Expand Down Expand Up @@ -86,13 +87,13 @@ func managePodNetworkConnectivityChecks(ctx context.Context, client kubernetes.I
operatorSpec *operatorv1.StaticPodOperatorSpec, endpointsLister corev1listers.EndpointsLister,
serviceLister corev1listers.ServiceLister, recorder events.Recorder) {

var addresses []string
// each etcd
addresses = append(addresses, listAddressesForEtcd(operatorSpec, recorder)...)
var templates []*v1alpha1.PodNetworkConnectivityCheck
// each storage endpoint
templates = append(templates, getTemplatesForStorageEndpoints(operatorSpec, recorder)...)
// oas service IP
addresses = append(addresses, listAddressesForOpenShiftAPIServerService(serviceLister, recorder)...)
templates = append(templates, getTemplatesForOpenShiftAPIServerService(serviceLister, recorder)...)
// each oas endpoint
addresses = append(addresses, listAddressesForOpenShiftAPIServerServiceEndpoints(endpointsLister, recorder)...)
templates = append(templates, getTemplatesForOpenShiftAPIServerServiceEndpoints(endpointsLister, recorder)...)

nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{
LabelSelector: labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector().String(),
Expand All @@ -104,18 +105,11 @@ func managePodNetworkConnectivityChecks(ctx context.Context, client kubernetes.I
var checks []*v1alpha1.PodNetworkConnectivityCheck
for _, node := range nodes.Items {
staticPodName := "kube-apiserver-" + node.Name
for _, address := range addresses {
checkName := staticPodName + "-" + regexp.MustCompile(`[.:\[\]]+`).ReplaceAllLiteralString(address, "-")
checks = append(checks, &v1alpha1.PodNetworkConnectivityCheck{
ObjectMeta: metav1.ObjectMeta{
Name: checkName,
Namespace: operatorclient.TargetNamespace,
},
Spec: v1alpha1.PodNetworkConnectivityCheckSpec{
SourcePod: staticPodName,
TargetEndpoint: address,
},
})
for _, template := range templates {
check := template.DeepCopy()
check.Name = strings.Replace(check.Name, "$(SOURCE)", staticPodName, -1)
check.Spec.SourcePod = staticPodName
checks = append(checks, check)
}
}

Expand All @@ -137,6 +131,14 @@ func managePodNetworkConnectivityChecks(ctx context.Context, client kubernetes.I
}
}

func getTemplatesForOpenShiftAPIServerService(serviceLister corev1listers.ServiceLister, recorder events.Recorder) []*v1alpha1.PodNetworkConnectivityCheck {
var templates []*v1alpha1.PodNetworkConnectivityCheck
for _, address := range listAddressesForOpenShiftAPIServerService(serviceLister, recorder) {
templates = append(templates, newPodNetworkProductivityCheck("openshift-apiserver-service", address))
}
return templates
}

func listAddressesForOpenShiftAPIServerService(serviceLister corev1listers.ServiceLister, recorder events.Recorder) []string {
service, err := serviceLister.Services("openshift-apiserver").Get("api")
if err != nil {
Expand All @@ -149,10 +151,18 @@ func listAddressesForOpenShiftAPIServerService(serviceLister corev1listers.Servi
return []string{fmt.Sprintf("%s:%d", service.Spec.ClusterIP, service.Spec.Ports[0].Port)}
}

func getTemplatesForOpenShiftAPIServerServiceEndpoints(endpointsLister corev1listers.EndpointsLister, recorder events.Recorder) []*v1alpha1.PodNetworkConnectivityCheck {
var templates []*v1alpha1.PodNetworkConnectivityCheck
for _, address := range listAddressesForOpenShiftAPIServerServiceEndpoints(endpointsLister, recorder) {
templates = append(templates, newPodNetworkProductivityCheck("openshift-apiserver-service-endpoint", address))
}
return templates
}

// listAddressesForOpenShiftAPIServerServiceEndpoints returns oas api service endpoints ip
func listAddressesForOpenShiftAPIServerServiceEndpoints(endpointsLister corev1listers.EndpointsLister, recorder events.Recorder) []string {
var results []string
endpoints, err := endpointsLister.Endpoints("openshift-apiserver").Get("api")
endpoints, err := endpointsLister.Endpoints("openshift-apiserver-service").Get("api")
if err != nil {
recorder.Warningf("EndpointDetectionFailure", "unable to determine openshift-apiserver endpoints: %v", err)
return nil
Expand All @@ -167,7 +177,15 @@ func listAddressesForOpenShiftAPIServerServiceEndpoints(endpointsLister corev1li
return results
}

func listAddressesForEtcd(operatorSpec *operatorv1.StaticPodOperatorSpec, recorder events.Recorder) []string {
func getTemplatesForStorageEndpoints(operatorSpec *operatorv1.StaticPodOperatorSpec, recorder events.Recorder) []*v1alpha1.PodNetworkConnectivityCheck {
var templates []*v1alpha1.PodNetworkConnectivityCheck
for _, address := range listAddressesForStorageEndpoints(operatorSpec, recorder) {
templates = append(templates, newPodNetworkProductivityCheck("storage-endpoint", address))
}
return templates
}

func listAddressesForStorageEndpoints(operatorSpec *operatorv1.StaticPodOperatorSpec, recorder events.Recorder) []string {
var results []string
var observedConfig map[string]interface{}
if err := yaml.Unmarshal(operatorSpec.ObservedConfig.Raw, &observedConfig); err != nil {
Expand All @@ -189,3 +207,15 @@ func listAddressesForEtcd(operatorSpec *operatorv1.StaticPodOperatorSpec, record
}
return results
}

func newPodNetworkProductivityCheck(label, address string) *v1alpha1.PodNetworkConnectivityCheck {
return &v1alpha1.PodNetworkConnectivityCheck{
ObjectMeta: metav1.ObjectMeta{
Name: "$(SOURCE)-to-" + label + "-" + regexp.MustCompile(`[.:\[\]]+`).ReplaceAllLiteralString(address, "-"),
Namespace: operatorclient.TargetNamespace,
},
Spec: v1alpha1.PodNetworkConnectivityCheckSpec{
TargetEndpoint: address,
},
}
}

0 comments on commit 0c87c5b

Please sign in to comment.