test/e2e: improve e2e test resiliency

We have a couple of potential e2e flake sources:

1. We don't check the response code from the prometheus query response.

That causes the vanilla http response in case of error to be parsed as JSON which fails.
We should bail out with the http status code instead.

2. It happens that we sometimes don't pick the correct token secret.

With every service account there are two token secrets.
One is the internal Kubernetes service account token,
the other one is associated with the internal OpenShift Docker registry.
We have to pick the right one otherwise non-deterministically the wrong token will be chosen.

This fixes the above two issues.

test/e2e/framework/route_client.go

@@ -64,7 +64,12 @@ func NewRouteClient(
 	var token string
 
 	for _, secret := range secrets.Items {
-		if strings.Contains(secret.Name, "cluster-monitoring-operator-e2e-token-") {
+		_, dockerToken := secret.Annotations["openshift.io/create-dockercfg-secrets"]
+		e2eToken := strings.Contains(secret.Name, "cluster-monitoring-operator-e2e-token-")
+
+		// we have to skip the token secret that contains the openshift.io/create-dockercfg-secrets annotation
+		// as this is the token to talk to the internal registry.
+		if !dockerToken && e2eToken {
 			token = string(secret.Data["token"])
 		}
 	}
@@ -101,6 +106,10 @@ func (c *RouteClient) PrometheusQuery(query string) ([]byte, error) {
 		return nil, err
 	}
 
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code response, want %d, got %d", http.StatusOK, resp.StatusCode)
+	}
+
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)
@@ -139,6 +148,10 @@ func (c *RouteClient) AlertmanagerQuery(kvs ...string) ([]byte, error) {
 		return nil, err
 	}
 
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code response, want %d, got %d", http.StatusOK, resp.StatusCode)
+	}
+
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)

test/e2e/main_test.go

@@ -82,11 +82,13 @@ func testMain(m *testing.M) error {
 		)
 		body, loopErr = f.ThanosQuerierClient.PrometheusQuery("count(up{job=\"prometheus-k8s\"})")
 		if loopErr != nil {
+			loopErr = errors.Wrap(loopErr, "error executing prometheus query")
 			return false, nil
 		}
 
 		v, loopErr = framework.GetFirstValueFromPromQuery(body)
 		if loopErr != nil {
+			loopErr = errors.Wrapf(loopErr, "error getting first value from prometheus response %q", string(body))
 			return false, nil
 		}