feat(operator): added Dynatrace DQL provider (#783)

Signed-off-by: Giovanni Liva <giovanni.liva@dynatrace.com>
Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>
Co-authored-by: Giovanni Liva <giovanni.liva@dynatrace.com>

operator/controllers/common/providers/common.go

@@ -6,6 +6,7 @@ import (
 )
 
 const DynatraceProviderName = "dynatrace"
+const DynatraceDQLProviderName = "dql"
 const PrometheusProviderName = "prometheus"
 const KeptnMetricProviderName = "keptn-metric"
 

operator/controllers/common/providers/dynatrace/client/client.go

@@ -0,0 +1,135 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/go-logr/logr"
+	"k8s.io/klog/v2"
+)
+
+//go:generate moq -pkg fake --skip-ensure -out ./fake/dt_client_mock.go . DTAPIClient
+type DTAPIClient interface {
+	Do(ctx context.Context, path, method string, payload []byte) ([]byte, error)
+}
+
+type apiClient struct {
+	Log        logr.Logger
+	httpClient http.Client
+	config     apiConfig
+}
+
+type APIClientOption func(client *apiClient)
+
+// WithLogger injects the given logger into an APIClient
+func WithLogger(logger logr.Logger) APIClientOption {
+	return func(client *apiClient) {
+		client.Log = logger
+	}
+}
+
+// WithHTTPClient injects the given HTTP client into an APIClient
+func WithHTTPClient(httpClient http.Client) APIClientOption {
+	return func(client *apiClient) {
+		client.httpClient = httpClient
+	}
+}
+
+// NewAPIClient creates and returns a new APIClient
+func NewAPIClient(config apiConfig, options ...APIClientOption) *apiClient {
+	client := &apiClient{
+		Log:        logr.New(klog.NewKlogr().GetSink()),
+		httpClient: http.Client{},
+		config:     config,
+	}
+
+	for _, o := range options {
+		o(client)
+	}
+
+	return client
+}
+
+// Do sends and API request to the Dynatrace API and returns its result as a string containing the raw response payload
+func (client *apiClient) Do(ctx context.Context, path, method string, payload []byte) ([]byte, error) {
+	if err := client.auth(ctx); err != nil {
+		return nil, err
+	}
+	api := fmt.Sprintf("%s%s", client.config.serverURL, path)
+	req, err := http.NewRequestWithContext(ctx, method, api, bytes.NewBuffer(payload))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", client.config.oAuthCredentials.accessToken))
+
+	res, err := client.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err := res.Body.Close()
+		if err != nil {
+			client.Log.Error(err, "Could not close request body")
+		}
+	}()
+	if isErrorStatus(res.StatusCode) {
+		return nil, ErrRequestFailed
+	}
+	b, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (client *apiClient) auth(ctx context.Context) error {
+	// return if we already have a token
+	if client.config.oAuthCredentials.accessToken != "" {
+		return nil
+	}
+	client.Log.V(10).Info("OAuth login")
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	values := client.config.oAuthCredentials.urlValues()
+	body := []byte(values.Encode())
+
+	req, err := http.NewRequestWithContext(ctx, "POST", client.config.authURL, bytes.NewBuffer(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	res, err := client.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := res.Body.Close()
+		if err != nil {
+			client.Log.Error(err, "Could not close request body")
+		}
+	}()
+	if isErrorStatus(res.StatusCode) {
+		return ErrRequestFailed
+	}
+	// we ignore the error here because we fail later while unmarshalling
+	oauthResponse := OAuthResponse{}
+	b, _ := io.ReadAll(res.Body)
+	err = json.Unmarshal(b, &oauthResponse)
+	if err != nil {
+		return err
+	}
+
+	if oauthResponse.AccessToken == "" {
+		return ErrAuthenticationFailed
+	}
+
+	client.config.oAuthCredentials.accessToken = oauthResponse.AccessToken
+	return nil
+}

operator/controllers/common/providers/dynatrace/client/client_test.go

@@ -0,0 +1,170 @@
+package client
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/go-logr/logr"
+	"github.com/stretchr/testify/require"
+	"k8s.io/klog/v2"
+)
+
+const mockSecret = "dt0s08.XX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+func TestNewConfigInvalidSecretFormat(t *testing.T) {
+
+	config, err := NewAPIConfig("", "my-secret")
+
+	require.ErrorIs(t, err, ErrClientSecretInvalid)
+	require.Nil(t, config)
+}
+
+func TestAPIClient(t *testing.T) {
+
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.URL.Path == "/auth" {
+			_, _ = writer.Write([]byte(`{"access_token": "my-token"}`))
+			return
+		}
+		_, _ = writer.Write([]byte("success"))
+	}))
+
+	defer server.Close()
+
+	config, err := NewAPIConfig(
+		server.URL,
+		mockSecret,
+		WithScopes([]OAuthScope{OAuthScopeStorageMetricsRead, OAuthScopeEnvironmentRoleViewer}),
+		WithAuthURL(server.URL+"/auth"),
+	)
+
+	require.Nil(t, err)
+	require.NotNil(t, config)
+
+	apiClient := NewAPIClient(
+		*config,
+		WithHTTPClient(http.Client{}),
+		WithLogger(logr.New(klog.NewKlogr().GetSink())),
+	)
+
+	require.NotNil(t, apiClient)
+
+	resp, err := apiClient.Do(context.TODO(), "/query", http.MethodPost, nil)
+
+	require.Nil(t, err)
+	require.Equal(t, "success", string(resp))
+
+	require.Equal(t, "my-token", apiClient.config.oAuthCredentials.accessToken)
+}
+
+func TestAPIClientAuthError(t *testing.T) {
+
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		writer.WriteHeader(http.StatusInternalServerError)
+	}))
+
+	defer server.Close()
+
+	mockSecret := "dt0s08.XX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+	config, err := NewAPIConfig(
+		server.URL,
+		mockSecret,
+		WithAuthURL(server.URL+"/auth"),
+	)
+
+	require.Nil(t, err)
+	require.NotNil(t, config)
+
+	apiClient := NewAPIClient(
+		*config,
+		WithHTTPClient(http.Client{}),
+		WithLogger(logr.New(klog.NewKlogr().GetSink())),
+	)
+
+	require.NotNil(t, apiClient)
+
+	resp, err := apiClient.Do(context.TODO(), "/query", http.MethodPost, nil)
+
+	require.ErrorIs(t, err, ErrRequestFailed)
+	require.Empty(t, resp)
+}
+
+func TestAPIClientAuthNoToken(t *testing.T) {
+
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.URL.Path == "/auth" {
+			_, _ = writer.Write([]byte(`{"something": "else"}`))
+			return
+		}
+		_, _ = writer.Write([]byte("success"))
+	}))
+
+	defer server.Close()
+
+	mockSecret := "dt0s08.XX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+	config, err := NewAPIConfig(
+		server.URL,
+		mockSecret,
+		WithAuthURL(server.URL+"/auth"),
+	)
+
+	require.Nil(t, err)
+	require.NotNil(t, config)
+
+	apiClient := NewAPIClient(
+		*config,
+		WithHTTPClient(http.Client{}),
+		WithLogger(logr.New(klog.NewKlogr().GetSink())),
+	)
+
+	require.NotNil(t, apiClient)
+
+	resp, err := apiClient.Do(context.TODO(), "/query", http.MethodPost, nil)
+
+	require.ErrorIs(t, err, ErrAuthenticationFailed)
+	require.Empty(t, resp)
+}
+
+func TestAPIClientRequestError(t *testing.T) {
+
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.URL.Path == "/auth" {
+			_, _ = writer.Write([]byte(`{"access_token": "my-token"}`))
+			return
+		}
+		writer.WriteHeader(http.StatusInternalServerError)
+	}))
+
+	defer server.Close()
+
+	mockSecret := "dt0s08.XX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+	config, err := NewAPIConfig(
+		server.URL,
+		mockSecret,
+		WithAuthURL(server.URL+"/auth"),
+	)
+
+	require.Nil(t, err)
+	require.NotNil(t, config)
+
+	apiClient := NewAPIClient(
+		*config,
+		WithHTTPClient(http.Client{}),
+		WithLogger(logr.New(klog.NewKlogr().GetSink())),
+	)
+
+	require.NotNil(t, apiClient)
+
+	resp, err := apiClient.Do(context.TODO(), "/query", http.MethodPost, nil)
+
+	// authentication should have worked
+	require.Equal(t, "my-token", apiClient.config.oAuthCredentials.accessToken)
+
+	require.ErrorIs(t, err, ErrRequestFailed)
+	require.Empty(t, resp)
+}

operator/controllers/common/providers/dynatrace/client/common.go

@@ -0,0 +1,45 @@
+package client
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var ErrClientSecretInvalid = errors.New("the Dynatrace token has an invalid format")
+var ErrRequestFailed = errors.New("the API returned a response with a status outside of the 2xx range")
+var ErrAuthenticationFailed = errors.New("could not retrieve an OAuth token from the API")
+
+const (
+	defaultAuthURL                  = "https://sso-dev.dynatracelabs.com/sso/oauth2/token"
+	oAuthGrantType                  = "grant_type"
+	oAuthGrantTypeClientCredentials = "client_credentials"
+	oAuthScope                      = "scope"
+	oAuthClientID                   = "client_id"
+	oAuthClientSecret               = "client_secret"
+)
+
+const dtTokenPrefix = "dt0s08"
+
+func validateOAuthSecret(token string) error {
+	// must start with dt0s08
+	// must have 2 dots
+	// third part (split by dot) must be 64 chars
+	if !strings.HasPrefix(token, dtTokenPrefix) {
+		return fmt.Errorf("secret does not start with required prefix %s: %w", dtTokenPrefix, ErrClientSecretInvalid)
+	}
+	split := strings.Split(token, ".")
+	if len(split) != 3 {
+		return fmt.Errorf("secret does not contain three components: %w", ErrClientSecretInvalid)
+	}
+	secret := split[2]
+	if secretLen := len(secret); secretLen != 64 {
+		return fmt.Errorf("length of secret is not equal to 64: %w", ErrClientSecretInvalid)
+	}
+	return nil
+}
+
+func isErrorStatus(statusCode int) bool {
+	return statusCode < 200 || statusCode >= 300
+}