discovery Maia service using keystone catalog

* Maia CLI searches Keystone catalog for public Maia service when neither --prometheus-url nor --maia-url are specified but --os-auth-url is
* misc. improvements to docu.

README.md

@@ -1,7 +1,7 @@
 # Maia
 
-Maia is a multi-tenant OpenStack-service for accessing metrics and alarms collected through Prometheus. It offers a [Prometheus-compatible](https://prometheus.io/docs/querying/api/) 
-API and supports federation.
+Maia is a multi-tenant OpenStack-service for accessing metrics and alarms collected through Prometheus. It offers 
+a [Prometheus-compatible](https://prometheus.io/docs/querying/api/) API and supports federation.
 
 At SAP we use it to share tenant-specific metrics from our Converged Cloud platform
 with our users. For their convenience we included a CLI, so that metrics can be discovered and
@@ -25,17 +25,20 @@ Maia CLI
 
 ## Concept
 
-Maia adds multi-tenant support to Prometheus by using dedicated labels to assign metrics to OpenStack projects and domains.
+Maia adds multi-tenant support to an existing Prometheus installation by using dedicated labels to assign metrics to
+OpenStack projects and domains. These labels either have to be supplied by the exporters or they have to be
+mapped from other labels using the [Prometheus relabelling](https://prometheus.io/docs/operating/configuration/#relabel_config)
+capabilities.
 
-The following labels have a special meaning in Maia. *Only metrics with these labels are visible though the Maia API.*
+The following labels have a special meaning in Maia. *Only metrics with these labels are visible through the Maia API.*
 
  | Label Key  | Description  |
  |------------|--------------|
  | project_id | OpenStack project UUID |
  | domain_id  | OpenStack domain UUID |
 
-Metrics without `domain_id` will not be available when authorized to domain scope. Likewise, metrics without `project_id`
-will be omitted when project scope is used. There is no inheritance of metrics to parent projects. Users authorized
+Metrics without `project_id` will be omitted when project scope is used. Likewise, metrics without `domain_id` will not
+be available when authorized to domain scope. There is no inheritance of metrics to parent projects. Users authorized
 to a domain will be able to access the metrics of all projects in that domain that have been labelled for the domain.
 
 # Installation
@@ -144,6 +147,18 @@ this amount can be changed:
 token_cache_time = "3600s"
 ```
 
+## Available Exporters
+
+As explained in the Concept chapter, Maia requires all series to be labelled with OpenStack project_id resp. domain_id.
+
+The following exporters are known to produce suitible metrics:
+* [VCenter Exporter](https://github.com/sapcc/vcenter-exporter) provides project-specific metrics from an OpenStack-
+controlled VCenter. 
+* [SNMP Exporter](https://github.com/prometheus/snmp_exporter) can be configured to extract project IDs from
+SNMP variables into labels. Since most of the SNMP-enabled devices are shared, only a few metrics can be mapped to
+OpenStack projects or domains.
+
+
 ## Starting the Service
 
 Once you have finalized the configuration file, you are set to go

pkg/cmd/client.go

@@ -66,33 +66,39 @@ func recoverAll() {
 	}
 }
 
-func fetchToken() string {
+func fetchToken() {
 	if scopedDomain != "" {
 		auth.Scope.DomainName = scopedDomain
 	}
 	// authenticate calls to Maia
-	if auth.TokenID != "" {
-		return auth.TokenID
+	if auth.TokenID != "" && maiaURL != "" {
+		return
 	}
 	if auth.Password == "$OS_PASSWORD" {
 		auth.Password = os.Getenv("OS_PASSWORD")
 	}
 	if (auth.Username == "" && auth.UserID == "") || auth.Password == "" {
 		panic(fmt.Errorf("You must at least specify --os-username / --os-user-id and --os-password"))
 	}
-	context, err := keystoneInstance().Authenticate(&auth)
+	context, url, err := keystoneInstance().Authenticate(&auth)
 	if err != nil {
 		panic(err)
 	}
-	return context.Auth["token"]
+	auth.TokenID = context.Auth["token"]
+	if maiaURL == "" {
+		maiaURL = url
+	}
 }
 
 func storageInstance() storage.Driver {
 	if storageDriver == nil {
-		if maiaURL != "" {
-			storageDriver = storage.NewPrometheusDriver(maiaURL, map[string]string{"X-Auth-Token": fetchToken()})
-		} else if promURL != "" {
+		if promURL != "" {
 			storageDriver = storage.NewPrometheusDriver(promURL, map[string]string{})
+		} else if auth.IdentityEndpoint != "" {
+			// authenticate and set maiaURL if missing
+			fetchToken()
+			storageDriver = storage.NewPrometheusDriver(maiaURL, map[string]string{"X-Auth-Token": auth.TokenID})
+		} else if promURL != "" {
 		} else {
 			panic(fmt.Errorf("Either --maia-url or --storageInstance-url need to be specified (or MAIA_URL resp. MAIA_PROMETHEUS_URL)"))
 		}

pkg/keystone/interface.go

@@ -33,11 +33,12 @@ import (
 type Driver interface {
 	// AuthenticateRequest authenticates a user using authOptionsFromRequest passed in the HTTP request header.
 	// After successful authentication, additional context information is added to the request header
-	// In addition a Context object is returned.
+	// In addition a Context object is returned for policy evaluation.
 	AuthenticateRequest(req *http.Request) (*policy.Context, error)
 
-	// Authenticate authenticates a user using the provided authOptionsFromRequest
-	Authenticate(options *tokens.AuthOptions) (*policy.Context, error)
+	// Authenticate authenticates a user using the provided authOptions.
+	// It returns a context for policy evaluation and the public endpoint retrieved from the service catalog
+	Authenticate(options *tokens.AuthOptions) (*policy.Context, string, error)
 }
 
 // NewKeystoneDriver is a factory method which chooses the right driver implementation based on configuration settings

pkg/keystone/keystone.go

@@ -53,7 +53,6 @@ type keystone struct {
 	mutex          *sync.Mutex
 	tokenCache     *cache.Cache
 	providerClient *gophercloud.ServiceClient
-	catalog        *tokens.ServiceCatalog
 	seqErrors      int
 }
 
@@ -149,13 +148,13 @@ func (d *keystone) reauthServiceUser() error {
 	result := tokens.Create(d.providerClient, authOpts)
 	token, err := result.ExtractToken()
 	if err != nil {
-		// wait up-to (2^errors)/2, i.e. 0..1, 2, 4, ... increasing with every sequential error
+		// wait ~ (2^errors)/2, i.e. 0..1, 0..2, 0..4, ... increasing with every sequential error
 		r := rand.Intn(int(math.Exp2(float64(d.seqErrors))))
 		time.Sleep(time.Duration(r) * time.Second)
 		d.seqErrors++
 		return fmt.Errorf("Cannot obtain token: %v (%d sequential errors)", err, d.seqErrors)
 	}
-	d.catalog, err = result.ExtractServiceCatalog()
+	catalog, err := result.ExtractServiceCatalog()
 	if err != nil {
 		return fmt.Errorf("cannot read service catalog: %v", err)
 	}
@@ -165,7 +164,7 @@ func (d *keystone) reauthServiceUser() error {
 	d.providerClient.TokenID = token.ID
 	d.providerClient.ReauthFunc = d.reauthServiceUser
 	d.providerClient.EndpointLocator = func(opts gophercloud.EndpointOpts) (string, error) {
-		return openstack.V3EndpointURL(d.catalog, opts)
+		return openstack.V3EndpointURL(catalog, opts)
 	}
 
 	return nil
@@ -201,28 +200,29 @@ func authOpts2StringKey(authOpts *tokens.AuthOptions) string {
 
 // Authenticate authenticates a non-service user using available authOptionsFromRequest (username+password or token)
 // It returns the authorization context
-func (d *keystone) Authenticate(authOpts *tokens.AuthOptions) (*policy.Context, error) {
+func (d *keystone) Authenticate(authOpts *tokens.AuthOptions) (*policy.Context, string, error) {
 	return d.authenticate(authOpts, false)
 }
 
 // authenticate authenticates a user using available authOptionsFromRequest (username+password or token)
 // It returns the authorization context
-func (d *keystone) authenticate(authOpts *tokens.AuthOptions, asServiceUser bool) (*policy.Context, error) {
+func (d *keystone) authenticate(authOpts *tokens.AuthOptions, asServiceUser bool) (*policy.Context, string, error) {
 	// check cache briefly
 	if entry, found := d.tokenCache.Get(authOpts2StringKey(authOpts)); found {
 		util.LogDebug("Token cache hit for %s", authOpts.TokenID)
-		return entry.(*policy.Context), nil
+		return entry.(*policy.Context), "", nil
 	}
 
 	// get identity connection
 	client, err := d.keystoneClient()
-	if client == nil || err != nil {
+	if err != nil {
 		util.LogError(err.Error())
-		return nil, err
+		return nil, "", err
 	}
 
 	//use a custom token struct instead of tokens.Token which is way incomplete
 	var tokenData keystoneToken
+	var catalog *tokens.ServiceCatalog
 	if authOpts.TokenID != "" && asServiceUser {
 		util.LogDebug("verifying token")
 		// need an authenticated service user to check tokens
@@ -233,11 +233,15 @@ func (d *keystone) authenticate(authOpts *tokens.AuthOptions, asServiceUser bool
 		response := tokens.Get(client, authOpts.TokenID)
 		if response.Err != nil {
 			//this includes 4xx responses, so after this point, we can be sure that the token is valid
-			return nil, response.Err
+			return nil, "", response.Err
 		}
 		err = response.ExtractInto(&tokenData)
 		if err != nil {
-			return nil, err
+			return nil, "", err
+		}
+		catalog, err = response.ExtractServiceCatalog()
+		if err != nil {
+			return nil, "", err
 		}
 	} else {
 		util.LogDebug("authenticate %s%s with scope %s.", authOpts.Username, authOpts.UserID, authOpts.Scope)
@@ -251,19 +255,25 @@ func (d *keystone) authenticate(authOpts *tokens.AuthOptions, asServiceUser bool
 			} else {
 				util.LogInfo("Failed login of with token %s ... for scope %s: %s", authOpts.TokenID[:1+len(authOpts.TokenID)/4], authOpts.Scope, response.Err.Error())
 			}
-			return nil, response.Err
+			return nil, "", response.Err
 		}
 		err = response.ExtractInto(&tokenData)
 		if err != nil {
-			return nil, err
+			return nil, "", err
+		}
+		catalog, err = response.ExtractServiceCatalog()
+		if err != nil {
+			return nil, "", err
 		}
 		// the token is passed separately
 		tokenData.Token = response.Header.Get("X-Subject-Token")
+		client.TokenID = tokenData.Token
 	}
 
 	context := tokenData.ToContext()
 	d.tokenCache.Set(authOpts2StringKey(authOpts), &context, cache.DefaultExpiration)
-	return &context, nil
+	endpointURL, err := openstack.V3EndpointURL(catalog, gophercloud.EndpointOpts{Type: "metrics", Availability: gophercloud.AvailabilityPublic})
+	return &context, endpointURL, nil
 }
 
 // AuthenticateRequest attempts to Authenticate a user using the request header contents
@@ -277,7 +287,7 @@ func (d *keystone) AuthenticateRequest(r *http.Request) (*policy.Context, error)
 		return nil, err
 	}
 
-	context, err := d.authenticate(authOpts, true)
+	context, _, err := d.authenticate(authOpts, true)
 	if err != nil {
 		return nil, err
 	}

pkg/keystone/mock.go

@@ -32,9 +32,9 @@ func Mock() Driver {
 	return mock{}
 }
 
-func (d mock) Authenticate(credentials *tokens.AuthOptions) (*policy.Context, error) {
+func (d mock) Authenticate(credentials *tokens.AuthOptions) (*policy.Context, string, error) {
 	return &policy.Context{Request: map[string]string{"user_id": credentials.UserID,
-		"project_id": credentials.Scope.ProjectID, "password": credentials.Password}}, nil
+		"project_id": credentials.Scope.ProjectID, "password": credentials.Password}}, "http://localhost:9091", nil
 }
 
 func (d mock) AuthenticateRequest(req *http.Request) (*policy.Context, error) {