Merge pull request #62 from sapcc/cfm

add CFM quota plugin

.gitignore

@@ -18,3 +18,4 @@ build/cover.html
 
 # custom configuration files used during development
 *.yaml
+*.json

docs/operators/config.md

@@ -223,6 +223,19 @@ behave like the `instances` resource. When subresources are scraped for the `ins
 scraped for these flavor-specific instance resources. The flavor-specific instance resources are in the `per_flavor`
 category.
 
+## `database`: SAP Cloud Frame Manager
+
+```yaml
+services:
+  - type: database
+```
+
+The area for this service is `storage`.
+
+| Resource | Unit |
+| --- | --- |
+| `cfm_share_capacity` | bytes |
+
 ## `dns`: Designate v2
 
 ```yaml

pkg/plugins/cfm.go

@@ -0,0 +1,256 @@
+/*******************************************************************************
+*
+* Copyright 2018 SAP SE
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You should have received a copy of the License along with this
+* program. If not, you may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*******************************************************************************/
+
+package plugins
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack"
+	"github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
+	"github.com/sapcc/go-bits/logg"
+	"github.com/sapcc/limes/pkg/limes"
+	"github.com/sapcc/limes/pkg/util"
+)
+
+type cfmPlugin struct {
+	cfg limes.ServiceConfiguration
+
+	shareserversCache        []cfmShareserver
+	shareserversCacheExpires time.Time
+}
+
+func init() {
+	limes.RegisterQuotaPlugin(func(c limes.ServiceConfiguration, scrapeSubresources map[string]bool) limes.QuotaPlugin {
+		return &cfmPlugin{cfg: c}
+	})
+}
+
+//Init implements the limes.QuotaPlugin interface.
+func (p *cfmPlugin) Init(provider *gophercloud.ProviderClient) error {
+	return nil
+}
+
+//ServiceInfo implements the limes.QuotaPlugin interface.
+func (p *cfmPlugin) ServiceInfo() limes.ServiceInfo {
+	return limes.ServiceInfo{
+		Type:        "database",
+		ProductName: "cfm",
+		Area:        "storage",
+	}
+}
+
+//Resources implements the limes.QuotaPlugin interface.
+func (p *cfmPlugin) Resources() []limes.ResourceInfo {
+	return []limes.ResourceInfo{{
+		Name: "cfm_share_capacity",
+		Unit: limes.UnitBytes,
+		//we cannot set quota for this service
+		ExternallyManaged: true,
+	}}
+}
+
+//Scrape implements the limes.QuotaPlugin interface.
+func (p *cfmPlugin) Scrape(provider *gophercloud.ProviderClient, clusterID, domainUUID, projectUUID string) (map[string]limes.ResourceData, error) {
+	//cache the result of cfmListShareservers(), it's mildly expensive
+	now := time.Now()
+	if p.shareserversCache == nil || p.shareserversCacheExpires.Before(now) {
+		projectID, err := cfmGetScopedProjectID(provider)
+		if err != nil {
+			return nil, err
+		}
+		shareservers, err := cfmListShareservers(provider, projectID)
+		if err != nil {
+			return nil, err
+		}
+		p.shareserversCache = shareservers
+		p.shareserversCacheExpires = now.Add(5 * time.Minute)
+	}
+	shareservers := p.shareserversCache
+
+	var projectID string
+	result := limes.ResourceData{Quota: 0, Usage: 0}
+	for _, shareserver := range shareservers {
+		if shareserver.ProjectUUID != projectUUID {
+			continue
+		}
+
+		if projectID == "" {
+			var err error
+			projectID, err = cfmGetScopedProjectID(provider)
+			if err != nil {
+				return nil, err
+			}
+		}
+		shareserverDetailed, err := cfmGetShareserver(provider, shareserver.DetailsURL, projectID)
+		if err != nil {
+			return nil, err
+		}
+
+		result.Quota += int64(shareserverDetailed.MaximumSizeBytes)
+		result.Usage += shareserverDetailed.SizeBytes
+	}
+
+	return map[string]limes.ResourceData{"cfm_share_capacity": result}, nil
+}
+
+//SetQuota implements the limes.QuotaPlugin interface.
+func (p *cfmPlugin) SetQuota(provider *gophercloud.ProviderClient, clusterID, domainUUID, projectUUID string, quotas map[string]uint64) error {
+	if len(quotas) > 0 {
+		return errors.New("the database/cfm_share_capacity resource is externally managed")
+	}
+	return nil
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+func cfmGetScopedProjectID(provider *gophercloud.ProviderClient) (string, error) {
+	//the CFM API is stupid and needs the caller to provide the scope of the
+	//token redundantly in the X-Project-Id header
+	identityClient, err := openstack.NewIdentityV3(provider, gophercloud.EndpointOpts{})
+	if err != nil {
+		return "", err
+	}
+	project, err := tokens.Get(identityClient, provider.Token()).ExtractProject()
+	return project.ID, err
+}
+
+type cfmShareserver struct {
+	Type        string
+	ProjectUUID string
+	DetailsURL  string
+	//fields that are only filled by cfmGetShareserver, not by cfmListShareservers
+	SizeBytes        uint64
+	MaximumSizeBytes uint64
+}
+
+func cfmListShareservers(provider *gophercloud.ProviderClient, projectID string) ([]cfmShareserver, error) {
+	baseURL, err := provider.EndpointLocator(gophercloud.EndpointOpts{
+		Type:         "database",
+		Name:         "cfm",
+		Availability: gophercloud.AvailabilityPublic,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	url := strings.TrimSuffix(baseURL, "/") + "/v1.0/shareservers/"
+	var data struct {
+		Shareservers []struct {
+			Links       []gophercloud.Link `json:"links"`
+			Type        string             `json:"type"`
+			ProjectUUID string             `json:"customer_id"`
+		} `json:"shareservers"`
+	}
+	err = cfmDoRequest(provider, url, &data, projectID)
+	if err != nil {
+		return nil, fmt.Errorf("GET %s failed: %s", url, err.Error())
+	}
+
+	result := make([]cfmShareserver, len(data.Shareservers))
+	for idx, srv := range data.Shareservers {
+		result[idx] = cfmShareserver{
+			Type:        srv.Type,
+			ProjectUUID: srv.ProjectUUID,
+			DetailsURL:  srv.Links[0].Href,
+		}
+	}
+	return result, nil
+}
+
+func cfmGetShareserver(provider *gophercloud.ProviderClient, url string, projectID string) (*cfmShareserver, error) {
+	var data struct {
+		Shareserver struct {
+			ID         string `json:"id"`
+			Properties struct {
+				SizeBytes        util.CFMBytes `json:"size"`
+				MaximumSizeBytes util.CFMBytes `json:"maximum_size"`
+			} `json:"properties"`
+			Links       []gophercloud.Link `json:"links"`
+			Type        string             `json:"type"`
+			ProjectUUID string             `json:"customer_id"`
+		} `json:"shareserver"`
+	}
+	err := cfmDoRequest(provider, url, &data, projectID)
+	if err != nil {
+		return nil, fmt.Errorf("GET %s failed: %s", url, err.Error())
+	}
+
+	srv := data.Shareserver
+	logg.Info("CFM shareserver %s (type %s) in project %s has size = %d bytes, maximum_size = %d bytes",
+		srv.ID, srv.Type, srv.ProjectUUID,
+		srv.Properties.SizeBytes,
+		srv.Properties.MaximumSizeBytes,
+	)
+	return &cfmShareserver{
+		Type:             srv.Type,
+		ProjectUUID:      srv.ProjectUUID,
+		DetailsURL:       srv.Links[0].Href,
+		SizeBytes:        uint64(srv.Properties.SizeBytes),
+		MaximumSizeBytes: uint64(srv.Properties.MaximumSizeBytes),
+	}, nil
+}
+
+func cfmDoRequest(provider *gophercloud.ProviderClient, url string, body interface{}, projectID string) error {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+	token := provider.Token()
+	req.Header.Set("Authorization", "Token "+token)
+	req.Header.Set("X-Project-Id", projectID)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	//success case
+	if resp.StatusCode == http.StatusOK {
+		return json.NewDecoder(resp.Body).Decode(&body)
+	}
+
+	//error case: read error message from body
+	buf, err := ioutil.ReadAll(resp.Body)
+	if err == nil {
+		err = errors.New(string(buf))
+	}
+
+	//detect when token has expired
+	//
+	//NOTE: We don't trust the resp.StatusCode here. The CFM API is known to return
+	//403 when it means 401.
+	if strings.Contains(err.Error(), "Invalid credentials") {
+		err = provider.Reauthenticate(token)
+		if err == nil {
+			//restart function call after successful reauth
+			return cfmDoRequest(provider, url, body, projectID)
+		}
+	}
+
+	return err
+}

pkg/util/datatypes.go

@@ -1,6 +1,6 @@
 /*******************************************************************************
 *
-* Copyright 2017 SAP SE
+* Copyright 2017-2018 SAP SE
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -21,6 +21,8 @@ package util
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
 	"time"
 
 	"encoding/json"
@@ -74,3 +76,46 @@ func (s *JSONString) UnmarshalJSON(b []byte) error {
 	*s = JSONString(b)
 	return nil
 }
+
+//CFMBytes is a unsigned integer type with custom JSON demarshaling logic to
+//read strings like "0 bytes", "734.24 GB" or "1.95 TB" that appear in the CFM
+//API.
+//
+//NOTE: If we ever add base-1000 units to limes.Unit, consider using
+//limes.ValueWithUnit instead.
+type CFMBytes uint64
+
+var cfmUnits = map[string]float64{
+	"bytes": 1,
+	"KB":    1e3,
+	"MB":    1e6,
+	"GB":    1e9,
+	"TB":    1e12,
+	"PB":    1e15,
+	"EB":    1e18,
+}
+
+//UnmarshalJSON implements the json.Unmarshaler interface
+func (x *CFMBytes) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+
+	fields := strings.SplitN(s, " ", 2)
+	if len(fields) != 2 {
+		return fmt.Errorf(`expected a string with the format "{value} {unit}", got %q instead`, s)
+	}
+	value, err := strconv.ParseFloat(fields[0], 10)
+	if err != nil {
+		return err
+	}
+	unitMultiplier, ok := cfmUnits[fields[1]]
+	if !ok {
+		return fmt.Errorf("unknown unit %q in value %q", fields[1], s)
+	}
+
+	*x = CFMBytes(value * unitMultiplier)
+	return nil
+}

pkg/util/datatypes_test.go

@@ -0,0 +1,48 @@
+/*******************************************************************************
+*
+* Copyright 2018 SAP SE
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You should have received a copy of the License along with this
+* program. If not, you may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*******************************************************************************/
+
+package util
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestCFMBytesDeserialization(t *testing.T) {
+	testcases := map[string]CFMBytes{
+		"5 MB":      5e6,
+		"1.75 TB":   1.75e12,
+		"454.53 GB": 454.53e9,
+		"0 bytes":   0,
+	}
+
+	for input, expected := range testcases {
+		jsonBytes, _ := json.Marshal(input)
+
+		var result CFMBytes
+		err := json.Unmarshal(jsonBytes, &result)
+		if err != nil {
+			t.Errorf("unexpected error while demarshaling %q: %s", input, err.Error())
+		}
+
+		if result != expected {
+			t.Errorf("expected %q to demarshal to %d, but got %d", input, expected, result)
+		}
+	}
+}