diff --git a/docs/sink-configuration.md b/docs/sink-configuration.md index 56a6b218b8..918b085340 100644 --- a/docs/sink-configuration.md +++ b/docs/sink-configuration.md @@ -99,29 +99,6 @@ can enable opentsdb sink like this: --sink=opentsdb:http://192.168.1.8:4242 -### Monasca -This sink supports monitoring metrics only. -To use the Monasca sink add the following flag: - - --sink=monasca:[?] - -The available options are listed below, and some of them are mandatory. You need to provide access to the Identity service of OpenStack (keystone). -Currently, only authorization through `username` / `userID` + `password` / `APIKey` is supported. If the agent access (for sending metrics) to Monasca is restricted to a role, please specify the corresponding `tenant-id` for automatic scoped authorization. - -The Monasca sink is then created with either the provided Monasca API Server URL, or the URL is discovered automatically if none is provided by the user. - -The following options are available: - -* `user-id` - ID of the OpenStack user -* `username` - Name of the OpenStack user -* `tenant-id` - ID of the OpenStack tenant (project) -* `keystone-url` - URL to the Keystone identity service (*mandatory*). Must be a v3 server (required by Monasca) -* `password` - Password of the OpenStack user -* `api-key` - API-Key for the OpenStack user -* `domain-id` - ID of the OpenStack user's domain -* `domain-name` - Name of the OpenStack user's domain -* `monasca-url` - URL of the Monasca API server (*optional*: the sink will attempt to discover the service if not provided) - ### Kafka This sink supports monitoring metrics only. To use the kafka sink add the following flag: diff --git a/docs/sink-owners.md b/docs/sink-owners.md index 856c0c125d..566d6bd5e1 100644 --- a/docs/sink-owners.md +++ b/docs/sink-owners.md @@ -31,10 +31,7 @@ List of Owners | InfluxDB | :heavy_check_mark: | :heavy_check_mark: | @kubernetes/heapster-maintainers / @andyxning | :ok: | | Metric (memory) | :heavy_check_mark: | :x: | @kubernetes/heapster-maintainers | :ok: | | Kafka | :heavy_check_mark: | :x: | @huangyuqi | :ok: | -| Monasca | :heavy_check_mark: | :x: | | :no_entry: [1] | | OpenTSDB | :heavy_check_mark: | :x: | @bluebreezecf | :ok: | | Riemann | :heavy_check_mark: | :x: :new: | @jamtur01 @mcorbin | :ok: | | Graphite | :heavy_check_mark: | :x: | @jsoriano / @theairkit | :new: #1341 | | Wavefront | :heavy_check_mark: | :x: | @ezeev | :new: #1400 | - -- [1] Monasca now has native support for Kubernetes, so this is no longer needed (see https://github.com/kubernetes/heapster/issues/1407#issuecomment-266008730 and https://github.com/openstack/monasca-agent/blob/master/docs/Plugins.md#docker) diff --git a/metrics/sinks/factory.go b/metrics/sinks/factory.go index 86d6846006..4e1d2569c1 100644 --- a/metrics/sinks/factory.go +++ b/metrics/sinks/factory.go @@ -29,7 +29,6 @@ import ( "k8s.io/heapster/metrics/sinks/kafka" "k8s.io/heapster/metrics/sinks/log" "k8s.io/heapster/metrics/sinks/metric" - "k8s.io/heapster/metrics/sinks/monasca" "k8s.io/heapster/metrics/sinks/opentsdb" "k8s.io/heapster/metrics/sinks/riemann" ) @@ -57,8 +56,6 @@ func (this *SinkFactory) Build(uri flags.Uri) (core.DataSink, error) { return metricsink.NewMetricSink(140*time.Second, 15*time.Minute, []string{ core.MetricCpuUsageRate.MetricDescriptor.Name, core.MetricMemoryUsage.MetricDescriptor.Name}), nil - case "monasca": - return monasca.CreateMonascaSink(&uri.Val) case "opentsdb": return opentsdb.CreateOpenTSDBSink(&uri.Val) case "riemann": diff --git a/metrics/sinks/monasca/config.go b/metrics/sinks/monasca/config.go deleted file mode 100644 index faa4f460d6..0000000000 --- a/metrics/sinks/monasca/config.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "net/url" - - "github.com/rackspace/gophercloud" -) - -// Config represents the configuration of the Monasca sink. -type Config struct { - gophercloud.AuthOptions - MonascaURL string -} - -// NewConfig builds a configuration object from the parameters of the monasca sink. -func NewConfig(opts url.Values) Config { - config := Config{} - if len(opts["keystone-url"]) >= 1 { - config.IdentityEndpoint = opts["keystone-url"][0] - } - if len(opts["tenant-id"]) >= 1 { - config.TenantID = opts["tenant-id"][0] - } - if len(opts["username"]) >= 1 { - config.Username = opts["username"][0] - } - if len(opts["user-id"]) >= 1 { - config.UserID = opts["user-id"][0] - } - if len(opts["password"]) >= 1 { - config.Password = opts["password"][0] - } - if len(opts["api-key"]) >= 1 { - config.APIKey = opts["api-key"][0] - } - if len(opts["domain-id"]) >= 1 { - config.DomainID = opts["domain-id"][0] - } - if len(opts["domain-name"]) >= 1 { - config.DomainName = opts["domain-name"][0] - } - if len(opts["monasca-url"]) >= 1 { - config.MonascaURL = opts["monasca-url"][0] - } - return config -} diff --git a/metrics/sinks/monasca/data_test.go b/metrics/sinks/monasca/data_test.go deleted file mode 100644 index 3598b55aeb..0000000000 --- a/metrics/sinks/monasca/data_test.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "time" - - "github.com/rackspace/gophercloud/openstack/identity/v3/tokens" - "k8s.io/heapster/metrics/core" -) - -var measureTime = time.Now() - -// common labels: -var testInput = &core.DataBatch{ - Timestamp: measureTime, - MetricSets: map[string]*core.MetricSet{ - "set1": { - MetricValues: map[string]core.MetricValue{ - "m2": {ValueType: core.ValueInt64, IntValue: 2 ^ 63}, - "m3": {ValueType: core.ValueFloat, FloatValue: -1023.0233}, - }, - Labels: map[string]string{ - core.LabelHostname.Key: "h1", - }, - LabeledMetrics: []core.LabeledMetric{}, - }, - "set2": { - MetricValues: map[string]core.MetricValue{}, - Labels: map[string]string{ - core.LabelHostname.Key: "10.140.32.11", - }, - LabeledMetrics: []core.LabeledMetric{ - { - Name: "cpu/usage", - Labels: map[string]string{ - core.LabelContainerName.Key: "POD", - core.LabelPodName.Key: "mypod-hc3s", - core.LabelLabels.Key: "run:test,pod.name:default/test-u2dc", - core.LabelHostID.Key: "", - }, - MetricValue: core.MetricValue{ - ValueType: core.ValueInt64, - IntValue: 1, - }, - }, - { - Name: "memory/usage", - Labels: map[string]string{ - core.LabelContainerName.Key: "machine", - core.LabelLabels.Key: "pod.name:default/test-u2dc,run:test2,foo:bar", - core.LabelHostID.Key: "myhost", - }, - MetricValue: core.MetricValue{ - ValueType: core.ValueFloat, - FloatValue: 64.0, - }, - }, - }, - }, - }, -} - -var expectedTransformed = []metric{ - { - Name: "m2", - Dimensions: map[string]string{ - "component": emptyValue, - "hostname": "h1", - "service": "kubernetes", - core.LabelContainerName.Key: emptyValue, - }, - Value: 2 ^ 63, - Timestamp: measureTime.UnixNano() / 1000000, - ValueMeta: map[string]string{}, - }, - { - Name: "m3", - Dimensions: map[string]string{ - "component": emptyValue, - "hostname": "h1", - "service": "kubernetes", - core.LabelContainerName.Key: emptyValue, - }, - Value: float64(float32(-1023.0233)), - Timestamp: measureTime.UnixNano() / 1000000, - ValueMeta: map[string]string{}, - }, - { - Name: "cpu.usage", - Dimensions: map[string]string{ - "component": "mypod-hc3s", - "hostname": "10.140.32.11", - "service": "kubernetes", - core.LabelContainerName.Key: "POD", - }, - Value: 1.0, - Timestamp: measureTime.UnixNano() / 1000000, - ValueMeta: map[string]string{ - core.LabelLabels.Key: "run:test pod.name:default/test-u2dc", - }, - }, - { - Name: "memory.usage", - Dimensions: map[string]string{ - "component": emptyValue, - "hostname": "10.140.32.11", - "service": "kubernetes", - core.LabelContainerName.Key: "machine", - }, - Value: float64(float32(64.0)), - Timestamp: measureTime.UnixNano() / 1000000, - ValueMeta: map[string]string{ - core.LabelLabels.Key: "pod.name:default/test-u2dc run:test2 foo:bar", - core.LabelHostID.Key: "myhost", - }, - }, -} - -const ( - testToken = "e80b74" - testScopedToken = "ac54e1" -) - -var invalidToken = &tokens.Token{ID: "invalidToken", ExpiresAt: time.Unix(time.Now().Unix()-5000, 0)} -var validToken = &tokens.Token{ID: testToken, ExpiresAt: time.Unix(time.Now().Unix()+50000, 0)} - -var testConfig = Config{} - -const ( - testUsername = "Joe" - testPassword = "bar" - testUserID = "0ca8f6" - testDomainID = "1789d1" - testDomainName = "example.com" - testTenantID = "8ca4e3" -) - -var ( - ksVersionResp string - ksUnscopedAuthResp string - ksScopedAuthResp string - ksServicesResp string - ksEndpointsResp string - monUnauthorizedResp string - monEmptyDimResp string -) - -func initKeystoneRespStubs() { - ksVersionResp = `{ - "versions": { - "values": [{ - "status": "stable", - "updated": "2015-03-30T00:00:00Z", - "id": "v3.4", - "links": [{ - "href": "` + keystoneAPIStub.URL + `", - "rel": "self" - }] - }] - } - }` - ksUnscopedAuthResp = `{ - "token": { - "audit_ids": ["VcxU2JYqT8OzfUVvrjEITQ", "qNUTIJntTzO1-XUk5STybw"], - "expires_at": "2013-02-27T18:30:59.999999Z", - "issued_at": "2013-02-27T16:30:59.999999Z", - "methods": [ - "password" - ], - "user": { - "domain": { - "id": "1789d1", - "name": "example.com" - }, - "id": "0ca8f6", - "name": "Joe" - } - } - }` - ksScopedAuthResp = `{ - "token":{ - "audit_ids":[ - "wQ19eUlHQcGi_MZka-CFPA" - ], - "issued_at":"2013-02-27T16:30:59.999999Z", - "expires_at":"2013-02-27T18:30:59.999999Z", - "is_domain":false, - "methods":[ - "password" - ], - "roles":[ - { - "id":"241497", - "name":"monasca-agent" - } - ], - "is_admin_project":false, - "project":{ - "domain":{ - "id":"` + testDomainID + `", - "name":"` + testDomainName + `" - }, - "id":"` + testTenantID + `", - "name":"monasca-project" - }, - "catalog":[ - { - "endpoints":[ - { - "region_id":"RegionOne", - "url":"` + monascaAPIStub.URL + `", - "region":"RegionOne", - "interface":"public", - "id":"3afbce" - } - ], - "type":"monitoring", - "id":"6d0a8d", - "name":"monasca" - }, - { - "endpoints":[ - { - "region_id":"RegionOne", - "url":"` + keystoneAPIStub.URL + `", - "region":"RegionOne", - "interface":"public", - "id":"94927e" - } - ], - "type":"identity", - "id":"73c7a2", - "name":"keystone" - } - ], - "user":{ - "domain":{ - "id":"` + testDomainID + `", - "name":"` + testDomainName + `" - }, - "id":"` + testUserID + `", - "name":"` + testUsername + `" - } - } - }` - ksServicesResp = `{ - "services": [{ - "description": "Monasca Service", - "id": "ee057c", - "links": { - "self": "` + keystoneAPIStub.URL + `/v3/services/ee057c" - }, - "name": "Monasca", - "type": "monitoring" - }], - "links": { - "self": "` + keystoneAPIStub.URL + `/v3/services", - "previous": null, - "next": null - } - }` - ksEndpointsResp = `{ - "endpoints": [ - { - "enabled": true, - "id": "6fedc0", - "interface": "public", - "links": { - "self": "` + keystoneAPIStub.URL + `/v3/endpoints/6fedc0" - }, - "region_id": "us-east-1", - "service_id": "ee057c", - "url": "` + monascaAPIStub.URL + `" - } - ], - "links": { - "self": "` + keystoneAPIStub.URL + `/v3/endpoints", - "previous": null, - "next": null - } - }` - monUnauthorizedResp = "Invaild token provided" - monEmptyDimResp = "Empty dimension detected" -} diff --git a/metrics/sinks/monasca/driver.go b/metrics/sinks/monasca/driver.go deleted file mode 100644 index 6d2f7512fa..0000000000 --- a/metrics/sinks/monasca/driver.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "fmt" - "net/http" - "net/url" - "reflect" - "strings" - "sync" - "time" - - "github.com/golang/glog" - "k8s.io/heapster/metrics/core" -) - -type monascaSink struct { - client Client - sync.RWMutex - numberOfFailures int -} - -// Pushes the specified metric measurement to the Monasca API. -// The Timeseries are transformed to monasca metrics beforehand. -// Timeseries that cannot be translated to monasca metrics are skipped. -func (sink *monascaSink) ExportData(dataBatch *core.DataBatch) { - sink.Lock() - defer sink.Unlock() - - metrics := sink.processMetrics(dataBatch) - code, response, err := sink.client.SendRequest("POST", "/metrics", metrics) - if err != nil { - glog.Errorf("%s", err) - sink.numberOfFailures++ - return - } - if code != http.StatusNoContent { - glog.Error(response) - sink.numberOfFailures++ - } -} - -// a monasca metric definition -type metric struct { - Name string `json:"name"` - Dimensions map[string]string `json:"dimensions"` - Timestamp int64 `json:"timestamp"` - Value float64 `json:"value"` - ValueMeta map[string]string `json:"value_meta"` -} - -func (sink *monascaSink) processMetrics(dataBatch *core.DataBatch) []metric { - metrics := []metric{} - for _, metricSet := range dataBatch.MetricSets { - m := sink.processMetricSet(dataBatch, metricSet) - metrics = append(metrics, m...) - } - return metrics -} - -func (sink *monascaSink) processMetricSet(dataBatch *core.DataBatch, metricSet *core.MetricSet) []metric { - metrics := []metric{} - - // process unlabeled metrics - for metricName, metricValue := range metricSet.MetricValues { - m := sink.processMetric(metricSet.Labels, metricName, dataBatch.Timestamp, metricValue.GetValue()) - if nil != m { - metrics = append(metrics, *m) - } - } - - // process labeled metrics - for _, metric := range metricSet.LabeledMetrics { - labels := map[string]string{} - for k, v := range metricSet.Labels { - labels[k] = v - } - for k, v := range metric.Labels { - labels[k] = v - } - m := sink.processMetric(labels, metric.Name, dataBatch.Timestamp, metric.GetValue()) - if nil != m { - metrics = append(metrics, *m) - } - } - return metrics -} - -func (sink *monascaSink) processMetric(labels map[string]string, name string, timestamp time.Time, value interface{}) *metric { - val, err := sink.convertValue(value) - if err != nil { - glog.Warningf("Metric cannot be pushed to monasca. %#v", value) - return nil - } - dims, valueMeta := sink.processLabels(labels) - m := metric{ - Name: strings.Replace(name, "/", ".", -1), - Dimensions: dims, - Timestamp: (timestamp.UnixNano() / 1000000), - Value: val, - ValueMeta: valueMeta, - } - return &m -} - -// convert the Timeseries value to a monasca value -func (sink *monascaSink) convertValue(val interface{}) (float64, error) { - switch val.(type) { - case int: - return float64(val.(int)), nil - case int64: - return float64(val.(int64)), nil - case bool: - if val.(bool) { - return 1.0, nil - } - return 0.0, nil - case float32: - return float64(val.(float32)), nil - case float64: - return val.(float64), nil - } - return 0.0, fmt.Errorf("Unsupported monasca metric value type %T", reflect.TypeOf(val)) -} - -const ( - emptyValue = "none" - monascaComponent = "component" - monascaService = "service" - monascaHostname = "hostname" -) - -// preprocesses heapster labels, splitting into monasca dimensions and monasca meta-values -func (sink *monascaSink) processLabels(labels map[string]string) (map[string]string, map[string]string) { - dims := map[string]string{} - valueMeta := map[string]string{} - - // labels to dimensions - dims[monascaComponent] = sink.processDimension(labels[core.LabelPodName.Key]) - dims[monascaHostname] = sink.processDimension(labels[core.LabelHostname.Key]) - dims[core.LabelContainerName.Key] = sink.processDimension(labels[core.LabelContainerName.Key]) - dims[monascaService] = "kubernetes" - - // labels to valueMeta - for i, v := range labels { - if i != core.LabelPodName.Key && i != core.LabelHostname.Key && - i != core.LabelContainerName.Key && v != "" { - valueMeta[i] = strings.Replace(v, ",", " ", -1) - } - } - return dims, valueMeta -} - -// creates a valid dimension value -func (sink *monascaSink) processDimension(value string) string { - if value != "" { - v := strings.Replace(value, "/", ".", -1) - return strings.Replace(v, ",", " ", -1) - } - return emptyValue -} - -func (sink *monascaSink) Name() string { - return "Monasca Sink" -} - -func (sink *monascaSink) Stop() { - // Nothing needs to be done -} - -// CreateMonascaSink creates a monasca sink that can consume the Monasca APIs to create metrics. -func CreateMonascaSink(uri *url.URL) (core.DataSink, error) { - opts := uri.Query() - config := NewConfig(opts) - client, err := NewMonascaClient(config) - if err != nil { - return nil, err - } - monascaSink := monascaSink{client: client} - glog.Infof("Created Monasca sink. Monasca server running on: %s", client.GetURL().String()) - return &monascaSink, nil -} diff --git a/metrics/sinks/monasca/driver_test.go b/metrics/sinks/monasca/driver_test.go deleted file mode 100644 index 0e73811bd7..0000000000 --- a/metrics/sinks/monasca/driver_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -// test the transformation of timeseries to monasca metrics -func TestTimeseriesTransform(t *testing.T) { - // setup - sut := monascaSink{} - - // do - metrics := sut.processMetrics(testInput) - - // assert - set1 := map[string]metric{} - set2 := map[string]metric{} - for _, m := range expectedTransformed { - set1[m.Name] = m - } - for _, m := range metrics { - set2[m.Name] = m - } - assert.Equal(t, set1, set2) -} - -// test if the sink creation fails when password is not provided -func TestMissingPasswordError(t *testing.T) { - // setup - uri, _ := url.Parse("monasca:?keystone-url=" + testConfig.IdentityEndpoint + "&user_id=" + testUserID) - - // do - _, err := CreateMonascaSink(uri) - - // assert - assert.Error(t, err) -} - -// test if the sink creation fails when keystone-url is not provided -func TestMissingKeystoneURLError(t *testing.T) { - // setup - uri, _ := url.Parse("monasca:?user_id=" + testUserID + "&password=" + testPassword) - - // do - _, err := CreateMonascaSink(uri) - - // assert - assert.Error(t, err) -} - -// test if the sink creation fails when neither user-id nor username are provided -func TestMissingUserError(t *testing.T) { - // setup - uri, _ := url.Parse("monasca:?keystone-url=" + testConfig.IdentityEndpoint + "&password=" + testPassword) - - // do - _, err := CreateMonascaSink(uri) - - // assert - assert.Error(t, err) -} - -// test if the sink creation fails when domain_id and domainname are missing -// and username is provided -func TestMissingDomainWhenUsernameError(t *testing.T) { - // setup - uri, _ := url.Parse("monasca:?keystone-url=" + testConfig.IdentityEndpoint + "&password=" + - testPassword + "&username=" + testUsername) - - // do - _, err := CreateMonascaSink(uri) - - // assert - assert.Error(t, err) -} - -// test if the sink creation fails when password is not provided -func TestWrongMonascaURLError(t *testing.T) { - // setup - uri, _ := url.Parse("monasca:?keystone-url=" + testConfig.IdentityEndpoint + "&password=" + - testConfig.Password + "&user-id=" + testConfig.UserID + "&monasca-url=_malformed") - - // do - _, err := CreateMonascaSink(uri) - - // assert - assert.Error(t, err) -} - -// test the successful creation of the monasca sink -func TestMonascaSinkCreation(t *testing.T) { - // setup - uri, _ := url.Parse("monasca:?keystone-url=" + testConfig.IdentityEndpoint + "&password=" + - testConfig.Password + "&user-id=" + testConfig.UserID) - - // do - _, err := CreateMonascaSink(uri) - - // assert - assert.NoError(t, err) -} - -// integration test of storing metrics -func TestStoreMetrics(t *testing.T) { - // setup - ks, _ := NewKeystoneClient(testConfig) - monURL, err := ks.MonascaURL() - assert.NoError(t, err) - sut := monascaSink{client: &ClientImpl{ksClient: ks, monascaURL: monURL}} - - // do - sut.ExportData(testInput) - - // assert - assert.Equal(t, 0, sut.numberOfFailures) -} - -// integration test of failure to create metrics -func TestStoreMetricsFailure(t *testing.T) { - // setup - ks, _ := NewKeystoneClient(testConfig) - monURL, _ := url.Parse("http://unexisting.monasca.com") - sut := monascaSink{client: &ClientImpl{ksClient: ks, monascaURL: monURL}} - - // do - sut.ExportData(testInput) - - // assert - assert.Equal(t, 1, sut.numberOfFailures) -} diff --git a/metrics/sinks/monasca/init_test.go b/metrics/sinks/monasca/init_test.go deleted file mode 100644 index cf7149b245..0000000000 --- a/metrics/sinks/monasca/init_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "os" - "sync" - "testing" - - "github.com/stretchr/testify/mock" -) - -type keystoneClientMock struct { - *KeystoneClientImpl - mock.Mock -} - -func (ks *keystoneClientMock) MonascaURL() (*url.URL, error) { - args := ks.Called() - return args.Get(0).(*url.URL), args.Error(1) -} - -func (ks *keystoneClientMock) GetToken() (string, error) { - args := ks.Called() - return args.String(0), args.Error(1) -} - -var monascaAPIStub *httptest.Server -var keystoneAPIStub *httptest.Server - -type ksAuthRequest struct { - Auth ksAuth `json:"auth"` -} - -type ksAuth struct { - Identity ksIdentity `json:"identity"` - Scope ksScope `json:"scope"` -} - -type ksIdentity struct { - Methods []string `json:"methods"` - Password ksPassword `json:"password"` -} - -type ksPassword struct { - User ksUser `json:"user"` -} - -type ksUser struct { - ID string `json:"id"` - Password string `json:"password"` -} - -type ksScope struct { - Project ksProject `json:"project"` -} - -type ksProject struct { - ID string `json:"id"` -} - -// prepare before testing -func TestMain(m *testing.M) { - // monasca stub - monascaMutex := &sync.Mutex{} - defer monascaAPIStub.Close() - monascaAPIStub = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - monascaMutex.Lock() - defer monascaMutex.Unlock() - w.Header().Set("Content-Type", "application/json") - switch r.URL.Path { - case "/metrics": - defer r.Body.Close() - contents, err := ioutil.ReadAll(r.Body) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%s", err))) - break - } - - // umarshal & do type checking on the fly - metrics := []metric{} - err = json.Unmarshal(contents, &metrics) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%s", err))) - break - } - - // check for empty dimensions - for _, metric := range metrics { - for _, dimVal := range metric.Dimensions { - if dimVal == "" { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(monEmptyDimResp)) - break - } - } - } - - // check token - token := r.Header.Get("X-Auth-Token") - if token != testToken { - w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(monUnauthorizedResp)) - break - } - w.WriteHeader(http.StatusNoContent) - break - case "/versions": - w.WriteHeader(http.StatusOK) - break - } - })) - - // keystone stub - keystoneMutex := &sync.Mutex{} - defer keystoneAPIStub.Close() - keystoneAPIStub = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - keystoneMutex.Lock() - defer keystoneMutex.Unlock() - w.Header().Add("Content-Type", "application/json") - switch r.URL.Path { - case "/": - w.Write([]byte(ksVersionResp)) - break - case "/v3/auth/tokens": - if r.Method == "HEAD" { - ksToken := r.Header.Get("X-Auth-Token") - if ksToken != testToken { - w.WriteHeader(http.StatusUnauthorized) - break - } - token := r.Header.Get("X-Subject-Token") - if token == testToken { - // token valid - w.WriteHeader(http.StatusNoContent) - break - } - // token invalid - w.WriteHeader(http.StatusNotFound) - break - } - - // read request - defer r.Body.Close() - contents, err := ioutil.ReadAll(r.Body) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return - } - req := ksAuthRequest{} - err = json.Unmarshal(contents, &req) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return - } - - // authenticate - if req.Auth.Identity.Password.User.ID != testConfig.UserID || - req.Auth.Identity.Password.User.Password != testConfig.Password { - w.WriteHeader(http.StatusUnauthorized) - return - } - - // check if the request is for a scoped token - returnedToken := testToken - returnedBody := ksUnscopedAuthResp - if req.Auth.Scope.Project.ID != "" { - if req.Auth.Scope.Project.ID != testTenantID { - w.WriteHeader(http.StatusInternalServerError) - return - } - returnedToken = testScopedToken - returnedBody = ksScopedAuthResp - } - // return a token - w.Header().Add("X-Subject-Token", returnedToken) - w.WriteHeader(http.StatusAccepted) - w.Write([]byte(returnedBody)) - break - case "/v3/services": - w.Write([]byte(ksServicesResp)) - break - case "/v3/endpoints": - w.Write([]byte(ksEndpointsResp)) - default: - w.WriteHeader(http.StatusInternalServerError) - } - })) - initKeystoneRespStubs() - - testConfig.Password = "bar" - testConfig.UserID = "0ca8f6" - testConfig.IdentityEndpoint = keystoneAPIStub.URL + "/v3" - os.Exit(m.Run()) -} diff --git a/metrics/sinks/monasca/keystone_client.go b/metrics/sinks/monasca/keystone_client.go deleted file mode 100644 index 90b616a350..0000000000 --- a/metrics/sinks/monasca/keystone_client.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "net/url" - - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack" - "github.com/rackspace/gophercloud/openstack/identity/v3/endpoints" - "github.com/rackspace/gophercloud/openstack/identity/v3/services" - "github.com/rackspace/gophercloud/openstack/identity/v3/tokens" - "github.com/rackspace/gophercloud/pagination" -) - -// KeystoneClient defines the interface of any client that can talk with Keystone. -type KeystoneClient interface { - MonascaURL() (*url.URL, error) - GetToken() (string, error) -} - -// KeystoneClientImpl can authenticate with keystone and provide -// tokens, required for accessing the Monasca APIs. -type KeystoneClientImpl struct { - client *gophercloud.ServiceClient - opts gophercloud.AuthOptions - token *tokens.Token - monascaURL *url.URL -} - -// MonascaURL Discovers the monasca service API endpoint and returns it. -func (ksClient *KeystoneClientImpl) MonascaURL() (*url.URL, error) { - if ksClient.monascaURL == nil { - monascaURL, err := ksClient.serviceEndpoint("monitoring", gophercloud.AvailabilityPublic) - if err != nil { - return nil, err - } - ksClient.monascaURL = monascaURL - } - return ksClient.monascaURL, nil -} - -// discovers a single service endpoint from a given service type -func (ksClient *KeystoneClientImpl) serviceEndpoint(serviceType string, availability gophercloud.Availability) (*url.URL, error) { - serviceID, err := ksClient.serviceID(serviceType) - if err != nil { - return nil, err - } - return ksClient.endpointURL(serviceID, availability) -} - -// finds the URL for a given service ID and availability -func (ksClient *KeystoneClientImpl) endpointURL(serviceID string, availability gophercloud.Availability) (*url.URL, error) { - opts := endpoints.ListOpts{Availability: availability, ServiceID: serviceID, PerPage: 1, Page: 1} - pager := endpoints.List(ksClient.client, opts) - endpointURL := (*url.URL)(nil) - err := pager.EachPage(func(page pagination.Page) (bool, error) { - endpointList, err := endpoints.ExtractEndpoints(page) - if err != nil { - return false, err - } - for _, e := range endpointList { - URL, err := url.Parse(e.URL) - if err != nil { - return false, err - } - endpointURL = URL - } - return false, nil - }) - if err != nil { - return nil, err - } - return endpointURL, nil -} - -// returns the first found service ID from a given service type -func (ksClient *KeystoneClientImpl) serviceID(serviceType string) (string, error) { - opts := services.ListOpts{ServiceType: serviceType, PerPage: 1, Page: 1} - pager := services.List(ksClient.client, opts) - serviceID := "" - err := pager.EachPage(func(page pagination.Page) (bool, error) { - serviceList, err := services.ExtractServices(page) - if err != nil { - return false, err - } - for _, s := range serviceList { - serviceID = s.ID - } - return false, nil - }) - if err != nil { - return "", err - } - return serviceID, nil -} - -// GetToken returns a valid X-Auth-Token. -func (ksClient *KeystoneClientImpl) GetToken() (string, error) { - // generate if needed - if ksClient.token == nil { - return ksClient.newToken() - } - // validate - valid, err := tokens.Validate(ksClient.client, ksClient.token.ID) - if err != nil || !valid { - return ksClient.newToken() - } - return ksClient.token.ID, nil -} - -// generates a brand new Keystone token -func (ksClient *KeystoneClientImpl) newToken() (string, error) { - opts := ksClient.opts - var scope *tokens.Scope - if opts.TenantID != "" { - scope = &tokens.Scope{ - ProjectID: opts.TenantID, - } - opts.TenantID = "" - opts.TenantName = "" - } - token, err := tokens.Create(ksClient.client, opts, scope).Extract() - if err != nil { - return "", err - } - ksClient.token = token - return token.ID, nil -} - -// NewKeystoneClient initializes a keystone client with the provided configuration. -func NewKeystoneClient(config Config) (KeystoneClient, error) { - opts := config.AuthOptions - provider, err := openstack.AuthenticatedClient(opts) - if err != nil { - return nil, err - } - client := openstack.NewIdentityV3(provider) - // build a closure for ksClient reauthentication - client.ReauthFunc = func() error { - return openstack.AuthenticateV3(client.ProviderClient, opts) - } - return &KeystoneClientImpl{client: client, opts: opts, token: nil, monascaURL: nil}, nil -} diff --git a/metrics/sinks/monasca/keystone_client_test.go b/metrics/sinks/monasca/keystone_client_test.go deleted file mode 100644 index 9f62806f0f..0000000000 --- a/metrics/sinks/monasca/keystone_client_test.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "testing" - - "github.com/rackspace/gophercloud/openstack" - "github.com/stretchr/testify/assert" -) - -func TestMonascaURLDiscovery(t *testing.T) { - // setup - ksClient, err := NewKeystoneClient(testConfig) - assert.NoError(t, err) - - // do - monURL, err := ksClient.MonascaURL() - - // assert - assert.NoError(t, err) - assert.Equal(t, monURL.String(), monascaAPIStub.URL) -} - -func TestCreateUnscopedToken(t *testing.T) { - // setup - ksClient, err := NewKeystoneClient(testConfig) - assert.NoError(t, err) - - // do - token, err := ksClient.GetToken() - - // assert - assert.NoError(t, err) - assert.Equal(t, token, testToken) -} - -func TestCreateScopedToken(t *testing.T) { - // setup - scopedConfig := testConfig - scopedConfig.TenantID = testTenantID - ksClient, err := NewKeystoneClient(scopedConfig) - - // do - token, err := ksClient.GetToken() - - // assert - assert.NoError(t, err) - assert.Equal(t, token, testScopedToken) -} - -func TestGetTokenWhenInvalid(t *testing.T) { - // setup - opts := testConfig.AuthOptions - provider, err := openstack.AuthenticatedClient(opts) - assert.NoError(t, err) - client := openstack.NewIdentityV3(provider) - ksClient := &KeystoneClientImpl{client: client, opts: opts, token: invalidToken, monascaURL: nil} - - // do - token, err := ksClient.GetToken() - - // assert - assert.NoError(t, err) - assert.Equal(t, token, testToken) -} - -func TestGetTokenWhenValid(t *testing.T) { - // setup - opts := testConfig.AuthOptions - provider, err := openstack.AuthenticatedClient(opts) - assert.NoError(t, err) - client := openstack.NewIdentityV3(provider) - ksClient := &KeystoneClientImpl{client: client, opts: opts, token: validToken, monascaURL: nil} - - // do - token, err := ksClient.GetToken() - - // assert - assert.NoError(t, err) - assert.Equal(t, token, testToken) -} - -func TestKeystoneClientReauthenticate(t *testing.T) { - // setup - opts := testConfig.AuthOptions - provider, err := openstack.AuthenticatedClient(opts) - assert.NoError(t, err) - client := openstack.NewIdentityV3(provider) - client.TokenID = "someinvalidtoken" - client.ReauthFunc = func() error { return openstack.AuthenticateV3(client.ProviderClient, opts) } - ksClient := &KeystoneClientImpl{client: client, opts: opts, token: validToken, monascaURL: nil} - - // do - token, err := ksClient.GetToken() - - // assert - assert.NoError(t, err) - assert.Equal(t, token, testToken) -} diff --git a/metrics/sinks/monasca/monasca_client.go b/metrics/sinks/monasca/monasca_client.go deleted file mode 100644 index 49178dd3dd..0000000000 --- a/metrics/sinks/monasca/monasca_client.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - - "github.com/golang/glog" -) - -// Client specifies the methods of any client ot the Monasca API -type Client interface { - SendRequest(method, path string, request interface{}) (int, string, error) - GetURL() *url.URL - CheckHealth() error -} - -// ClientImpl implements the monasca API client interface. -type ClientImpl struct { - ksClient KeystoneClient - monascaURL *url.URL -} - -// SendRequest to the Monasca API, authenticating on the fly. -// Returns 0, "", err if the request cannot be built. -// Returns statusCode, response, nil if communication with the server was OK. -func (monClient *ClientImpl) SendRequest(method, path string, request interface{}) (int, string, error) { - req, err := monClient.prepareRequest(method, path, request) - if err != nil { - return 0, "", err - } - statusCode, response, err := monClient.receiveResponse(req) - if err != nil { - return 0, "", err - } - return statusCode, response, nil -} - -func (monClient *ClientImpl) prepareRequest(method string, path string, request interface{}) (*http.Request, error) { - // authenticate - token, err := monClient.ksClient.GetToken() - if err != nil { - return nil, err - } - // marshal - var rawRequest io.Reader - if request != nil { - jsonRequest, err := json.Marshal(request) - if err != nil { - return nil, err - } - rawRequest = bytes.NewReader(jsonRequest) - } - // build request - req, err := http.NewRequest(method, monClient.GetURL().String()+path, rawRequest) - if err != nil { - return nil, err - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("X-Auth-Token", token) - return req, nil -} - -func (monClient *ClientImpl) receiveResponse(req *http.Request) (int, string, error) { - resp, err := (&http.Client{}).Do(req) - if err != nil { - return 0, "", err - } - defer resp.Body.Close() - contents, err := ioutil.ReadAll(resp.Body) - respString := "" - if err != nil { - glog.Warning("Cannot read monasca API's response.") - respString = "Cannot read monasca API's response." - } else { - respString = string(contents) - } - return resp.StatusCode, fmt.Sprintf("%s", respString), nil -} - -// GetURL of the Monasca API server. -func (monClient *ClientImpl) GetURL() *url.URL { - return monClient.monascaURL -} - -// CheckHealth of the monasca API server. -func (monClient *ClientImpl) CheckHealth() error { - code, _, err := monClient.SendRequest("GET", "/", nil) - if err != nil { - return fmt.Errorf("Failed to connect to Monasca: %s", err) - } - if code != http.StatusOK { - return fmt.Errorf("Monasca is not running on the provided/discovered URL: %s", monClient.GetURL().String()) - } - return nil -} - -// NewMonascaClient creates a monasca client. -func NewMonascaClient(config Config) (Client, error) { - // create keystone client - ksClient, err := NewKeystoneClient(config) - if err != nil { - return nil, err - } - - // detect monasca URL - monascaURL := (*url.URL)(nil) - if config.MonascaURL != "" { - monascaURL, err = url.Parse(config.MonascaURL) - if err != nil { - return nil, fmt.Errorf("Malformed monasca-url sink parameter. %s", err) - } - } else { - monascaURL, err = ksClient.MonascaURL() - if err != nil { - return nil, fmt.Errorf("Failed to automatically detect Monasca service: %s", err) - } - } - - // create monasca client - client := &ClientImpl{ksClient: ksClient, monascaURL: monascaURL} - err = client.CheckHealth() - if err != nil { - return nil, err - } - return client, nil -} diff --git a/metrics/sinks/monasca/monasca_client_test.go b/metrics/sinks/monasca/monasca_client_test.go deleted file mode 100644 index eb87748e68..0000000000 --- a/metrics/sinks/monasca/monasca_client_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monasca - -import ( - "net/http" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func initClientSUT() (*keystoneClientMock, Client) { - ksClientMock := new(keystoneClientMock) - monURL, _ := url.Parse(monascaAPIStub.URL) - sut := &ClientImpl{ksClient: ksClientMock, monascaURL: monURL} - return ksClientMock, sut -} - -func TestValidRequest(t *testing.T) { - // setup - ksClientMock, sut := initClientSUT() - ksClientMock.On("GetToken").Return(testToken, nil).Once() - - // do - status, resp, err := sut.SendRequest("POST", "/metrics", expectedTransformed) - - // assert - assert.NoError(t, err) - assert.Equal(t, "", resp) - assert.Equal(t, status, http.StatusNoContent) - ksClientMock.AssertExpectations(t) -} - -func TestBadTokenRequest(t *testing.T) { - // setup - ksClientMock, sut := initClientSUT() - ksClientMock.On("GetToken").Return("blob", nil).Once() - - // do - status, resp, err := sut.SendRequest("POST", "/metrics", expectedTransformed) - - // assert - assert.NoError(t, err) - assert.Equal(t, monUnauthorizedResp, resp) - assert.Equal(t, http.StatusUnauthorized, status) - ksClientMock.AssertExpectations(t) -} - -func TestBadMetricsRequest(t *testing.T) { - // setup - ksClientMock, sut := initClientSUT() - ksClientMock.On("GetToken").Return(testToken, nil).Once() - - // do - status, _, err := sut.SendRequest("POST", "/metrics", "[1,2,3,4]") - - // assert - assert.NoError(t, err) - assert.Equal(t, status, http.StatusInternalServerError) - ksClientMock.AssertExpectations(t) -} - -func TestWrongURLRequest(t *testing.T) { - // setup - ksClientMock, sut := initClientSUT() - ksClientMock.On("GetToken").Return(testToken, nil).Once() - - // do - status, resp, err := sut.SendRequest("POST", "http:/malformed", expectedTransformed) - - // assert - assert.Error(t, err) - assert.Equal(t, "", resp) - assert.Equal(t, status, 0) - ksClientMock.AssertExpectations(t) -} - -func TestMonascaHealthy(t *testing.T) { - // setup - ksClientMock, sut := initClientSUT() - ksClientMock.On("GetToken").Return(testToken, nil).Once() - - // do - err := sut.CheckHealth() - - // assert - assert.NoError(t, err) - ksClientMock.AssertExpectations(t) -} - -func TestMonascaUnhealthy(t *testing.T) { - - // setup - ksClientMock := new(keystoneClientMock) - nonReachableMonURL, _ := url.Parse("http://127.0.0.1:9") - sut := &ClientImpl{ksClient: ksClientMock, monascaURL: nonReachableMonURL} - ksClientMock.On("GetToken").Return(testToken, nil).Once() - - // do - err := sut.CheckHealth() - - // assert - assert.Error(t, err) - ksClientMock.AssertExpectations(t) -}