Skip to content
This repository has been archived by the owner on Oct 3, 2023. It is now read-only.

Commit

Permalink
Stats/Stackdriver: Use resource util to generate MonitoredResource (#314
Browse files Browse the repository at this point in the history 
)

* Exporter/Stats/Stackdriver: Use resource util to generate MonitoredResource

* fix review comments
  • Loading branch information
@mayurkale22
mayurkale22 committed Jan 30, 2019
1 parent 9414faf commit 4daea53
Show file tree
Hide file tree
Showing 8 changed files with 186 additions and 22 deletions.
1 change: 1 addition & 0 deletions packages/opencensus-exporter-stackdriver/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
},
"dependencies": {
"@opencensus/core": "^0.0.8",
"@opencensus/resource-util": "^0.0.1",
"google-auth-library": "^1.5.0",
"googleapis": "27.0.0",
"hex2dec": "^1.1.0"
Expand Down
15 changes: 6 additions & 9 deletions packages/opencensus-exporter-stackdriver/src/stackdriver-monitoring.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@
import {logger, Logger, Measurement, Metric, MetricDescriptor as OCMetricDescriptor, MetricProducerManager, Metrics, StatsEventListener, TagKey, TagValue, View} from '@opencensus/core';
import {auth, JWT} from 'google-auth-library';
import {google} from 'googleapis';

import {createMetricDescriptorData, createTimeSeriesList} from './stackdriver-stats-utils';
import {StackdriverExporterOptions, TimeSeries} from './types';
import {createMetricDescriptorData, createTimeSeriesList, getDefaultResource} from './stackdriver-stats-utils';
import {MonitoredResource, StackdriverExporterOptions, TimeSeries} from './types';

google.options({headers: {'x-opencensus-outgoing-request': 0x1}});
const monitoring = google.monitoring('v3');
Expand All @@ -36,11 +35,10 @@ export class StackdriverStatsExporter implements StatsEventListener {
static readonly DEFAULT_DISPLAY_NAME_PREFIX: string = 'OpenCensus';
static readonly CUSTOM_OPENCENSUS_DOMAIN: string =
'custom.googleapis.com/opencensus';
static readonly GLOBAL: string = 'global';
static readonly PERIOD: number = 60000;
private registeredMetricDescriptors: Map<string, OCMetricDescriptor> =
new Map();

private DEFAULT_RESOURCE: Promise<MonitoredResource>;
logger: Logger;

constructor(options: StackdriverExporterOptions) {
Expand All @@ -54,6 +52,7 @@ export class StackdriverStatsExporter implements StatsEventListener {
options.prefix || StackdriverStatsExporter.DEFAULT_DISPLAY_NAME_PREFIX;
this.logger = options.logger || logger.logger();
this.onMetricUploadError = options.onMetricUploadError;
this.DEFAULT_RESOURCE = getDefaultResource(this.projectId);
}

/**
Expand Down Expand Up @@ -132,11 +131,9 @@ export class StackdriverStatsExporter implements StatsEventListener {
* be uploaded to StackDriver.
* @param metricsList The List of Metric.
*/
private createTimeSeries(metricsList: Metric[]) {
private async createTimeSeries(metricsList: Metric[]) {
const timeSeries: TimeSeries[] = [];
const resourceLabels = {project_id: this.projectId};
const monitoredResource = {type: 'global', labels: resourceLabels};

const monitoredResource = await this.DEFAULT_RESOURCE;
for (const metric of metricsList) {
timeSeries.push(...createTimeSeriesList(
metric, monitoredResource, this.metricPrefix));
Expand Down
61 changes: 60 additions & 1 deletion packages/opencensus-exporter-stackdriver/src/stackdriver-stats-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
* limitations under the License.
*/

import {BucketOptions, DistributionBucket, DistributionValue, LabelKey, LabelValue, Metric, MetricDescriptor as OCMetricDescriptor, MetricDescriptorType, TimeSeriesPoint, Timestamp} from '@opencensus/core';
import {BucketOptions, DistributionBucket, DistributionValue, LabelKey, Labels, LabelValue, Metric, MetricDescriptor as OCMetricDescriptor, MetricDescriptorType, TimeSeriesPoint, Timestamp} from '@opencensus/core';
import * as resource from '@opencensus/resource-util';
import * as os from 'os';
import * as path from 'path';

Expand All @@ -23,6 +24,26 @@ import {Distribution, LabelDescriptor, MetricDescriptor, MetricKind, MonitoredRe
const OPENCENSUS_TASK = 'opencensus_task';
const OPENCENSUS_TASK_DESCRIPTION = 'Opencensus task identifier';
export const OPENCENSUS_TASK_VALUE_DEFAULT = generateDefaultTaskValue();
const STACKDRIVER_PROJECT_ID_KEY = 'project_id';

/* Return a self-configured StackDriver monitored resource. */
export async function getDefaultResource(projectId: string):
Promise<MonitoredResource> {
const labels: Labels = {project_id: projectId};
const autoDetectedResource = await resource.detectResource();
const [type, mappings] = getTypeAndMappings(autoDetectedResource.type);
Object.keys(mappings).forEach((key) => {
if (autoDetectedResource.labels[mappings[key]]) {
if (mappings[key] === resource.AWS_REGION_KEY) {
labels[key] = `${resource.AWS_REGION_VALUE_PREFIX}${
autoDetectedResource.labels[mappings[key]]}`;
} else {
labels[key] = autoDetectedResource.labels[mappings[key]];
}
}
});
return {type, labels};
}

/** Converts a OpenCensus MetricDescriptor to a StackDriver MetricDescriptor. */
export function createMetricDescriptorData(
Expand Down Expand Up @@ -237,6 +258,44 @@ function leftZeroPad(ns: number) {
return `${pad}${str}`;
}

function getTypeAndMappings(resourceType: string): [string, Labels] {
switch (resourceType) {
case resource.GCP_GCE_INSTANCE_TYPE:
// https://cloud.google.com/monitoring/api/resources#tag_gce_instance
return [
'gce_instance', {
'project_id': STACKDRIVER_PROJECT_ID_KEY,
'instance_id': resource.GCP_INSTANCE_ID_KEY,
'zone': resource.GCP_ZONE_KEY
}
];
case resource.K8S_CONTAINER_TYPE:
// https://cloud.google.com/monitoring/api/resources#tag_k8s_container
return [
'k8s_container', {
'project_id': STACKDRIVER_PROJECT_ID_KEY,
'location': resource.GCP_ZONE_KEY,
'cluster_name': resource.K8S_CLUSTER_NAME_KEY,
'namespace_name': resource.K8S_NAMESPACE_NAME_KEY,
'pod_name': resource.K8S_POD_NAME_KEY,
'container_name': resource.K8S_CONTAINER_NAME_KEY
}
];
case resource.AWS_EC2_INSTANCE_TYPE:
// https://cloud.google.com/monitoring/api/resources#tag_aws_ec2_instance
return [
'aws_ec2_instance', {
'project_id': STACKDRIVER_PROJECT_ID_KEY,
'instance_id': resource.AWS_INSTANCE_ID_KEY,
'region': resource.AWS_REGION_KEY,
'aws_account': resource.AWS_ACCOUNT_KEY
}
];
default:
return ['global', {}];
}
}

export const TEST_ONLY = {
createMetricType,
createDisplayName,
Expand Down
31 changes: 30 additions & 1 deletion packages/opencensus-exporter-stackdriver/test/nocks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ import * as nock from 'nock';

const accept = () => true;

const HEADERS = {
['metadata-flavor']: 'Google'
};

export function oauth2<T extends {} = {}>(validator?: (body: T) => boolean):
nock.Scope {
validator = validator || accept;
Expand All @@ -45,6 +49,31 @@ export function projectId(status: number|(() => string), reply?: () => string) {
.reply(status, reply, {'Metadata-Flavor': 'Google'});
}

export function noDetectResource() {
const scopes = [
nock('http://metadata.google.internal')
.get('/computeMetadata/v1/instance')
.once()
.replyWithError({code: 'ENOTFOUND'}),
nock('http://169.254.169.254/latest/dynamic/instance-identity/document')
.get('')
.replyWithError({code: 'ENOTFOUND'})
];
return scopes;
}

export function detectGceResource() {
return nock('http://metadata.google.internal')
.get('/computeMetadata/v1/instance')
.reply(200, {}, HEADERS)
.get('/computeMetadata/v1/project/project-id')
.reply(200, () => 'my-project-id', HEADERS)
.get('/computeMetadata/v1/instance/zone')
.reply(200, () => 'project/zone/my-zone', HEADERS)
.get('/computeMetadata/v1/instance/id')
.reply(200, () => 4520031799277581759, HEADERS);
}

export function instanceId(
status: number|(() => string), reply?: () => string) {
if (typeof status === 'function') {
Expand Down Expand Up @@ -118,4 +147,4 @@ export function metricDescriptors<T extends {} = {}>(
scope = interceptor.reply(200, reply);
}
return scope;
}
}
4 changes: 2 additions & 2 deletions packages/opencensus-exporter-stackdriver/test/test-stackdriver-monitoring.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import {globalStats, LabelKey, Logger, MeasureUnit, Metrics} from '@opencensus/core';
import * as assert from 'assert';

import * as nock from 'nock';
import {StackdriverStatsExporter} from '../src/stackdriver-monitoring';
import {MetricKind, StackdriverExporterOptions, ValueType} from '../src/types';

Expand Down Expand Up @@ -65,8 +65,8 @@ describe('Stackdriver Stats Exporter', () => {

before(() => {
exporterOptions = {period: 0, projectId: PROJECT_ID, logger: mockLogger};
nocks.noDetectResource();
exporter = new StackdriverStatsExporter(exporterOptions);

nocks.oauth2();
});

Expand Down
78 changes: 76 additions & 2 deletions packages/opencensus-exporter-stackdriver/test/test-stackdriver-stats-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@
* limitations under the License.
*/

import {DistributionValue, LabelKey, LabelValue, MetricDescriptor as OCMetricDescriptor, MetricDescriptorType, TimeSeriesPoint, Timestamp} from '@opencensus/core';
import {CoreResource, DistributionValue, LabelKey, LabelValue, MetricDescriptor as OCMetricDescriptor, MetricDescriptorType, TimeSeriesPoint, Timestamp} from '@opencensus/core';
import * as assert from 'assert';

import {StackdriverStatsExporter} from '../src/stackdriver-monitoring';
import {createMetricDescriptorData, createTimeSeriesList, OPENCENSUS_TASK_VALUE_DEFAULT, TEST_ONLY} from '../src/stackdriver-stats-utils';
import {createMetricDescriptorData, createTimeSeriesList, getDefaultResource, OPENCENSUS_TASK_VALUE_DEFAULT, TEST_ONLY} from '../src/stackdriver-stats-utils';
import {Distribution, MetricDescriptor, MetricKind, ValueType} from '../src/types';

import * as nocks from './nocks';

const METRIC_NAME = 'metric-name';
const METRIC_DESCRIPTION = 'metric-description';
const DEFAULT_UNIT = '1';
Expand Down Expand Up @@ -413,4 +415,76 @@ describe('Stackdriver Stats Exporter Utils', () => {
assert.deepStrictEqual(point2.value, {doubleValue: 15});
});
});

describe('getDefaultResource()', () => {
beforeEach(() => {
delete process.env.OC_RESOURCE_TYPE;
delete process.env.OC_RESOURCE_LABELS;
CoreResource.setup();
});

it('should return a global MonitoredResource', async () => {
nocks.noDetectResource();
const monitoredResource = await getDefaultResource('my-project-id');
const {type, labels} = monitoredResource;

assert.equal(type, 'global');
assert.deepEqual(labels, {project_id: 'my-project-id'});
});

it('should return a k8s MonitoredResource', async () => {
process.env.OC_RESOURCE_TYPE = 'k8s.io/container';
process.env.OC_RESOURCE_LABELS = 'k8s.io/pod/name=pod-xyz-123,' +
'k8s.io/container/name=c1,k8s.io/namespace/name=default,' +
'cloud.google.com/gce/zone=zone1';
CoreResource.setup();
const monitoredResource = await getDefaultResource('my-project-id');
const {type, labels} = monitoredResource;

assert.equal(type, 'k8s_container');
assert.equal(Object.keys(labels).length, 5);
assert.deepStrictEqual(labels, {
'container_name': 'c1',
'namespace_name': 'default',
'pod_name': 'pod-xyz-123',
'project_id': 'my-project-id',
'location': 'zone1'
});
});

it('should return a gce MonitoredResource', async () => {
process.env.OC_RESOURCE_TYPE = 'cloud.google.com/gce/instance';
process.env.OC_RESOURCE_LABELS = 'cloud.google.com/gce/instance_id=id1,' +
'cloud.google.com/gce/zone=zone1';
CoreResource.setup();
const monitoredResource = await getDefaultResource('my-project-id');
const {type, labels} = monitoredResource;

assert.equal(type, 'gce_instance');
assert.equal(Object.keys(labels).length, 3);
assert.deepStrictEqual(labels, {
'instance_id': 'id1',
'project_id': 'my-project-id',
'zone': 'zone1'
});
});

it('should return a aws MonitoredResource', async () => {
process.env.OC_RESOURCE_TYPE = 'aws.com/ec2/instance';
process.env.OC_RESOURCE_LABELS = 'aws.com/ec2/account_id=id1,' +
'aws.com/ec2/instance_id=instance1,aws.com/ec2/region=region1';
CoreResource.setup();
const monitoredResource = await getDefaultResource('my-project-id');
const {type, labels} = monitoredResource;

assert.equal(type, 'aws_ec2_instance');
assert.equal(Object.keys(labels).length, 4);
assert.deepStrictEqual(labels, {
'instance_id': 'instance1',
'region': 'aws:region1',
'project_id': 'my-project-id',
'aws_account': 'id1'
});
});
});
});
3 changes: 2 additions & 1 deletion packages/opencensus-resource-util/src/resource-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ async function getProjectId() {
/** Gets instance id from GCP instance metadata. */
async function getInstanceId() {
try {
return await gcpMetadata.instance('id');
const id = await gcpMetadata.instance('id');
return id.toString();
} catch {
return '';
}
Expand Down
15 changes: 9 additions & 6 deletions packages/opencensus-resource-util/test/test-detect-resource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,15 +154,16 @@ describe('detectResource', () => {
.get(ZONE_PATH)
.reply(200, () => 'project/zone/my-zone', HEADERS)
.get(INSTANCE_ID_PATH)
.reply(200, () => 'my-instance', HEADERS);
.reply(200, () => 4520031799277581759, HEADERS);
const {type, labels} = await resource.detectResource();
scope.done();

assert.deepEqual(type, resource.GCP_GCE_INSTANCE_TYPE);
assert.equal(Object.keys(labels).length, 3);
assert.strictEqual(labels[resource.GCP_ACCOUNT_ID_KEY], 'my-project-id');
assert.strictEqual(labels[resource.GCP_ZONE_KEY], 'my-zone');
assert.strictEqual(labels[resource.GCP_INSTANCE_ID_KEY], 'my-instance');
assert.strictEqual(
labels[resource.GCP_INSTANCE_ID_KEY], '4520031799277582000');
});

it('should retry if the initial request fails', async () => {
Expand All @@ -177,15 +178,16 @@ describe('detectResource', () => {
.get(ZONE_PATH)
.reply(200, () => 'project/zone/my-zone', HEADERS)
.get(INSTANCE_ID_PATH)
.reply(200, () => 'my-instance', HEADERS);
.reply(200, () => 4520031799277581759, HEADERS);
const {type, labels} = await resource.detectResource();
scope.done();

assert.deepEqual(type, resource.GCP_GCE_INSTANCE_TYPE);
assert.equal(Object.keys(labels).length, 3);
assert.strictEqual(labels[resource.GCP_ACCOUNT_ID_KEY], 'my-project-id');
assert.strictEqual(labels[resource.GCP_ZONE_KEY], 'my-zone');
assert.strictEqual(labels[resource.GCP_INSTANCE_ID_KEY], 'my-instance');
assert.strictEqual(
labels[resource.GCP_INSTANCE_ID_KEY], '4520031799277582000');
});

it('should return GCP_GCE_INSTANCE resource and empty data for non avaiable metadata attribute',
Expand Down Expand Up @@ -229,15 +231,16 @@ describe('detectResource', () => {
.get(ZONE_PATH)
.reply(200, () => 'project/zone/my-zone', HEADERS)
.get(INSTANCE_ID_PATH)
.reply(200, () => 'my-instance', HEADERS);
.reply(200, () => 4520031799277581759, HEADERS);
const {type, labels} = await resource.detectResource();
scope.done();

assert.deepEqual(type, 'global');
assert.equal(Object.keys(labels).length, 5);
assert.strictEqual(labels[resource.GCP_ACCOUNT_ID_KEY], 'my-project-id');
assert.strictEqual(labels[resource.GCP_ZONE_KEY], 'zone1');
assert.strictEqual(labels[resource.GCP_INSTANCE_ID_KEY], 'my-instance');
assert.strictEqual(
labels[resource.GCP_INSTANCE_ID_KEY], '4520031799277582000');
assert.strictEqual(labels['user'], 'user1');
assert.strictEqual(labels['version'], '1.0');
});
Expand Down

0 comments on commit 4daea53

Please sign in to comment.