Skip to content

Commit

Permalink
Fix prometheus-operator test to wait for the CRD to be ready before use
Browse files Browse the repository at this point in the history
  • Loading branch information
@metral
metral committed Jun 18, 2020
1 parent 7c4a21b commit 8219ffc
Show file tree
Hide file tree
Showing 5 changed files with 171 additions and 94 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
## HEAD (Unreleased)

### Improvements

- Fix prometheus-operator test to wait for the CRD to be ready before use (https://github.com/pulumi/pulumi-kubernetes/pull/1172)

## 2.3.1 (June 17, 2020)

### Improvements
Expand Down
57 changes: 28 additions & 29 deletions tests/examples/examples_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,35 +188,34 @@ func TestAccHelmLocal(t *testing.T) {
integration.ProgramTest(t, &test)
}

// TODO: uncomment once https://github.com/pulumi/pulumi-kubernetes/issues/1137 is fixed.
//func TestAccPrometheusOperator(t *testing.T) {
// skipIfShort(t)
// test := getBaseOptions(t).
// With(integration.ProgramTestOptions{
// Dir: path.Join(getCwd(t), "prometheus-operator"),
// SkipRefresh: true,
// ExtraRuntimeValidation: func(
// t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
// ) {
// assert.NotNil(t, stackInfo.Deployment)
// assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
// },
// EditDirs: []integration.EditDir{
// {
// Dir: path.Join(getCwd(t), "prometheus-operator", "steps"),
// Additive: true,
// ExtraRuntimeValidation: func(
// t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
// ) {
// assert.NotNil(t, stackInfo.Deployment)
// assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
// },
// },
// },
// })
//
// integration.ProgramTest(t, &test)
//}
func TestAccPrometheusOperator(t *testing.T) {
skipIfShort(t)
test := getBaseOptions(t).
With(integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "prometheus-operator"),
SkipRefresh: true,
ExtraRuntimeValidation: func(
t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
) {
assert.NotNil(t, stackInfo.Deployment)
assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
},
EditDirs: []integration.EditDir{
{
Dir: path.Join(getCwd(t), "prometheus-operator", "steps"),
Additive: true,
ExtraRuntimeValidation: func(
t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
) {
assert.NotNil(t, stackInfo.Deployment)
assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
},
},
},
})

integration.ProgramTest(t, &test)
}

func TestAccMariadb(t *testing.T) {
skipIfShort(t)
Expand Down
100 changes: 68 additions & 32 deletions tests/examples/prometheus-operator/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import * as k8s from "@pulumi/kubernetes";
import * as pulumi from "@pulumi/pulumi";
import * as k8sClient from "@kubernetes/client-node";

// PrometheusOperatorArgs are the options to configure on the CoreOS
// PrometheusOperator.
Expand All @@ -9,8 +10,9 @@ interface PrometheusOperatorArgs {
}

// PrometheusOperator implements the CoreOS Prometheus Operator.
export class PrometheusOperator extends pulumi.ComponentResource {
class PrometheusOperator extends pulumi.ComponentResource {
public readonly configFile: k8s.yaml.ConfigFile;
public readonly serviceMonitorReady: pulumi.Output<boolean>;
constructor(
name: string,
args: PrometheusOperatorArgs,
Expand All @@ -33,6 +35,36 @@ export class PrometheusOperator extends pulumi.ComponentResource {
},
],
}, { parent: this });

// Wait for the CRD to be established.
// Use the k8s JS client (https://github.com/kubernetes-client/javascript)
// to retrieve resource created by the operator as a workaround for:
// https://github.com/pulumi/pulumi-kubernetes/issues/1056
this.serviceMonitorReady = pulumi.output(this.configFile).apply(async () => {
const kc = new k8sClient.KubeConfig();
kc.loadFromDefault();
const kApi = kc.makeApiClient(k8sClient.ApiextensionsV1Api)

for (let i = 0; i < 20; i++){
try {
const crd = await kApi.readCustomResourceDefinition("servicemonitors.monitoring.coreos.com");
let conditions = crd?.body?.status?.conditions;
if (conditions) {
for (let c of conditions) {
if (c.type == "Established" && c.status == "True") {
console.log("ServiceMonitor CRD is ready");
return true;
}
}
}
break;
} catch(e) {
console.log(`Waiting for ServiceMonitor CRD to be ready (${i+1})`)
await new Promise(resolve => setTimeout(resolve, 3 * 1000));
}
}
return false;
})
}
}

Expand All @@ -42,37 +74,41 @@ const prometheusOperator = new PrometheusOperator("prometheus", {
});

// Create the Prometheus Operator ServiceMonitor.
const myMonitoring = new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
endpoints: [
{
port: 'http',
interval: '65s',
// start with the following
relabelings: [
const myMonitoring = prometheusOperator.serviceMonitorReady.apply(ready => {
if (ready){
return new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
endpoints: [
{
regex: '(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
port: 'http',
interval: '65s',
// start with the following
relabelings: [
{
regex: '(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
],
// try to add the following in replacement of above in steps/step1.ts
// metricRelabelings: [
// {
// sourceLabels: ['__name__'],
// regex: 'typhoon_(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ]
},
],
// try to add the following in replacement of above in steps/step1.ts
// metricRelabelings: [
// {
// sourceLabels: ['__name__'],
// regex: 'typhoon_(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ]
},
],
},
}, {dependsOn: prometheusOperator});
});
}
});
2 changes: 2 additions & 0 deletions tests/examples/prometheus-operator/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"name": "prometheus-operator",
"version": "0.1.0",
"dependencies": {
"@kubernetes/client-node": "latest",
"@pulumi/kubernetes": "^2.3.1",
"@pulumi/pulumi": "latest"
},
"peerDependencies": {
Expand Down
102 changes: 69 additions & 33 deletions tests/examples/prometheus-operator/step1/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import * as k8s from "@pulumi/kubernetes";
import * as pulumi from "@pulumi/pulumi";
import * as k8sClient from "@kubernetes/client-node";

// PrometheusOperatorArgs are the options to configure on the CoreOS
// PrometheusOperator.
Expand All @@ -9,8 +10,9 @@ interface PrometheusOperatorArgs {
}

// PrometheusOperator implements the CoreOS Prometheus Operator.
export class PrometheusOperator extends pulumi.ComponentResource {
class PrometheusOperator extends pulumi.ComponentResource {
public readonly configFile: k8s.yaml.ConfigFile;
public readonly serviceMonitorReady: pulumi.Output<boolean>;
constructor(
name: string,
args: PrometheusOperatorArgs,
Expand All @@ -33,6 +35,36 @@ export class PrometheusOperator extends pulumi.ComponentResource {
},
],
}, { parent: this });

// Wait for the CRD to be established.
// Use the k8s JS client (https://github.com/kubernetes-client/javascript)
// to retrieve resource created by the operator as a workaround for:
// https://github.com/pulumi/pulumi-kubernetes/issues/1056
this.serviceMonitorReady = pulumi.output(this.configFile).apply(async () => {
const kc = new k8sClient.KubeConfig();
kc.loadFromDefault();
const kApi = kc.makeApiClient(k8sClient.ApiextensionsV1Api)

for (let i = 0; i < 20; i++){
try {
const crd = await kApi.readCustomResourceDefinition("servicemonitors.monitoring.coreos.com");
let conditions = crd?.body?.status?.conditions;
if (conditions) {
for (let c of conditions) {
if (c.type == "Established" && c.status == "True") {
console.log("ServiceMonitor CRD is ready");
return true;
}
}
}
break;
} catch(e) {
console.log(`Waiting for ServiceMonitor CRD to be ready (${i+1})`)
await new Promise(resolve => setTimeout(resolve, 3 * 1000));
}
}
return false;
})
}
}

Expand All @@ -42,37 +74,41 @@ const prometheusOperator = new PrometheusOperator("prometheus", {
});

// Create the Prometheus Operator ServiceMonitor.
const myMonitoring = new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
endpoints: [
{
port: 'http',
interval: '65s',
// removing the following in index.ts in favor of below
// relabelings: [
// {
// regex: '(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ],
// add the following in replacement of above in index.ts
metricRelabelings: [
const myMonitoring = prometheusOperator.serviceMonitorReady.apply(ready => {
if (ready){
return new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
endpoints: [
{
sourceLabels: ['__name__'],
regex: 'typhoon_(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
]
port: 'http',
interval: '65s',
// removing the following in index.ts in favor of below
// relabelings: [
// {
// regex: '(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ],
// add the following in replacement of above in index.ts
metricRelabelings: [
{
sourceLabels: ['__name__'],
regex: 'typhoon_(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
]
},
],
},
],
},
}, {dependsOn: prometheusOperator});
});
}
})

0 comments on commit 8219ffc

Please sign in to comment.