From 931570b3b6fbf17656a80d78392be2007f062346 Mon Sep 17 00:00:00 2001 From: Levi Blackstone Date: Thu, 6 Feb 2020 16:39:04 -0700 Subject: [PATCH 1/4] Specify provider version for invokes The SDK previously used invoke for Helm and YAML support, but was not specifying the parent or version explicitly. As a result, the provider plugin to use was nondeterministic, and could select an incompatible version that lead to an error. --- CHANGELOG.md | 4 ++++ pkg/gen/nodejs-templates/helm/v2/helm.ts | 17 ++++++++++++++--- pkg/gen/nodejs-templates/yaml.ts.mustache | 21 ++++++++++++++++----- sdk/nodejs/helm/v2/helm.ts | 17 ++++++++++++++--- sdk/nodejs/yaml/yaml.ts | 21 ++++++++++++++++----- 5 files changed, 64 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e57856c7..f4dcba3ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## HEAD (Unreleased) +### Bug fixes + +- Specify provider version for invokes. (https://github.com/pulumi/pulumi-kubernetes/pull/982). + ## 1.5.0 (February 4, 2020) ### Improvements diff --git a/pkg/gen/nodejs-templates/helm/v2/helm.ts b/pkg/gen/nodejs-templates/helm/v2/helm.ts index 9d2c71f40c..b76441ff8e 100644 --- a/pkg/gen/nodejs-templates/helm/v2/helm.ts +++ b/pkg/gen/nodejs-templates/helm/v2/helm.ts @@ -18,11 +18,11 @@ import * as pulumi from "@pulumi/pulumi"; import { execSync } from "child_process"; import * as fs from "fs"; -import * as jsyaml from "js-yaml"; import * as nodepath from "path"; import * as shell from "shell-quote"; import * as tmp from "tmp"; import * as path from "../../path"; +import { getVersion } from "../../version"; import * as yaml from "../../yaml/index"; interface BaseChartOpts { @@ -218,7 +218,7 @@ export class Chart extends yaml.CollectionComponentResource { }, ).toString(); return this.parseTemplate( - yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace); + yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace, opts); } catch (e) { // Shed stack trace, only emit the error. throw new pulumi.RunError(e.toString()); @@ -236,9 +236,20 @@ export class Chart extends yaml.CollectionComponentResource { resourcePrefix: string | undefined, dependsOn: pulumi.Resource[], defaultNamespace: string | undefined, + opts?: pulumi.ComponentResourceOptions, ): pulumi.Output<{ [key: string]: pulumi.CustomResource }> { + // Rather than using the default provider for the following invoke call, determine the + // provider from the parent if specified, or fallback to using the version specified + // in package.json. + let invokeOpts: pulumi.InvokeOptions = {async: true}; + if (opts?.parent) { + invokeOpts = {...invokeOpts, parent: opts.parent}; + } else { + invokeOpts = {...invokeOpts, version: getVersion()}; + } + const promise = pulumi.runtime.invoke( - "kubernetes:yaml:decode", {text, defaultNamespace}, {async: true}); + "kubernetes:yaml:decode", {text, defaultNamespace}, invokeOpts); return pulumi.output(promise).apply<{[key: string]: pulumi.CustomResource}>(p => yaml.parse( { resourcePrefix: resourcePrefix, diff --git a/pkg/gen/nodejs-templates/yaml.ts.mustache b/pkg/gen/nodejs-templates/yaml.ts.mustache index 72d40f9b42..bb7d93862e 100644 --- a/pkg/gen/nodejs-templates/yaml.ts.mustache +++ b/pkg/gen/nodejs-templates/yaml.ts.mustache @@ -6,6 +6,7 @@ import * as fs from "fs"; import * as glob from "glob"; import fetch from "node-fetch"; import * as k8s from "../index"; +import { getVersion } from "../version"; import * as outputs from "../types/output"; export interface ConfigGroupOpts { @@ -120,9 +121,19 @@ import * as outputs from "../types/output"; resourcePrefix?: string; } - function yamlLoadAll(text: string): Promise { - const promise = pulumi.runtime.invoke("kubernetes:yaml:decode", {text}, {async: true}); - return promise.then(p => p.result); + function yamlLoadAll(text: string, opts?: pulumi.CustomResourceOptions): Promise { + // Rather than using the default provider for the following invoke call, determine the + // provider from the parent if specified, or fallback to using the version specified + // in package.json. + let invokeOpts: pulumi.InvokeOptions = {async: true}; + if (opts?.parent) { + invokeOpts = {...invokeOpts, parent: opts.parent}; + } else { + invokeOpts = {...invokeOpts, version: getVersion()}; + } + + return pulumi.runtime.invoke("kubernetes:yaml:decode", {text}, invokeOpts) + .then((p => p.result)); } /** @ignore */ export function parse( @@ -172,7 +183,7 @@ import * as outputs from "../types/output"; for (const text of yamlTexts) { const docResources = parseYamlDocument({ - objs: yamlLoadAll(text), + objs: yamlLoadAll(text, opts), transformations: config.transformations, resourcePrefix: config.resourcePrefix }, @@ -320,7 +331,7 @@ import * as outputs from "../types/output"; this.resources = pulumi.output(text.then(t => { try { return parseYamlDocument({ - objs: yamlLoadAll(t), + objs: yamlLoadAll(t, opts), transformations: config && config.transformations || [], resourcePrefix: config && config.resourcePrefix || undefined }, {parent: this}) diff --git a/sdk/nodejs/helm/v2/helm.ts b/sdk/nodejs/helm/v2/helm.ts index 9d2c71f40c..b76441ff8e 100644 --- a/sdk/nodejs/helm/v2/helm.ts +++ b/sdk/nodejs/helm/v2/helm.ts @@ -18,11 +18,11 @@ import * as pulumi from "@pulumi/pulumi"; import { execSync } from "child_process"; import * as fs from "fs"; -import * as jsyaml from "js-yaml"; import * as nodepath from "path"; import * as shell from "shell-quote"; import * as tmp from "tmp"; import * as path from "../../path"; +import { getVersion } from "../../version"; import * as yaml from "../../yaml/index"; interface BaseChartOpts { @@ -218,7 +218,7 @@ export class Chart extends yaml.CollectionComponentResource { }, ).toString(); return this.parseTemplate( - yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace); + yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace, opts); } catch (e) { // Shed stack trace, only emit the error. throw new pulumi.RunError(e.toString()); @@ -236,9 +236,20 @@ export class Chart extends yaml.CollectionComponentResource { resourcePrefix: string | undefined, dependsOn: pulumi.Resource[], defaultNamespace: string | undefined, + opts?: pulumi.ComponentResourceOptions, ): pulumi.Output<{ [key: string]: pulumi.CustomResource }> { + // Rather than using the default provider for the following invoke call, determine the + // provider from the parent if specified, or fallback to using the version specified + // in package.json. + let invokeOpts: pulumi.InvokeOptions = {async: true}; + if (opts?.parent) { + invokeOpts = {...invokeOpts, parent: opts.parent}; + } else { + invokeOpts = {...invokeOpts, version: getVersion()}; + } + const promise = pulumi.runtime.invoke( - "kubernetes:yaml:decode", {text, defaultNamespace}, {async: true}); + "kubernetes:yaml:decode", {text, defaultNamespace}, invokeOpts); return pulumi.output(promise).apply<{[key: string]: pulumi.CustomResource}>(p => yaml.parse( { resourcePrefix: resourcePrefix, diff --git a/sdk/nodejs/yaml/yaml.ts b/sdk/nodejs/yaml/yaml.ts index 994d4ff713..7a8c58aba2 100644 --- a/sdk/nodejs/yaml/yaml.ts +++ b/sdk/nodejs/yaml/yaml.ts @@ -6,6 +6,7 @@ import * as fs from "fs"; import * as glob from "glob"; import fetch from "node-fetch"; import * as k8s from "../index"; +import { getVersion } from "../version"; import * as outputs from "../types/output"; export interface ConfigGroupOpts { @@ -120,9 +121,19 @@ import * as outputs from "../types/output"; resourcePrefix?: string; } - function yamlLoadAll(text: string): Promise { - const promise = pulumi.runtime.invoke("kubernetes:yaml:decode", {text}, {async: true}); - return promise.then(p => p.result); + function yamlLoadAll(text: string, opts?: pulumi.CustomResourceOptions): Promise { + // Rather than using the default provider for the following invoke call, determine the + // provider from the parent if specified, or fallback to using the version specified + // in package.json. + let invokeOpts: pulumi.InvokeOptions = {async: true}; + if (opts?.parent) { + invokeOpts = {...invokeOpts, parent: opts.parent}; + } else { + invokeOpts = {...invokeOpts, version: getVersion()}; + } + + return pulumi.runtime.invoke("kubernetes:yaml:decode", {text}, invokeOpts) + .then((p => p.result)); } /** @ignore */ export function parse( @@ -172,7 +183,7 @@ import * as outputs from "../types/output"; for (const text of yamlTexts) { const docResources = parseYamlDocument({ - objs: yamlLoadAll(text), + objs: yamlLoadAll(text, opts), transformations: config.transformations, resourcePrefix: config.resourcePrefix }, @@ -2462,7 +2473,7 @@ import * as outputs from "../types/output"; this.resources = pulumi.output(text.then(t => { try { return parseYamlDocument({ - objs: yamlLoadAll(t), + objs: yamlLoadAll(t, opts), transformations: config && config.transformations || [], resourcePrefix: config && config.resourcePrefix || undefined }, {parent: this}) From 0b28e355be6d0995e91771d5a7b39c1b84abc2d9 Mon Sep 17 00:00:00 2001 From: Levi Blackstone Date: Thu, 6 Feb 2020 17:03:37 -0700 Subject: [PATCH 2/4] Remove unused js-yaml dependency --- pkg/gen/nodejs-templates/package.json.mustache | 2 -- sdk/nodejs/package.json | 2 -- 2 files changed, 4 deletions(-) diff --git a/pkg/gen/nodejs-templates/package.json.mustache b/pkg/gen/nodejs-templates/package.json.mustache index 21ad0febdc..a6d3d2777e 100644 --- a/pkg/gen/nodejs-templates/package.json.mustache +++ b/pkg/gen/nodejs-templates/package.json.mustache @@ -14,8 +14,6 @@ }, "dependencies": { "@pulumi/pulumi": "^1.8.1", - "@types/js-yaml": "^3.11.2", - "js-yaml": "^3.12.0", "shell-quote": "^1.6.1", "tmp": "^0.0.33", "@types/tmp": "^0.0.33", diff --git a/sdk/nodejs/package.json b/sdk/nodejs/package.json index 21ad0febdc..a6d3d2777e 100755 --- a/sdk/nodejs/package.json +++ b/sdk/nodejs/package.json @@ -14,8 +14,6 @@ }, "dependencies": { "@pulumi/pulumi": "^1.8.1", - "@types/js-yaml": "^3.11.2", - "js-yaml": "^3.12.0", "shell-quote": "^1.6.1", "tmp": "^0.0.33", "@types/tmp": "^0.0.33", From cf200e39fcbb61b54255e483016302ecf2ff763a Mon Sep 17 00:00:00 2001 From: Levi Blackstone Date: Thu, 6 Feb 2020 17:48:07 -0700 Subject: [PATCH 3/4] Update python SDK --- pkg/gen/python-templates/helm/v2/helm.py | 13 ++++++++++++- pkg/gen/python-templates/yaml.py.mustache | 12 +++++++++++- sdk/python/pulumi_kubernetes/helm/v2/helm.py | 13 ++++++++++++- sdk/python/pulumi_kubernetes/yaml.py | 12 +++++++++++- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/pkg/gen/python-templates/helm/v2/helm.py b/pkg/gen/python-templates/helm/v2/helm.py index 63aedf4462..c574c29ed1 100644 --- a/pkg/gen/python-templates/helm/v2/helm.py +++ b/pkg/gen/python-templates/helm/v2/helm.py @@ -9,6 +9,7 @@ from typing import Any, Callable, List, Optional, TextIO, Tuple, Union import pulumi.runtime +from ...version import get_version from pulumi_kubernetes.yaml import _parse_yaml_document @@ -347,9 +348,19 @@ def _parse_chart(all_config: Tuple[str, Union[ChartOpts, LocalChartOpts], pulumi cmd.extend(home_arg) chart_resources = pulumi.Output.all(cmd, data).apply(_run_helm_cmd) + + # Rather than using the default provider for the following invoke call, determine the + # provider from the parent if specified, or fallback to using the version specified + # in package.json. + invoke_opts = pulumi.InvokeOptions() + if opts.parent is not None: + invoke_opts.parent = opts.parent + else: + invoke_opts.version = get_version() + objects = chart_resources.apply( lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', { - 'text': text, 'defaultNamespace': config.namespace}).value['result']) + 'text': text, 'defaultNamespace': config.namespace}, invoke_opts).value['result']) # Parse the manifest and create the specified resources. resources = objects.apply( diff --git a/pkg/gen/python-templates/yaml.py.mustache b/pkg/gen/python-templates/yaml.py.mustache index 1294b03d4e..ab4300c5cf 100644 --- a/pkg/gen/python-templates/yaml.py.mustache +++ b/pkg/gen/python-templates/yaml.py.mustache @@ -57,10 +57,20 @@ class ConfigFile(pulumi.ComponentResource): opts = pulumi.ResourceOptions.merge(opts, pulumi.ResourceOptions(parent=self)) + # Rather than using the default provider for the following invoke call, determine the + # provider from the parent if specified, or fallback to using the version specified + # in package.json. + invoke_opts = pulumi.InvokeOptions() + if opts.parent is not None: + invoke_opts.parent = opts.parent + else: + invoke_opts.version = get_version() + + __ret__ = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}, invoke_opts).value['result'] + # Note: Unlike NodeJS, Python requires that we "pull" on our futures in order to get them scheduled for # execution. In order to do this, we leverage the engine's RegisterResourceOutputs to wait for the # resolution of all resources that this YAML document created. - __ret__ = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result'] self.resources = _parse_yaml_document(__ret__, opts, transformations, resource_prefix) self.register_outputs({"resources": self.resources}) diff --git a/sdk/python/pulumi_kubernetes/helm/v2/helm.py b/sdk/python/pulumi_kubernetes/helm/v2/helm.py index 63aedf4462..c574c29ed1 100644 --- a/sdk/python/pulumi_kubernetes/helm/v2/helm.py +++ b/sdk/python/pulumi_kubernetes/helm/v2/helm.py @@ -9,6 +9,7 @@ from typing import Any, Callable, List, Optional, TextIO, Tuple, Union import pulumi.runtime +from ...version import get_version from pulumi_kubernetes.yaml import _parse_yaml_document @@ -347,9 +348,19 @@ def _parse_chart(all_config: Tuple[str, Union[ChartOpts, LocalChartOpts], pulumi cmd.extend(home_arg) chart_resources = pulumi.Output.all(cmd, data).apply(_run_helm_cmd) + + # Rather than using the default provider for the following invoke call, determine the + # provider from the parent if specified, or fallback to using the version specified + # in package.json. + invoke_opts = pulumi.InvokeOptions() + if opts.parent is not None: + invoke_opts.parent = opts.parent + else: + invoke_opts.version = get_version() + objects = chart_resources.apply( lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', { - 'text': text, 'defaultNamespace': config.namespace}).value['result']) + 'text': text, 'defaultNamespace': config.namespace}, invoke_opts).value['result']) # Parse the manifest and create the specified resources. resources = objects.apply( diff --git a/sdk/python/pulumi_kubernetes/yaml.py b/sdk/python/pulumi_kubernetes/yaml.py index af4934af11..64435a9a38 100644 --- a/sdk/python/pulumi_kubernetes/yaml.py +++ b/sdk/python/pulumi_kubernetes/yaml.py @@ -57,10 +57,20 @@ def __init__(self, name, file_id, opts=None, transformations=None, resource_pref opts = pulumi.ResourceOptions.merge(opts, pulumi.ResourceOptions(parent=self)) + # Rather than using the default provider for the following invoke call, determine the + # provider from the parent if specified, or fallback to using the version specified + # in package.json. + invoke_opts = pulumi.InvokeOptions() + if opts.parent is not None: + invoke_opts.parent = opts.parent + else: + invoke_opts.version = get_version() + + __ret__ = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}, invoke_opts).value['result'] + # Note: Unlike NodeJS, Python requires that we "pull" on our futures in order to get them scheduled for # execution. In order to do this, we leverage the engine's RegisterResourceOutputs to wait for the # resolution of all resources that this YAML document created. - __ret__ = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result'] self.resources = _parse_yaml_document(__ret__, opts, transformations, resource_prefix) self.register_outputs({"resources": self.resources}) From ed17c7671e003453340448eb6a061fc3a04aec0c Mon Sep 17 00:00:00 2001 From: Levi Blackstone Date: Thu, 6 Feb 2020 17:49:45 -0700 Subject: [PATCH 4/4] Update test assertions Using an explicit provider now, so resource counts drop by one on these tests --- tests/examples/examples_test.go | 4 ++-- tests/integration/aliases/aliases_test.go | 6 +++--- tests/integration/yaml-url/yaml_url_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/examples/examples_test.go b/tests/examples/examples_test.go index a34278f4a4..cb18999803 100644 --- a/tests/examples/examples_test.go +++ b/tests/examples/examples_test.go @@ -164,7 +164,7 @@ func TestAccHelmApiVersions(t *testing.T) { t *testing.T, stackInfo integration.RuntimeValidationStackInfo, ) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 7, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 6, len(stackInfo.Deployment.Resources)) }, }) @@ -181,7 +181,7 @@ func TestAccHelmLocal(t *testing.T) { t *testing.T, stackInfo integration.RuntimeValidationStackInfo, ) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 16, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 15, len(stackInfo.Deployment.Resources)) }, }) diff --git a/tests/integration/aliases/aliases_test.go b/tests/integration/aliases/aliases_test.go index 06f3f79610..f75cd6d30c 100644 --- a/tests/integration/aliases/aliases_test.go +++ b/tests/integration/aliases/aliases_test.go @@ -37,7 +37,7 @@ func TestAliases(t *testing.T) { Quick: true, ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 5, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 4, len(stackInfo.Deployment.Resources)) tests.SortResourcesByURN(stackInfo) @@ -51,7 +51,7 @@ func TestAliases(t *testing.T) { Additive: true, ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 5, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 4, len(stackInfo.Deployment.Resources)) tests.SortResourcesByURN(stackInfo) @@ -69,7 +69,7 @@ func TestAliases(t *testing.T) { Additive: true, ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 5, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 4, len(stackInfo.Deployment.Resources)) tests.SortResourcesByURN(stackInfo) diff --git a/tests/integration/yaml-url/yaml_url_test.go b/tests/integration/yaml-url/yaml_url_test.go index 058357b9b9..aa9087f196 100644 --- a/tests/integration/yaml-url/yaml_url_test.go +++ b/tests/integration/yaml-url/yaml_url_test.go @@ -37,7 +37,7 @@ func TestYAMLURL(t *testing.T) { assert.NotNil(t, stackInfo.Deployment) // Assert that we've retrieved the YAML from the URL and provisioned them. - assert.Equal(t, 19, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 18, len(stackInfo.Deployment.Resources)) }, }) }