Upgrade output values in source_eval

changelog/pending/20240206--engine--translate-all-computed-and-secret-values-to-outputvalues-for-construct-and-call-methods.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: engine
+  description: Translate all Computed and Secret values to OutputValues for Construct and Call methods.

pkg/engine/lifecycletest/pulumi_test.go

@@ -4783,3 +4783,89 @@ func TestAutomaticDiff(t *testing.T) {
 		})
 	assert.NoError(t, err)
 }
+
+// Test that the engine only sends OutputValues for Construct and Call
+func TestConstructCallSecretsUnknowns(t *testing.T) {
+	t.Parallel()
+
+	loaders := []*deploytest.ProviderLoader{
+		deploytest.NewProviderLoader("pkgA", semver.MustParse("1.0.0"), func() (plugin.Provider, error) {
+			return &deploytest.Provider{
+				CreateF: func(urn resource.URN, inputs resource.PropertyMap, timeout float64,
+					preview bool,
+				) (resource.ID, resource.PropertyMap, resource.Status, error) {
+					return "created-id", inputs, resource.StatusOK, nil
+				},
+				ReadF: func(urn resource.URN, id resource.ID,
+					inputs, state resource.PropertyMap,
+				) (plugin.ReadResult, resource.Status, error) {
+					return plugin.ReadResult{Inputs: inputs, Outputs: state}, resource.StatusOK, nil
+				},
+				ConstructF: func(monitor *deploytest.ResourceMonitor, typ, name string, parent resource.URN,
+					inputs resource.PropertyMap, info plugin.ConstructInfo, options plugin.ConstructOptions,
+				) (plugin.ConstructResult, error) {
+					// Assert that "foo" is secret and "bar" is unknown
+					foo := inputs["foo"]
+					assert.True(t, foo.IsOutput())
+					assert.True(t, foo.OutputValue().Known)
+					assert.True(t, foo.OutputValue().Secret)
+
+					bar := inputs["bar"]
+					assert.True(t, bar.IsOutput())
+					assert.False(t, bar.OutputValue().Known)
+					assert.False(t, bar.OutputValue().Secret)
+
+					urn, _, _, err := monitor.RegisterResource(tokens.Type(typ), name, false, deploytest.ResourceOptions{})
+					assert.NoError(t, err)
+
+					return plugin.ConstructResult{
+						URN: urn,
+					}, nil
+				},
+				CallF: func(monitor *deploytest.ResourceMonitor, tok tokens.ModuleMember, args resource.PropertyMap, info plugin.CallInfo,
+					options plugin.CallOptions,
+				) (plugin.CallResult, error) {
+					// Assert that "foo" is secret and "bar" is unknown
+					foo := args["foo"]
+					assert.True(t, foo.IsOutput())
+					assert.True(t, foo.OutputValue().Known)
+					assert.True(t, foo.OutputValue().Secret)
+
+					bar := args["bar"]
+					assert.True(t, bar.IsOutput())
+					assert.False(t, bar.OutputValue().Known)
+					assert.False(t, bar.OutputValue().Secret)
+
+					return plugin.CallResult{}, nil
+				},
+			}, nil
+		}),
+	}
+
+	programF := deploytest.NewLanguageRuntimeF(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error {
+		inputs := resource.PropertyMap{
+			"foo": resource.MakeSecret(resource.NewStringProperty("foo")),
+			"bar": resource.MakeComputed(resource.NewStringProperty("")),
+		}
+
+		_, _, _, err := monitor.RegisterResource("pkgA:m:typA", "resA", false, deploytest.ResourceOptions{
+			Remote: true,
+			Inputs: inputs,
+		})
+		assert.NoError(t, err)
+
+		_, _, _, err = monitor.Call("pkgA:m:typA", inputs, "", "")
+		assert.NoError(t, err)
+
+		return nil
+	})
+	hostF := deploytest.NewPluginHostF(nil, nil, programF, loaders...)
+
+	p := &TestPlan{
+		Options: TestUpdateOptions{HostF: hostF},
+	}
+
+	project := p.GetProject()
+	_, err := TestOp(Update).Run(project, p.GetTarget(t, nil), p.Options, false, p.BackendClient, nil)
+	assert.NoError(t, err)
+}

pkg/resource/deploy/deploytest/resourcemonitor.go

@@ -394,8 +394,10 @@ func (rm *ResourceMonitor) Call(tok tokens.ModuleMember, inputs resource.Propert
 ) {
 	// marshal inputs
 	ins, err := plugin.MarshalProperties(inputs, plugin.MarshalOptions{
-		KeepUnknowns:  true,
-		KeepResources: true,
+		KeepUnknowns:     true,
+		KeepResources:    true,
+		KeepSecrets:      true,
+		KeepOutputValues: true,
 	})
 	if err != nil {
 		return nil, nil, nil, err
@@ -419,8 +421,10 @@ func (rm *ResourceMonitor) Call(tok tokens.ModuleMember, inputs resource.Propert
 
 	// unmarshal outputs
 	outs, err := plugin.UnmarshalProperties(resp.Return, plugin.MarshalOptions{
-		KeepUnknowns:  true,
-		KeepResources: true,
+		KeepUnknowns:     true,
+		KeepResources:    true,
+		KeepSecrets:      true,
+		KeepOutputValues: true,
 	})
 	if err != nil {
 		return nil, nil, nil, err

pkg/resource/deploy/source_eval.go

@@ -828,7 +828,8 @@ func (rm *resmon) Call(ctx context.Context, req *pulumirpc.CallRequest) (*pulumi
 			KeepResources: true,
 			// To initially scope the use of this new feature, we only keep output values when unmarshaling
 			// properties for RegisterResource (when remote is true for multi-lang components) and Call.
-			KeepOutputValues: true,
+			KeepOutputValues:      true,
+			UpgradeToOutputValues: true,
 		})
 	if err != nil {
 		return nil, fmt.Errorf("failed to unmarshal %v args: %w", tok, err)
@@ -1363,14 +1364,13 @@ func (rm *resmon) RegisterResource(ctx context.Context,
 
 	props, err := plugin.UnmarshalProperties(
 		req.GetObject(), plugin.MarshalOptions{
-			Label:              label,
-			KeepUnknowns:       true,
-			ComputeAssetHashes: true,
-			KeepSecrets:        true,
-			KeepResources:      true,
-			// To initially scope the use of this new feature, we only keep output values when unmarshaling
-			// properties for RegisterResource (when remote is true for multi-lang components) and Call.
-			KeepOutputValues: remote,
+			Label:                 label,
+			KeepUnknowns:          true,
+			ComputeAssetHashes:    true,
+			KeepSecrets:           true,
+			KeepResources:         true,
+			KeepOutputValues:      true,
+			UpgradeToOutputValues: true,
 		})
 	if err != nil {
 		return nil, err
@@ -1430,6 +1430,16 @@ func (rm *resmon) RegisterResource(ctx context.Context,
 		deleteBeforeReplace = &deleteBeforeReplaceValue
 	}
 
+	// At this point we're going to forward these properties to the rest of the engine and potentially to providers. As
+	// we add features to the code above (most notably transforms) we could end up with more instances of `OutputValue`
+	// than the rest of the system historically expects. To minimize the disruption we downgrade `OutputValue`s with no
+	// dependencies down to `Computed` and `Secret` or their plain values. We only do this for non-remote resources.
+	// Remote resources already deal with `OutputValue`s and even though it would be more consistent to downgrade them
+	// here it would be a break change.
+	if !remote {
+		props = plugin.DowngradeOutputValues(props)
+	}
+
 	logging.V(5).Infof(
 		"ResourceMonitor.RegisterResource received: t=%v, name=%v, custom=%v, #props=%v, parent=%v, protect=%v, "+
 			"provider=%v, deps=%v, deleteBeforeReplace=%v, ignoreChanges=%v, aliases=%v, customTimeouts=%v, "+

sdk/go/common/resource/plugin/rpc.go

@@ -646,3 +646,54 @@ func MarshalArchive(v *archive.Archive, opts MarshalOptions) (*structpb.Value, e
 	pk := resource.PropertyKey(v.URI)
 	return MarshalPropertyValue(pk, resource.NewObjectProperty(serap), opts)
 }
+
+// DowngradeOutputValues recursively replaces all Output values with `Computed`, `Secret`, or their plain
+// value. This loses all dependency information.
+func DowngradeOutputValues(v resource.PropertyMap) resource.PropertyMap {
+	var downgradeOutputPropertyValue func(v resource.PropertyValue) resource.PropertyValue
+
+	downgradeOutputPropertyValue = func(v resource.PropertyValue) resource.PropertyValue {
+		if v.IsOutput() {
+			output := v.OutputValue()
+			var result resource.PropertyValue
+			if output.Known {
+				result = downgradeOutputPropertyValue(output.Element)
+			} else {
+				result = resource.MakeComputed(resource.NewStringProperty(""))
+			}
+			if output.Secret {
+				result = resource.MakeSecret(result)
+			}
+			return result
+		}
+		if v.IsObject() {
+			return resource.NewObjectProperty(DowngradeOutputValues(v.ObjectValue()))
+		}
+		if v.IsArray() {
+			var result []resource.PropertyValue
+			for _, elem := range v.ArrayValue() {
+				result = append(result, downgradeOutputPropertyValue(elem))
+			}
+			return resource.NewArrayProperty(result)
+		}
+		if v.IsSecret() {
+			return resource.MakeSecret(downgradeOutputPropertyValue(v.SecretValue().Element))
+		}
+		if v.IsResourceReference() {
+			ref := v.ResourceReferenceValue()
+			return resource.NewResourceReferenceProperty(
+				resource.ResourceReference{
+					URN:            ref.URN,
+					ID:             downgradeOutputPropertyValue(ref.ID),
+					PackageVersion: ref.PackageVersion,
+				})
+		}
+		return v
+	}
+
+	result := make(resource.PropertyMap)
+	for k, pv := range v {
+		result[k] = downgradeOutputPropertyValue(pv)
+	}
+	return result
+}