Merge #13437

13437: [engine] Add support for source positions r=pgavlin a=pgavlin

These changes add support for passing source position information in gRPC metadata and recording the source position that corresponds to a resource registration in the statefile.

Enabling source position information in the resource model can provide substantial benefits, including but not limited to:

- Better errors from the Pulumi CLI
- Go-to-defintion for resources in state
- Editor integration for errors, etc. from `pulumi preview`

Source positions are (file, line) or (file, line, column) tuples represented as URIs. The line and column are stored in the fragment portion of the URI as "line(,column)?". The scheme of the URI and the form of its path component depends on the context in which it is generated or used:

- During an active update, the URI's scheme is `file` and paths are absolute filesystem paths. This allows consumers to easily access arbitrary files that are available on the host.
- In a statefile, the URI's scheme is `project` and paths are relative to the project root. This allows consumers to resolve source positions relative to the project file in different contexts irrespective of the location of the project itself (e.g. given a project-relative path and the URL of the project's root on GitHub, one can build a GitHub URL for the source position).

During an update, source position information may be attached to gRPC calls as "source-position" metadata. This allows arbitrary calls to be associated with source positions without changes to their protobuf payloads. Modifying the protobuf payloads is also a viable approach, but is somewhat more invasive than attaching metadata, and requires changes to every call signature.

Source positions should reflect the position in user code that initiated a resource model operation (e.g. the source position passed with `RegisterResource` for `pet` in the example above should be the source position in `index.ts`, _not_ the source position in the Pulumi SDK). In general, the Pulumi SDK should be able to infer the source position of the resource registration, as the relationship between a resource registration and its corresponding user code should be static per SDK.

Source positions in state files will be stored as a new `registeredAt` property on each resource. This property is optional.

Part of #13436

Co-authored-by: Pat Gavlin <pat@pulumi.com>

pkg/backend/display/json.go

@@ -85,7 +85,7 @@ func stateForJSONOutput(s *resource.State, opts Options) *resource.State {
 	return resource.NewState(s.Type, s.URN, s.Custom, s.Delete, s.ID, inputs,
 		outputs, s.Parent, s.Protect, s.External, s.Dependencies, s.InitErrors, s.Provider,
 		s.PropertyDependencies, s.PendingReplacement, s.AdditionalSecretOutputs, s.Aliases, &s.CustomTimeouts,
-		s.ImportID, s.RetainOnDelete, s.DeletedWith, s.Created, s.Modified)
+		s.ImportID, s.RetainOnDelete, s.DeletedWith, s.Created, s.Modified, s.SourcePosition)
 }
 
 // ShowJSONEvents renders incremental engine events to stdout.

pkg/backend/snapshot.go

@@ -243,6 +243,12 @@ func (ssm *sameSnapshotMutation) mustWrite(step *deploy.SameStep) bool {
 		return true
 	}
 
+	// If the source position of this resource has changed, we must write the checkpoint.
+	if old.SourcePosition != new.SourcePosition {
+		logging.V(9).Infof("SnapshotManager: mustWrite() true because of SourcePosition")
+		return true
+	}
+
 	// If the inputs or outputs of this resource have changed, we must write the checkpoint. Note that it is possible
 	// for the inputs of a "same" resource to have changed even if the contents of the input bags are different if the
 	// resource's provider deems the physical change to be semantically irrelevant.

pkg/backend/snapshot_test.go

@@ -342,6 +342,10 @@ func TestSamesWithOtherMeaningfulChanges(t *testing.T) {
 	changes = append(changes, NewResource(string(resourceA.URN)))
 	changes[3].Outputs = resource.PropertyMap{"foo": resource.NewStringProperty("bar")}
 
+	// Change the resource source position.
+	changes = append(changes, NewResource(string(resourceA.URN)))
+	changes[4].SourcePosition = "project:///foo.ts#1,2"
+
 	snap := NewSnapshot([]*resource.State{
 		provider,
 		resourceP,

pkg/engine/deployment.go

@@ -152,7 +152,7 @@ type deploymentOptions struct {
 
 // deploymentSourceFunc is a callback that will be used to prepare for, and evaluate, the "new" state for a stack.
 type deploymentSourceFunc func(
-	client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main string,
+	client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main, projectRoot string,
 	target *deploy.Target, plugctx *plugin.Context, dryRun bool) (deploy.Source, error)
 
 // newDeployment creates a new deployment with the given context and options.
@@ -186,7 +186,7 @@ func newDeployment(ctx *Context, info *deploymentContext, opts deploymentOptions
 	opts.trustDependencies = proj.TrustResourceDependencies()
 	// Now create the state source.  This may issue an error if it can't create the source.  This entails,
 	// for example, loading any plugins which will be required to execute a program, among other things.
-	source, err := opts.SourceFunc(ctx.BackendClient, opts, proj, pwd, main, target, plugctx, dryRun)
+	source, err := opts.SourceFunc(ctx.BackendClient, opts, proj, pwd, main, projinfo.Root, target, plugctx, dryRun)
 	if err != nil {
 		contract.IgnoreClose(plugctx)
 		return nil, err

pkg/engine/destroy.go

@@ -64,7 +64,7 @@ func Destroy(
 }
 
 func newDestroySource(
-	client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main string,
+	client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main, projectRoot string,
 	target *deploy.Target, plugctx *plugin.Context, dryRun bool,
 ) (deploy.Source, error) {
 	// Like Update, we need to gather the set of plugins necessary to delete everything in the snapshot.

pkg/engine/lifecycletest/import_test.go

@@ -67,7 +67,7 @@ func TestImportOption(t *testing.T) {
 	program := deploytest.NewLanguageRuntime(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error {
 		var err error
 		if readID != "" {
-			_, _, err = monitor.ReadResource("pkgA:m:typA", "resA", readID, "", resource.PropertyMap{}, "", "")
+			_, _, err = monitor.ReadResource("pkgA:m:typA", "resA", readID, "", resource.PropertyMap{}, "", "", "")
 		} else {
 			_, _, _, err = monitor.RegisterResource("pkgA:m:typA", "resA", true, deploytest.ResourceOptions{
 				Inputs:   inputs,

pkg/engine/lifecycletest/pulumi_test.go

@@ -431,7 +431,7 @@ func TestBadResourceType(t *testing.T) {
 		assert.Equal(t, codes.InvalidArgument, rpcerr.Code())
 		assert.Contains(t, rpcerr.Message(), "Type 'very:bad' is not a valid type token")
 
-		_, _, err = mon.ReadResource("very:bad", "someResource", "someId", "", resource.PropertyMap{}, "", "")
+		_, _, err = mon.ReadResource("very:bad", "someResource", "someId", "", resource.PropertyMap{}, "", "", "")
 		assert.Error(t, err)
 		rpcerr, ok = rpcerror.FromError(err)
 		assert.True(t, ok)
@@ -2862,7 +2862,7 @@ func TestMissingRead(t *testing.T) {
 
 	// Our program reads a resource and exits.
 	program := deploytest.NewLanguageRuntime(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error {
-		_, _, err := monitor.ReadResource("pkgA:m:typA", "resA", "resA-some-id", "", resource.PropertyMap{}, "", "")
+		_, _, err := monitor.ReadResource("pkgA:m:typA", "resA", "resA-some-id", "", resource.PropertyMap{}, "", "", "")
 		assert.Error(t, err)
 		return nil
 	})
@@ -4253,7 +4253,7 @@ func TestInvalidGetIDReportsUserError(t *testing.T) {
 	}
 
 	program := deploytest.NewLanguageRuntime(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error {
-		_, _, err := monitor.ReadResource("pkgA:m:typA", "resA", "", "", resource.PropertyMap{}, "", "")
+		_, _, err := monitor.ReadResource("pkgA:m:typA", "resA", "", "", resource.PropertyMap{}, "", "", "")
 		assert.Error(t, err)
 		return nil
 	})
@@ -5389,3 +5389,62 @@ func TestResourceNames(t *testing.T) {
 		})
 	}
 }
+
+func TestSourcePositions(t *testing.T) {
+	t.Parallel()
+
+	loaders := []*deploytest.ProviderLoader{
+		deploytest.NewProviderLoader("pkgA", semver.MustParse("1.0.0"), func() (plugin.Provider, error) {
+			return &deploytest.Provider{
+				CreateF: func(urn resource.URN, inputs resource.PropertyMap, timeout float64,
+					preview bool,
+				) (resource.ID, resource.PropertyMap, resource.Status, error) {
+					return "created-id", inputs, resource.StatusOK, nil
+				},
+				ReadF: func(urn resource.URN, id resource.ID,
+					inputs, state resource.PropertyMap,
+				) (plugin.ReadResult, resource.Status, error) {
+					return plugin.ReadResult{Inputs: inputs, Outputs: state}, resource.StatusOK, nil
+				},
+			}, nil
+		}),
+	}
+
+	const regPos = "/test/source/positions#1,2"
+	const readPos = "/test/source/positions#3,4"
+	inputs := resource.PropertyMap{}
+	program := deploytest.NewLanguageRuntime(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error {
+		_, _, _, err := monitor.RegisterResource("pkgA:m:typA", "resA", true, deploytest.ResourceOptions{
+			Inputs:         inputs,
+			SourcePosition: "file://" + regPos,
+		})
+		require.NoError(t, err)
+
+		_, _, err = monitor.ReadResource("pkgA:m:typA", "resB", "id", "", inputs, "", "", "file://"+readPos)
+		require.NoError(t, err)
+
+		return nil
+	})
+	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
+
+	p := &TestPlan{
+		Options: UpdateOptions{Host: host},
+	}
+	regURN := p.NewURN("pkgA:m:typA", "resA", "")
+	readURN := p.NewURN("pkgA:m:typA", "resB", "")
+
+	// Run the initial update.
+	project := p.GetProject()
+	snap, res := TestOp(Update).Run(project, p.GetTarget(t, nil), p.Options, false, p.BackendClient, nil)
+	assert.Nil(t, res)
+
+	assert.Len(t, snap.Resources, 3)
+
+	reg := snap.Resources[1]
+	assert.Equal(t, regURN, reg.URN)
+	assert.Equal(t, "project://"+regPos, reg.SourcePosition)
+
+	read := snap.Resources[2]
+	assert.Equal(t, readURN, read.URN)
+	assert.Equal(t, "project://"+readPos, read.SourcePosition)
+}

pkg/engine/lifecycletest/refresh_test.go

@@ -91,7 +91,7 @@ func TestExternalRefresh(t *testing.T) {
 
 	// Our program reads a resource and exits.
 	program := deploytest.NewLanguageRuntime(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error {
-		_, _, err := monitor.ReadResource("pkgA:m:typA", "resA", "resA-some-id", "", resource.PropertyMap{}, "", "")
+		_, _, err := monitor.ReadResource("pkgA:m:typA", "resA", "resA-some-id", "", resource.PropertyMap{}, "", "", "")
 		if !assert.NoError(t, err) {
 			t.FailNow()
 		}

pkg/engine/lifecycletest/step_generator_test.go

@@ -30,7 +30,7 @@ func TestDuplicateURN(t *testing.T) {
 		assert.Error(t, err)
 
 		// Reads use the same URN namespace as register so make sure this also errors
-		_, _, err = monitor.ReadResource("pkgA:m:typA", "resA", "id", "", resource.PropertyMap{}, "", "")
+		_, _, err = monitor.ReadResource("pkgA:m:typA", "resA", "id", "", resource.PropertyMap{}, "", "", "")
 		assert.Error(t, err)
 
 		return nil

pkg/engine/refresh.go

@@ -67,8 +67,8 @@ func Refresh(
 	}, dryRun)
 }
 
-func newRefreshSource(client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main string,
-	target *deploy.Target, plugctx *plugin.Context, dryRun bool,
+func newRefreshSource(client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main,
+	projectRoot string, target *deploy.Target, plugctx *plugin.Context, dryRun bool,
 ) (deploy.Source, error) {
 	// Like Update, we need to gather the set of plugins necessary to refresh everything in the snapshot.
 	// Unlike Update, we don't actually run the user's program so we only need the set of plugins described

pkg/engine/update.go

@@ -368,7 +368,7 @@ func installAndLoadPolicyPlugins(plugctx *plugin.Context, d diag.Sink, policies
 }
 
 func newUpdateSource(
-	client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main string,
+	client deploy.BackendClient, opts deploymentOptions, proj *workspace.Project, pwd, main, projectRoot string,
 	target *deploy.Target, plugctx *plugin.Context, dryRun bool,
 ) (deploy.Source, error) {
 	//
@@ -418,11 +418,12 @@ func newUpdateSource(
 
 	// If that succeeded, create a new source that will perform interpretation of the compiled program.
 	return deploy.NewEvalSource(plugctx, &deploy.EvalRunInfo{
-		Proj:    proj,
-		Pwd:     pwd,
-		Program: main,
-		Args:    args,
-		Target:  target,
+		Proj:        proj,
+		Pwd:         pwd,
+		Program:     main,
+		ProjectRoot: projectRoot,
+		Args:        args,
+		Target:      target,
 	}, defaultProviderVersions, dryRun), nil
 }
 

pkg/resource/deploy/deploytest/resourcemonitor.go

@@ -17,6 +17,9 @@ package deploytest
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
@@ -79,6 +82,35 @@ func supportsFeature(ctx context.Context, resmon pulumirpc.ResourceMonitorClient
 	return resp.GetHasSupport(), nil
 }
 
+func parseSourcePosition(raw string) (*pulumirpc.SourcePosition, error) {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return nil, err
+	}
+
+	pos := pulumirpc.SourcePosition{
+		Uri: fmt.Sprintf("%v://%v", u.Scheme, u.Path),
+	}
+
+	line, col, _ := strings.Cut(u.Fragment, ",")
+	if line != "" {
+		l, err := strconv.ParseInt(line, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		pos.Line = int32(l)
+	}
+	if col != "" {
+		c, err := strconv.ParseInt(col, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		pos.Column = int32(c)
+	}
+
+	return &pos, nil
+}
+
 func (rm *ResourceMonitor) Close() error {
 	return rm.conn.Close()
 }
@@ -111,6 +143,7 @@ type ResourceOptions struct {
 	AdditionalSecretOutputs []resource.PropertyKey
 	AliasSpecs              bool
 
+	SourcePosition            string
 	DisableSecrets            bool
 	DisableResourceReferences bool
 	GrpcRequestHeaders        map[string]string
@@ -203,6 +236,15 @@ func (rm *ResourceMonitor) RegisterResource(t tokens.Type, name string, custom b
 	for i, v := range opts.AdditionalSecretOutputs {
 		additionalSecretOutputs[i] = string(v)
 	}
+
+	var sourcePosition *pulumirpc.SourcePosition
+	if opts.SourcePosition != "" {
+		sourcePosition, err = parseSourcePosition(opts.SourcePosition)
+		if err != nil {
+			return "", "", nil, err
+		}
+	}
+
 	requestInput := &pulumirpc.RegisterResourceRequest{
 		Type:                       string(t),
 		Name:                       name,
@@ -232,6 +274,7 @@ func (rm *ResourceMonitor) RegisterResource(t tokens.Type, name string, custom b
 		Aliases:                    aliasObjects,
 		DeletedWith:                string(opts.DeletedWith),
 		AliasSpecs:                 opts.AliasSpecs,
+		SourcePosition:             sourcePosition,
 	}
 
 	ctx := context.Background()
@@ -279,7 +322,7 @@ func (rm *ResourceMonitor) RegisterResourceOutputs(urn resource.URN, outputs res
 }
 
 func (rm *ResourceMonitor) ReadResource(t tokens.Type, name string, id resource.ID, parent resource.URN,
-	inputs resource.PropertyMap, provider string, version string,
+	inputs resource.PropertyMap, provider, version, sourcePosition string,
 ) (resource.URN, resource.PropertyMap, error) {
 	// marshal inputs
 	ins, err := plugin.MarshalProperties(inputs, plugin.MarshalOptions{
@@ -290,15 +333,24 @@ func (rm *ResourceMonitor) ReadResource(t tokens.Type, name string, id resource.
 		return "", nil, err
 	}
 
+	var sourcePos *pulumirpc.SourcePosition
+	if sourcePosition != "" {
+		sourcePos, err = parseSourcePosition(sourcePosition)
+		if err != nil {
+			return "", nil, err
+		}
+	}
+
 	// submit request
 	resp, err := rm.resmon.ReadResource(context.Background(), &pulumirpc.ReadResourceRequest{
-		Type:       string(t),
-		Name:       name,
-		Id:         string(id),
-		Parent:     string(parent),
-		Provider:   provider,
-		Properties: ins,
-		Version:    version,
+		Type:           string(t),
+		Name:           name,
+		Id:             string(id),
+		Parent:         string(parent),
+		Provider:       provider,
+		Properties:     ins,
+		Version:        version,
+		SourcePosition: sourcePos,
 	})
 	if err != nil {
 		return "", nil, err

pkg/resource/deploy/import.go

@@ -167,7 +167,7 @@ func (i *importer) getOrCreateStackResource(ctx context.Context) (resource.URN,
 	typ, name := resource.RootStackType, fmt.Sprintf("%s-%s", projectName, stackName)
 	urn := resource.NewURN(stackName.Q(), projectName, "", typ, tokens.QName(name))
 	state := resource.NewState(typ, urn, false, false, "", resource.PropertyMap{}, nil, "", false, false, nil, nil, "",
-		nil, false, nil, nil, nil, "", false, "", nil, nil)
+		nil, false, nil, nil, nil, "", false, "", nil, nil, "")
 	// TODO(seqnum) should stacks be created with 1? When do they ever get recreated/replaced?
 	if !i.executeSerial(ctx, NewCreateStep(i.deployment, noopEvent(0), state)) {
 		return "", false, false
@@ -255,7 +255,7 @@ func (i *importer) registerProviders(ctx context.Context) (map[resource.URN]stri
 		}
 
 		state := resource.NewState(typ, urn, true, false, "", inputs, nil, "", false, false, nil, nil, "", nil, false,
-			nil, nil, nil, "", false, "", nil, nil)
+			nil, nil, nil, "", false, "", nil, nil, "")
 		// TODO(seqnum) should default providers be created with 1? When do they ever get recreated/replaced?
 		if issueCheckErrors(i.deployment, state, urn, failures) {
 			return nil, nil, false
@@ -355,7 +355,7 @@ func (i *importer) importResources(ctx context.Context) result.Result {
 
 		// Create the new desired state. Note that the resource is protected.
 		new := resource.NewState(urn.Type(), urn, true, false, imp.ID, resource.PropertyMap{}, nil, parent, imp.Protect,
-			false, nil, nil, provider, nil, false, nil, nil, nil, "", false, "", nil, nil)
+			false, nil, nil, provider, nil, false, nil, nil, nil, "", false, "", nil, nil, "")
 		steps = append(steps, newImportDeploymentStep(i.deployment, new, randomSeed))
 	}
 

pkg/resource/deploy/source.go

@@ -125,6 +125,8 @@ type ReadResourceEvent interface {
 	Done(result *ReadResult)
 	// The names of any additional outputs that should be treated as secrets.
 	AdditionalSecretOutputs() []resource.PropertyKey
+	// The source position of the resource read
+	SourcePosition() string
 }
 
 type ReadResult struct {