filestate/upgrade: Support backfilling projects for detached stack

Adds an option to the filestate upgrade operation that, when supplied,
allows the caller to fill project names for stacks
for which we could not guess a project name.

The caller supplies a function with the following signature,
taking a list of stack names and returning a list of project names
in the same order.

    func(stacks []Name) (projects []Name, err error)

A caller like the CLI can use this hook to prompt the user for input,
allowing users to fill in project names for stacks.

changelog/pending/20230601--backend-filestate--add-an-option-to-the-upgrade-operation-allowing-injection-of-an-external-source-of-project-names-for-stacks-where-the-project-name-could-not-be-automatically-determined.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: chore
+  scope: backend/filestate
+  description: Add an option to the Upgrade operation allowing injection of an external source of project names for stacks where the project name could not be automatically determined.

pkg/backend/filestate/backend.go

@@ -91,7 +91,23 @@ var (
 )
 
 // UpgradeOptions customizes the behavior of the upgrade operation.
-type UpgradeOptions struct{}
+type UpgradeOptions struct {
+	// ProjectsForDetachedStacks is an optional function that is able to
+	// backfill project names for stacks that have no project specified otherwise.
+	//
+	// It is called with a list of stack names that have no project specified.
+	// It should return a list of project names to use for each stack name
+	// in the same order.
+	// If a returned name is blank, the stack at that position will be skipped
+	// in the upgrade process.
+	//
+	// The length of 'projects' MUST match the length of 'stacks'.
+	// If it does not, the upgrade will panic.
+	//
+	// If this function is not specified,
+	// stacks without projects will be skipped during the upgrade.
+	ProjectsForDetachedStacks func(stacks []tokens.Name) (projects []tokens.Name, err error)
+}
 
 // Backend extends the base backend interface with specific information about local backends.
 type Backend interface {
@@ -396,6 +412,45 @@ func (b *localBackend) Upgrade(ctx context.Context, opts *UpgradeOptions) error
 		return err
 	}
 
+	// If there are any stacks without projects
+	// and the user provided a callback to fill them,
+	// use it to fill in the missing projects.
+	if opts.ProjectsForDetachedStacks != nil {
+		var (
+			// Names of stacks in 'olds' that don't have a project
+			detached []tokens.Name
+
+			// reverseIdx[i] is the index of detached[i]
+			// in olds and projects.
+			//
+			// In other words:
+			//
+			//   detached[i] == olds[reverseIdx[i]].Name()
+			//   projects[reverseIdx[i]] == ""
+			reverseIdx []int
+		)
+		for i, ref := range olds {
+			if projects[i] == "" {
+				detached = append(detached, ref.Name())
+				reverseIdx = append(reverseIdx, i)
+			}
+		}
+
+		if len(detached) != 0 {
+			detachedProjects, err := opts.ProjectsForDetachedStacks(detached)
+			if err != nil {
+				return err
+			}
+			contract.Assertf(len(detached) == len(detachedProjects),
+				"ProjectsForDetachedStacks returned the wrong number of projects: "+
+					"expected %d, got %d", len(detached), len(detachedProjects))
+
+			for i, project := range detachedProjects {
+				projects[reverseIdx[i]] = project
+			}
+		}
+	}
+
 	// It's important that we attempt to write the new metadata file
 	// before we attempt the upgrade.
 	// This ensures that if permissions are borked for any reason,

pkg/backend/filestate/backend_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -33,6 +34,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/testing/diagtest"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/testing/iotest"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
 )
@@ -1033,6 +1035,125 @@ func TestLegacyUpgrade_partial(t *testing.T) {
 	assert.Equal(t, tokens.QName("organization/project/foo"), ref.FullyQualifiedName())
 }
 
+// When a stack project could not be determined,
+// we should fill it in with ProjectsForDetachedStacks.
+func TestLegacyUpgrade_ProjectsForDetachedStacks(t *testing.T) {
+	t.Parallel()
+
+	stateDir := t.TempDir()
+	bucket, err := fileblob.OpenBucket(stateDir, nil)
+	require.NoError(t, err)
+
+	// Write a few empty stacks.
+	// These stacks have no resources, so we can't guess the project name.
+	ctx := context.Background()
+	for _, stack := range []string{"foo", "bar", "baz"} {
+		statePath := path.Join(".pulumi", "stacks", stack+".json")
+		require.NoError(t,
+			bucket.WriteAll(ctx, statePath,
+				[]byte(`{"latest": {"resources": []}}`), nil),
+			"write stack %s", stack)
+	}
+
+	var stderr bytes.Buffer
+	sink := diag.DefaultSink(io.Discard, &stderr, diag.FormatOptions{Color: colors.Never})
+	b, err := New(ctx, sink, "file://"+filepath.ToSlash(stateDir), nil)
+	require.NoError(t, err, "initialize backend")
+
+	// For the first two stacks, we'll return project names to upgrade them.
+	// For the third stack, we will not set a project name, and it should be skipped.
+	err = b.Upgrade(ctx, &UpgradeOptions{
+		ProjectsForDetachedStacks: func(stacks []tokens.Name) (projects []tokens.Name, err error) {
+			assert.ElementsMatch(t, []tokens.Name{"foo", "bar", "baz"}, stacks)
+
+			projects = make([]tokens.Name, len(stacks))
+			for idx, stack := range stacks {
+				switch stack {
+				case "foo":
+					projects[idx] = "proj1"
+				case "bar":
+					projects[idx] = "proj2"
+				case "baz":
+					// Leave baz detached.
+				}
+			}
+			return projects, nil
+		},
+	})
+	require.NoError(t, err)
+
+	for _, stack := range []string{"foo", "bar"} {
+		assert.NotContains(t, stderr.String(), fmt.Sprintf("Skipping stack %q", stack))
+	}
+	assert.Contains(t, stderr.String(), fmt.Sprintf("Skipping stack %q", "baz"))
+
+	wantFiles := []string{
+		".pulumi/stacks/proj1/foo.json",
+		".pulumi/stacks/proj2/bar.json",
+		".pulumi/stacks/baz.json",
+	}
+	for _, file := range wantFiles {
+		exists, err := bucket.Exists(ctx, file)
+		require.NoError(t, err, "exists(%q)", file)
+		assert.True(t, exists, "file %q must exist", file)
+	}
+}
+
+// When a stack project could not be determined
+// and ProjectsForDetachedStacks returns an error,
+// the upgrade should fail.
+func TestLegacyUpgrade_ProjectsForDetachedStacks_error(t *testing.T) {
+	t.Parallel()
+
+	stateDir := t.TempDir()
+	bucket, err := fileblob.OpenBucket(stateDir, nil)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+
+	// We have one stack with a guessable project name, and one without.
+	// If ProjectsForDetachedStacks returns an error, the upgrade should
+	// fail for both because the user likely cancelled the upgrade.
+	require.NoError(t,
+		bucket.WriteAll(ctx, ".pulumi/stacks/foo.json", []byte(`{
+		"latest": {
+			"resources": [
+				{
+					"type": "package:module:resource",
+					"urn": "urn:pulumi:stack::project::package:module:resource::name"
+				}
+			]
+		}
+	}`), nil))
+	require.NoError(t,
+		bucket.WriteAll(ctx, ".pulumi/stacks/bar.json",
+			[]byte(`{"latest": {"resources": []}}`), nil))
+
+	sink := diag.DefaultSink(io.Discard, iotest.LogWriter(t), diag.FormatOptions{Color: colors.Never})
+	b, err := New(ctx, sink, "file://"+filepath.ToSlash(stateDir), nil)
+	require.NoError(t, err)
+
+	giveErr := errors.New("canceled operation")
+	err = b.Upgrade(ctx, &UpgradeOptions{
+		ProjectsForDetachedStacks: func(stacks []tokens.Name) (projects []tokens.Name, err error) {
+			assert.Equal(t, []tokens.Name{"bar"}, stacks)
+			return nil, giveErr
+		},
+	})
+	require.Error(t, err)
+	assert.ErrorIs(t, err, giveErr)
+
+	wantFiles := []string{
+		".pulumi/stacks/foo.json",
+		".pulumi/stacks/bar.json",
+	}
+	for _, file := range wantFiles {
+		exists, err := bucket.Exists(ctx, file)
+		require.NoError(t, err, "exists(%q)", file)
+		assert.True(t, exists, "file %q must exist", file)
+	}
+}
+
 // If an upgrade failed because we couldn't write the meta.yaml,
 // the stacks should be left in legacy mode.
 func TestLegacyUpgrade_writeMetaError(t *testing.T) {