filestate: Re-add project support

This re-adds project support back to the filestate backend
by implementing a new referenceStore: projectReferenceStore.

We will use this reference store for all new filestate stores.
Existing states will continue to use the legacyReferenceStore.

To accomplish this, and to plan for the future,
we introduce a 'meta.yaml' file inside the .pulumi directory.
This file contains metadata about the storage state.
Currently, this only holds a version number:

    # .pulumi/meta.yaml
    version: 1

Version 1 is the number we've chosen for the initial release
of project support.
If we ever need to make breaking changes to the storage protocol
we can bump the format version.

Notes:

- Stack references produced by filestate will shorten to
  just the stack name if the project name for the stack
  matches the currently selected project.
  This required turning currentProject on localBackend
  into an atomic pointer because otherwise
  SetCurrentProject and localBackendReference.String may race.

Extracted from #12134

Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>

changelog/pending/20230128--backend-filestate--the-filestate-backend-now-supports-project-scoped-stacks.yaml

@@ -0,0 +1,8 @@
+changes:
+- type: feat
+  scope: backend/filestate
+  description: |
+    Add support for project-scoped stacks.
+    Newly initialized storage will automatically use this mode.
+    This mode needs write access to the root of the .pulumi directory;
+    if you're using a cloud storage, be sure to update your ACLs.

pkg/backend/backend.go

@@ -74,7 +74,7 @@ type StackReference interface {
 	// but that information is not part of the StackName() we pass to the engine.
 	Name() tokens.Name
 
-	// Fully qualified name of the stack.
+	// Fully qualified name of the stack, including any organization, project, or other information.
 	FullyQualifiedName() tokens.QName
 }
 

pkg/backend/filestate/backend.go

@@ -91,29 +91,65 @@ type localBackend struct {
 	currentProject atomic.Pointer[workspace.Project]
 
 	// The store controls the layout of stacks in the backend.
+	// We use different layouts based on the version of the backend
+	// specified in the metadata file.
+	// If the metadata file is missing, we use the legacy layout.
 	store referenceStore
 }
 
 type localBackendReference struct {
-	name tokens.Name
+	name    tokens.Name
+	project tokens.Name
+
+	// A thread-safe way to get the current project.
+	// The function reference or the pointer returned by the function may be nil.
+	currentProject func() *workspace.Project
 
 	// referenceStore that created this reference.
+	//
+	// This is necessary because
+	// the referenceStore for a backend may change over time,
+	// but the store for this reference should not.
 	store referenceStore
 }
 
 func (r *localBackendReference) String() string {
-	return string(r.name)
+	// If project is blank this is a legacy non-project scoped stack reference, just return the name.
+	if r.project == "" {
+		return string(r.name)
+	}
+
+	if r.currentProject != nil {
+		proj := r.currentProject()
+		// For project scoped references when stringifying backend references,
+		// we take the current project (if present) into account.
+		// If the project names match, we can elide them.
+		if proj != nil && string(r.project) == string(proj.Name) {
+			return string(r.name)
+		}
+	}
+
+	// Else return a new style fully qualified reference.
+	return fmt.Sprintf("organization/%s/%s", r.project, r.name)
 }
 
 func (r *localBackendReference) Name() tokens.Name {
 	return r.name
 }
 
+func (r *localBackendReference) Project() tokens.Name {
+	return r.project
+}
+
 func (r *localBackendReference) FullyQualifiedName() tokens.QName {
-	return r.Name().Q()
+	if r.project == "" {
+		return r.name.Q()
+	}
+	return tokens.QName(fmt.Sprintf("organization/%s/%s", r.project, r.name))
 }
 
 // Helper methods that delegate to the underlying referenceStore.
+func (r *localBackendReference) Validate() error       { return r.store.ValidateReference(r) }
 func (r *localBackendReference) StackBasePath() string { return r.store.StackBasePath(r) }
 func (r *localBackendReference) HistoryDir() string    { return r.store.HistoryDir(r) }
 func (r *localBackendReference) BackupDir() string     { return r.store.BackupDir(r) }
@@ -129,6 +165,10 @@ func IsFileStateBackendURL(urlstr string) bool {
 
 const FilePathPrefix = "file://"
 
+// New constructs a new filestate backend,
+// using the given URL as the root for storage.
+// The URL must use one of the schemes supported by the go-cloud blob package.
+// Thes inclue: file, s3, gs, azblob.
 func New(ctx context.Context, d diag.Sink, originalURL string, project *workspace.Project) (Backend, error) {
 	if !IsFileStateBackendURL(originalURL) {
 		return nil, fmt.Errorf("local URL %s has an illegal prefix; expected one of: %s",
@@ -190,17 +230,23 @@ func New(ctx context.Context, d diag.Sink, originalURL string, project *workspac
 		bucket:      wbucket,
 		lockID:      lockID.String(),
 		gzip:        gzipCompression,
-		store:       newLegacyReferenceStore(wbucket),
 	}
 	backend.currentProject.Store(project)
 
 	// Read the Pulumi state metadata
 	// and ensure that it is compatible with this version of the CLI.
+	// The version in the metadata file informs which store we use.
 	meta, err := ensurePulumiMeta(ctx, wbucket)
 	if err != nil {
 		return nil, err
 	}
-	if meta.Version != 0 {
+
+	switch meta.Version {
+	case 0:
+		backend.store = newLegacyReferenceStore(wbucket)
+	case 1:
+		backend.store = newProjectReferenceStore(wbucket, backend.currentProject.Load)
+	default:
 		return nil, fmt.Errorf(
 			"state store unsupported: 'meta.yaml' version (%d) is not supported "+
 				"by this version of the Pulumi CLI", meta.Version)
@@ -267,7 +313,7 @@ func (b *localBackend) getReference(ref backend.StackReference) (*localBackendRe
 	if !ok {
 		return nil, fmt.Errorf("bad stack reference type")
 	}
-	return stackRef, nil
+	return stackRef, stackRef.Validate()
 }
 
 func (b *localBackend) local() {}
@@ -330,10 +376,56 @@ func (b *localBackend) ValidateStackName(stackRef string) error {
 }
 
 func (b *localBackend) DoesProjectExist(ctx context.Context, projectName string) (bool, error) {
-	// Local backends don't really have multiple projects, so just return false here.
+	projStore, ok := b.store.(*projectReferenceStore)
+	if !ok {
+		// Legacy stores don't have projects
+		// so the project does not exist.
+		return false, nil
+	}
+
+	// TODO[pulumi/pulumi#12547]:
+	// This could be faster if we list "$project/" instead of all projects.
+	projects, err := projStore.ListProjects()
+	if err != nil {
+		return false, err
+	}
+
+	for _, project := range projects {
+		if string(project) == projectName {
+			return true, nil
+		}
+	}
+
 	return false, nil
 }
 
+// Confirm the specified stack's project doesn't contradict the meta.yaml of the current project.
+// If the CWD is not in a Pulumi project, does not contradict.
+// If the project name in Pulumi.yaml is "foo", a stack with a name of bar/foo should not work.
+func currentProjectContradictsWorkspace(stack *localBackendReference) bool {
+	contract.Requiref(stack != nil, "stack", "is nil")
+
+	if stack.project == "" {
+		return false
+	}
+
+	projPath, err := workspace.DetectProjectPath()
+	if err != nil {
+		return false
+	}
+
+	if projPath == "" {
+		return false
+	}
+
+	proj, err := workspace.LoadProject(projPath)
+	if err != nil {
+		return false
+	}
+
+	return proj.Name.String() != stack.project.String()
+}
+
 func (b *localBackend) CreateStack(ctx context.Context, stackRef backend.StackReference,
 	root string, opts *backend.CreateStackOptions,
 ) (backend.Stack, error) {
@@ -352,6 +444,10 @@ func (b *localBackend) CreateStack(ctx context.Context, stackRef backend.StackRe
 	}
 	defer b.Unlock(ctx, stackRef)
 
+	if currentProjectContradictsWorkspace(localStackRef) {
+		return nil, fmt.Errorf("provided project name %q doesn't match Pulumi.yaml", localStackRef.project)
+	}
+
 	stackName := localStackRef.FullyQualifiedName()
 	if stackName == "" {
 		return nil, errors.New("invalid empty stack name")
@@ -617,6 +713,10 @@ func (b *localBackend) apply(
 		return nil, nil, result.FromError(err)
 	}
 
+	if currentProjectContradictsWorkspace(localStackRef) {
+		return nil, nil, result.Errorf("provided project name %q doesn't match Pulumi.yaml", localStackRef.project)
+	}
+
 	stackName := stackRef.FullyQualifiedName()
 	actionLabel := backend.ActionLabel(kind, opts.DryRun)