diff --git a/changelog/pending/20230128--backend-filestate--the-filestate-backend-now-supports-project-scoped-stacks.yaml b/changelog/pending/20230128--backend-filestate--the-filestate-backend-now-supports-project-scoped-stacks.yaml new file mode 100644 index 000000000000..210ab9d70cbd --- /dev/null +++ b/changelog/pending/20230128--backend-filestate--the-filestate-backend-now-supports-project-scoped-stacks.yaml @@ -0,0 +1,6 @@ +changes: +- type: feat + scope: backend/filestate + description: | + The filestate backend now supports project scoped stacks. + Newly initialized storage will automatically use this mode. diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index 3d026e75f8ff..538a3212d775 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -74,7 +74,7 @@ type StackReference interface { // but that information is not part of the StackName() we pass to the engine. Name() tokens.Name - // Fully qualified name of the stack. + // Fully qualified name of the stack, including any organization, project, or other information. FullyQualifiedName() tokens.QName } diff --git a/pkg/backend/filestate/backend.go b/pkg/backend/filestate/backend.go index b116f3e2e72b..1cbeb099422d 100644 --- a/pkg/backend/filestate/backend.go +++ b/pkg/backend/filestate/backend.go @@ -91,29 +91,64 @@ type localBackend struct { currentProject atomic.Pointer[workspace.Project] // The store controls the layout of stacks in the backend. + // We use different layouts based on the version of the backend + // specified in the metadata file. + // If the metadata file is missing, we use the legacy layout. store referenceStore } type localBackendReference struct { - name tokens.Name + name tokens.Name + project tokens.Name + + // A thread-safe way to get the current project. + // The function reference or the pointer returned by the function may be nil. + currentProject func() *workspace.Project // referenceStore that created this reference. + // + // This is necessary because the referenceStore for a backend + // may change over time. store referenceStore } func (r *localBackendReference) String() string { - return string(r.name) + // If project is blank this is a legacy non-project scoped stack reference, just return the name. + if r.project == "" { + return string(r.name) + } + + if r.currentProject != nil { + proj := r.currentProject() + // For project scoped references when stringifying backend references, + // we take the current project (if present) into account. + // If the project names match, we can elide them. + if proj != nil && string(r.project) == string(proj.Name) { + return string(r.name) + } + } + + // Else return a new style fully qualified reference. + return fmt.Sprintf("organization/%s/%s", r.project, r.name) } func (r *localBackendReference) Name() tokens.Name { return r.name } +func (r *localBackendReference) Project() tokens.Name { + return r.project +} + func (r *localBackendReference) FullyQualifiedName() tokens.QName { - return r.Name().Q() + if r.project == "" { + return r.name.Q() + } + return tokens.QName(fmt.Sprintf("organization/%s/%s", r.project, r.name)) } // Helper methods that delegate to the underlying referenceStore. +func (r *localBackendReference) Validate() error { return r.store.ValidateReference(r) } func (r *localBackendReference) StackBasePath() string { return r.store.StackBasePath(r) } func (r *localBackendReference) HistoryDir() string { return r.store.HistoryDir(r) } func (r *localBackendReference) BackupDir() string { return r.store.BackupDir(r) } @@ -129,6 +164,10 @@ func IsFileStateBackendURL(urlstr string) bool { const FilePathPrefix = "file://" +// New constructs a new filestate backend, +// using the given URL as the root for storage. +// The URL must use one of the schemes supported by the go-cloud blob package. +// Thes inclue: file, s3, gs, azblob. func New(ctx context.Context, d diag.Sink, originalURL string, project *workspace.Project) (Backend, error) { if !IsFileStateBackendURL(originalURL) { return nil, fmt.Errorf("local URL %s has an illegal prefix; expected one of: %s", @@ -190,17 +229,22 @@ func New(ctx context.Context, d diag.Sink, originalURL string, project *workspac bucket: wbucket, lockID: lockID.String(), gzip: gzipCompression, - store: newLegacyReferenceStore(wbucket), } backend.currentProject.Store(project) - // Read the Pulumi state metadata - // and ensure that it is compatible with this version of the CLI. + // Read the Pulumi state metadata, if any. + // This will tell us which store to use. meta, err := ensurePulumiMeta(ctx, wbucket) if err != nil { return nil, err } - if meta.Version != 0 { + + switch meta.Version { + case 0: + backend.store = newLegacyReferenceStore(wbucket) + case 1: + backend.store = newProjectReferenceStore(wbucket, backend.currentProject.Load) + default: return nil, fmt.Errorf( "state store unsupported: 'Pulumi.yaml' version (%d) is not supported "+ "by this version of the Pulumi CLI", meta.Version) @@ -267,7 +311,7 @@ func (b *localBackend) getReference(ref backend.StackReference) (*localBackendRe if !ok { return nil, fmt.Errorf("bad stack reference type") } - return stackRef, nil + return stackRef, stackRef.Validate() } func (b *localBackend) local() {} @@ -335,10 +379,54 @@ func (b *localBackend) ValidateStackName(stackRef string) error { } func (b *localBackend) DoesProjectExist(ctx context.Context, projectName string) (bool, error) { - // Local backends don't really have multiple projects, so just return false here. + projStore, ok := b.store.(*projectReferenceStore) + if !ok { + // Legacy stores don't have projects + // so the project does not exist. + return false, nil + } + + projects, err := projStore.ListProjects() + if err != nil { + return false, err + } + + for _, project := range projects { + if string(project) == projectName { + return true, nil + } + } + return false, nil } +// Confirm the specified stack's project doesn't contradict the Pulumi.yaml of the current project. If the CWD +// is not in a Pulumi project, does not contradict. If the project name in Pulumi.yaml is "foo", a stack with a +// name of bar/foo should not work. +func currentProjectContradictsWorkspace(stack *localBackendReference) bool { + contract.Requiref(stack != nil, "stack", "is nil") + + if stack.project == "" { + return false + } + + projPath, err := workspace.DetectProjectPath() + if err != nil { + return false + } + + if projPath == "" { + return false + } + + proj, err := workspace.LoadProject(projPath) + if err != nil { + return false + } + + return proj.Name.String() != stack.project.String() +} + func (b *localBackend) CreateStack(ctx context.Context, stackRef backend.StackReference, root string, opts *backend.CreateStackOptions, ) (backend.Stack, error) { @@ -359,6 +447,10 @@ func (b *localBackend) CreateStack(ctx context.Context, stackRef backend.StackRe } defer b.Unlock(ctx, stackRef) + if currentProjectContradictsWorkspace(localStackRef) { + return nil, fmt.Errorf("provided project name %q doesn't match Pulumi.yaml", localStackRef.project) + } + stackName := localStackRef.FullyQualifiedName() if stackName == "" { return nil, errors.New("invalid empty stack name") @@ -624,6 +716,10 @@ func (b *localBackend) apply( return nil, nil, result.FromError(err) } + if currentProjectContradictsWorkspace(localStackRef) { + return nil, nil, result.Errorf("provided project name %q doesn't match Pulumi.yaml", localStackRef.project) + } + stackName := stackRef.FullyQualifiedName() actionLabel := backend.ActionLabel(kind, opts.DryRun) diff --git a/pkg/backend/filestate/backend_test.go b/pkg/backend/filestate/backend_test.go index 8ecf272a6de1..3ab5262798f8 100644 --- a/pkg/backend/filestate/backend_test.go +++ b/pkg/backend/filestate/backend_test.go @@ -8,6 +8,7 @@ import ( "path" "path/filepath" "runtime" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -27,6 +28,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/testing/diagtest" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) func TestMassageBlobPath(t *testing.T) { @@ -152,7 +154,7 @@ func TestListStacksWithMultiplePassphrases(t *testing.T) { assert.NoError(t, err) // Create stack "a" and import a checkpoint with a secret - aStackRef, err := b.ParseStackReference("a") + aStackRef, err := b.ParseStackReference("organization/project/a") assert.NoError(t, err) aStack, err := b.CreateStack(ctx, aStackRef, "", nil) assert.NoError(t, err) @@ -170,7 +172,7 @@ func TestListStacksWithMultiplePassphrases(t *testing.T) { assert.NoError(t, err) // Create stack "b" and import a checkpoint with a secret - bStackRef, err := b.ParseStackReference("b") + bStackRef, err := b.ParseStackReference("organization/project/b") assert.NoError(t, err) bStack, err := b.CreateStack(ctx, bStackRef, "", nil) assert.NoError(t, err) @@ -212,7 +214,7 @@ func TestDrillError(t *testing.T) { assert.NoError(t, err) // Get a non-existent stack and expect a nil error because it won't be found. - stackRef, err := b.ParseStackReference("dev") + stackRef, err := b.ParseStackReference("organization/project/dev") if err != nil { t.Fatalf("unexpected error %v when parsing stack reference", err) } @@ -230,7 +232,7 @@ func TestCancel(t *testing.T) { assert.NoError(t, err) // Check that trying to cancel a stack that isn't created yet doesn't error - aStackRef, err := b.ParseStackReference("a") + aStackRef, err := b.ParseStackReference("organization/project/a") assert.NoError(t, err) err = b.CancelCurrentUpdate(ctx, aStackRef) assert.NoError(t, err) @@ -296,7 +298,7 @@ func TestRemoveMakesBackups(t *testing.T) { assert.NotNil(t, lb) // Check that creating a new stack doesn't make a backup file - aStackRef, err := lb.parseStackReference("a") + aStackRef, err := lb.parseStackReference("organization/project/a") assert.NoError(t, err) aStack, err := b.CreateStack(ctx, aStackRef, "", nil) assert.NoError(t, err) @@ -339,7 +341,7 @@ func TestRenameWorks(t *testing.T) { assert.NotNil(t, lb) // Create a new stack - aStackRef, err := lb.parseStackReference("a") + aStackRef, err := lb.parseStackReference("organization/project/a") assert.NoError(t, err) aStack, err := b.CreateStack(ctx, aStackRef, "", nil) assert.NoError(t, err) @@ -358,9 +360,9 @@ func TestRenameWorks(t *testing.T) { assert.NoError(t, err) // Rename the stack - bStackRefI, err := b.RenameStack(ctx, aStack, "b") + bStackRefI, err := b.RenameStack(ctx, aStack, "organization/project/b") assert.NoError(t, err) - assert.Equal(t, "b", bStackRefI.String()) + assert.Equal(t, "organization/project/b", bStackRefI.String()) bStackRef := bStackRefI.(*localBackendReference) // Check the new stack file now exists and the old one is gone @@ -374,9 +376,9 @@ func TestRenameWorks(t *testing.T) { // Rename again bStack, err := b.GetStack(ctx, bStackRef) assert.NoError(t, err) - cStackRefI, err := b.RenameStack(ctx, bStack, "c") + cStackRefI, err := b.RenameStack(ctx, bStack, "organization/project/c") assert.NoError(t, err) - assert.Equal(t, "c", cStackRefI.String()) + assert.Equal(t, "organization/project/c", cStackRefI.String()) cStackRef := cStackRefI.(*localBackendReference) // Check the new stack file now exists and the old one is gone @@ -408,7 +410,6 @@ func TestLoginToNonExistingFolderFails(t *testing.T) { // an error when the stack name is the empty string.TestParseEmptyStackFails func TestParseEmptyStackFails(t *testing.T) { t.Parallel() - tmpDir := t.TempDir() ctx := context.Background() b, err := New(ctx, diagtest.LogSink(t), "file://"+filepath.ToSlash(tmpDir), nil) @@ -457,7 +458,7 @@ func TestHtmlEscaping(t *testing.T) { assert.NoError(t, err) // Create stack "a" and import a checkpoint with a secret - aStackRef, err := b.ParseStackReference("a") + aStackRef, err := b.ParseStackReference("organization/project/a") assert.NoError(t, err) aStack, err := b.CreateStack(ctx, aStackRef, "", nil) assert.NoError(t, err) @@ -494,7 +495,7 @@ func TestLocalBackendRejectsStackInitOptions(t *testing.T) { ctx := context.Background() // • Simulate `pulumi stack init`, passing non-nil init options - fakeStackRef, err := local.ParseStackReference("foobar") + fakeStackRef, err := local.ParseStackReference("organization/b/foobar") assert.NoError(t, err) assert.Panics(t, func() { // • Expect a panic because the options provided illegally @@ -507,6 +508,225 @@ func TestLocalBackendRejectsStackInitOptions(t *testing.T) { }) } +func TestLegacyFolderStructure(t *testing.T) { + t.Parallel() + + // Make a dummy stack file in the legacy location + tmpDir := t.TempDir() + err := os.MkdirAll(path.Join(tmpDir, ".pulumi", "stacks"), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(path.Join(tmpDir, ".pulumi", "stacks", "a.json"), []byte("{}"), os.ModePerm) + require.NoError(t, err) + + // Login to a temp dir filestate backend + ctx := context.Background() + b, err := New(ctx, diagtest.LogSink(t), "file://"+filepath.ToSlash(tmpDir), nil) + require.NoError(t, err) + // Check the backend says it's NOT in project mode + lb, ok := b.(*localBackend) + assert.True(t, ok) + assert.NotNil(t, lb) + assert.IsType(t, &legacyReferenceStore{}, lb.store) + + // Check that list stack shows that stack + stacks, token, err := b.ListStacks(ctx, backend.ListStacksFilter{}, nil /* inContToken */) + assert.NoError(t, err) + assert.Nil(t, token) + assert.Len(t, stacks, 1) + assert.Equal(t, "a", stacks[0].Name().String()) + + // Create a new non-project stack + bRef, err := b.ParseStackReference("b") + assert.NoError(t, err) + assert.Equal(t, "b", bRef.String()) + bStack, err := b.CreateStack(ctx, bRef, "", nil) + assert.NoError(t, err) + assert.Equal(t, "b", bStack.Ref().String()) + assert.FileExists(t, path.Join(tmpDir, ".pulumi", "stacks", "b.json")) +} + +// Verifies that the StackReference.String method +// takes the current project name into account, +// even if the current project name changes +// after the stack reference is created. +func TestStackReferenceString_currentProjectChange(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + ctx := context.Background() + + b, err := New(ctx, diagtest.LogSink(t), "file://"+filepath.ToSlash(dir), nil) + require.NoError(t, err) + + foo, err := b.ParseStackReference("organization/proj1/foo") + require.NoError(t, err) + + bar, err := b.ParseStackReference("organization/proj2/bar") + require.NoError(t, err) + + assert.Equal(t, "organization/proj1/foo", foo.String()) + assert.Equal(t, "organization/proj2/bar", bar.String()) + + // Change the current project name + b.SetCurrentProject(&workspace.Project{Name: "proj1"}) + + assert.Equal(t, "foo", foo.String()) + assert.Equal(t, "organization/proj2/bar", bar.String()) +} + +// Verifies that there's no data race in calling StackReference.String +// and localBackend.SetCurrentProject concurrently. +func TestStackReferenceString_currentProjectChange_race(t *testing.T) { + t.Parallel() + + const N = 1000 + + dir := t.TempDir() + ctx := context.Background() + + b, err := New(ctx, diagtest.LogSink(t), "file://"+filepath.ToSlash(dir), nil) + require.NoError(t, err) + + projects := make([]*workspace.Project, N) + refs := make([]backend.StackReference, N) + for i := 0; i < N; i++ { + name := fmt.Sprintf("proj%d", i) + projects[i] = &workspace.Project{Name: tokens.PackageName(name)} + refs[i], err = b.ParseStackReference(fmt.Sprintf("organization/%v/foo", name)) + require.NoError(t, err) + } + + // To exercise this data race, we'll have two goroutines. + // One goroutine will call StackReference.String repeatedly + // on all the stack references, + // and the other goroutine will call localBackend.SetCurrentProject + // with all the projects. + + var wg sync.WaitGroup + ready := make(chan struct{}) // both goroutines wait on this + + wg.Add(1) + go func() { + defer wg.Done() + <-ready + for i := 0; i < N; i++ { + _ = refs[i].String() + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + <-ready + for i := 0; i < N; i++ { + b.SetCurrentProject(projects[i]) + } + }() + + close(ready) // start racing + wg.Wait() +} + +func TestProjectFolderStructure(t *testing.T) { + t.Parallel() + + // Login to a temp dir filestate backend + tmpDir := t.TempDir() + ctx := context.Background() + b, err := New(ctx, diagtest.LogSink(t), "file://"+filepath.ToSlash(tmpDir), nil) + assert.NoError(t, err) + + // Check the backend says it's in project mode + lb, ok := b.(*localBackend) + assert.True(t, ok) + assert.NotNil(t, lb) + assert.IsType(t, &projectReferenceStore{}, lb.store) + + // Make a dummy stack file in the new project location + err = os.MkdirAll(path.Join(tmpDir, ".pulumi", "stacks", "testproj"), os.ModePerm) + assert.NoError(t, err) + err = os.WriteFile(path.Join(tmpDir, ".pulumi", "stacks", "testproj", "a.json"), []byte("{}"), os.ModePerm) + assert.NoError(t, err) + + // Check that testproj is reported as existing + exists, err := b.DoesProjectExist(ctx, "testproj") + assert.NoError(t, err) + assert.True(t, exists) + + // Check that list stack shows that stack + stacks, token, err := b.ListStacks(ctx, backend.ListStacksFilter{}, nil /* inContToken */) + assert.NoError(t, err) + assert.Nil(t, token) + assert.Len(t, stacks, 1) + assert.Equal(t, "organization/testproj/a", stacks[0].Name().String()) + + // Create a new project stack + bRef, err := b.ParseStackReference("organization/testproj/b") + assert.NoError(t, err) + assert.Equal(t, "organization/testproj/b", bRef.String()) + bStack, err := b.CreateStack(ctx, bRef, "", nil) + assert.NoError(t, err) + assert.Equal(t, "organization/testproj/b", bStack.Ref().String()) + assert.FileExists(t, path.Join(tmpDir, ".pulumi", "stacks", "testproj", "b.json")) +} + +func chdir(t *testing.T, dir string) { + cwd, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Chdir(dir)) // Set directory + t.Cleanup(func() { + require.NoError(t, os.Chdir(cwd)) // Restore directory + restoredDir, err := os.Getwd() + require.NoError(t, err) + require.Equal(t, cwd, restoredDir) + }) +} + +//nolint:paralleltest // mutates cwd +func TestProjectNameMustMatch(t *testing.T) { + // Create a new project + projectDir := t.TempDir() + pyaml := filepath.Join(projectDir, "Pulumi.yaml") + err := os.WriteFile(pyaml, []byte("name: my-project\nruntime: test"), 0o600) + require.NoError(t, err) + proj, err := workspace.LoadProject(pyaml) + require.NoError(t, err) + + chdir(t, projectDir) + + // Login to a temp dir filestate backend + tmpDir := t.TempDir() + ctx := context.Background() + b, err := New(ctx, diagtest.LogSink(t), "file://"+filepath.ToSlash(tmpDir), proj) + require.NoError(t, err) + + // Create a new implicit-project stack + aRef, err := b.ParseStackReference("a") + assert.NoError(t, err) + assert.Equal(t, "a", aRef.String()) + aStack, err := b.CreateStack(ctx, aRef, "", nil) + assert.NoError(t, err) + assert.Equal(t, "a", aStack.Ref().String()) + assert.FileExists(t, path.Join(tmpDir, ".pulumi", "stacks", "my-project", "a.json")) + + // Create a new project stack with the wrong project name + bRef, err := b.ParseStackReference("organization/not-my-project/b") + assert.NoError(t, err) + assert.Equal(t, "organization/not-my-project/b", bRef.String()) + bStack, err := b.CreateStack(ctx, bRef, "", nil) + assert.Error(t, err) + assert.Nil(t, bStack) + + // Create a new project stack with the right project name + cRef, err := b.ParseStackReference("organization/my-project/c") + assert.NoError(t, err) + assert.Equal(t, "c", cRef.String()) + cStack, err := b.CreateStack(ctx, cRef, "", nil) + assert.NoError(t, err) + assert.Equal(t, "c", cStack.Ref().String()) + assert.FileExists(t, path.Join(tmpDir, ".pulumi", "stacks", "my-project", "c.json")) +} + func TestNew_unsupportedStoreVersion(t *testing.T) { t.Parallel() diff --git a/pkg/backend/filestate/bucket.go b/pkg/backend/filestate/bucket.go index d33303bdb70a..6adf42eff852 100644 --- a/pkg/backend/filestate/bucket.go +++ b/pkg/backend/filestate/bucket.go @@ -87,7 +87,12 @@ func listBucket(bucket Bucket, dir string) ([]*blob.ListObject, error) { // objectName returns the filename of a ListObject (an object from a bucket). func objectName(obj *blob.ListObject) string { - _, filename := path.Split(obj.Key) + // If obj.Key ends in "/" we want to trim that to get the name just before + key := obj.Key + if key[len(key)-1] == '/' { + key = key[0 : len(key)-1] + } + _, filename := path.Split(key) return filename } diff --git a/pkg/backend/filestate/meta.go b/pkg/backend/filestate/meta.go index 169703da1459..90d74c365a17 100644 --- a/pkg/backend/filestate/meta.go +++ b/pkg/backend/filestate/meta.go @@ -39,6 +39,7 @@ type pulumiMeta struct { // // Version 0 is the starting version. // It does not support project-scoped stacks. + // Version 1 adds support for project-scoped stacks. // // Does not use "omitempty" to differentiate // between a missing field and a zero value. @@ -64,7 +65,27 @@ func ensurePulumiMeta(ctx context.Context, b Bucket) (*pulumiMeta, error) { } // If there's no metadata file, we need to create one. - meta = &pulumiMeta{Version: 0} + // + // The version we pick for the new file decides how we lay out the state. + // + // - Version 0 is legacy mode, which is the old layout. + // - Version 1 added support for project-scoped stacks. + // + // To avoid breaking old stacks, we want to use version 0 + // if the bucket is not empty. + // + // Otherwise, for entirely new states, we'll use version 1 + // to give new users access to the latest features. + empty, err := isStateEmpty(ctx, b) + if err != nil { + return nil, err + } + + if empty { + meta = &pulumiMeta{Version: 1} + } else { + meta = &pulumiMeta{Version: 0} + } if err := meta.WriteTo(ctx, b); err != nil { return nil, err diff --git a/pkg/backend/filestate/meta_test.go b/pkg/backend/filestate/meta_test.go index bad737378961..272cbad918d8 100644 --- a/pkg/backend/filestate/meta_test.go +++ b/pkg/backend/filestate/meta_test.go @@ -29,22 +29,30 @@ func TestEnsurePulumiMeta(t *testing.T) { tests := []struct { desc string give map[string]string // files in the bucket - env map[string]string // environment variables want pulumiMeta }{ { // Empty bucket should be initialized to // the current version by default. desc: "empty", - want: pulumiMeta{Version: 0}, + want: pulumiMeta{Version: 1}, }, { - desc: "version 0", + // Non-empty bucket without a version file + // should get version 0 for legacy mode. + desc: "legacy", give: map[string]string{ - ".pulumi/Pulumi.yaml": `version: 0`, + ".pulumi/stacks/a.json": `{}`, }, want: pulumiMeta{Version: 0}, }, + { + desc: "version 1", + give: map[string]string{ + ".pulumi/Pulumi.yaml": `version: 1`, + }, + want: pulumiMeta{Version: 1}, + }, } for _, tt := range tests { diff --git a/pkg/backend/filestate/state.go b/pkg/backend/filestate/state.go index 985a2e31edc3..1813038b8b8a 100644 --- a/pkg/backend/filestate/state.go +++ b/pkg/backend/filestate/state.go @@ -16,6 +16,7 @@ package filestate import ( "context" + "errors" "fmt" "io" "os" @@ -127,7 +128,7 @@ func (b *localBackend) getTarget( } return &deploy.Target{ Name: stack.Name(), - Organization: "", // filestate has no organizations + Organization: "organization", // filestate has no organizations really, but we just always say it's "organization" Config: cfg, Decrypter: dec, Snapshot: snapshot, @@ -553,3 +554,24 @@ func (b *localBackend) addToHistory(ref *localBackendReference, update backend.U checkpointFile := fmt.Sprintf("%s.checkpoint.%s", pathPrefix, ext) return b.bucket.Copy(context.TODO(), checkpointFile, b.stackPath(ref), nil) } + +// isStateEmpty reports whether the state inside the given bucket is empty. +func isStateEmpty(ctx context.Context, b Bucket) (bool, error) { + iter := b.List(&blob.ListOptions{ + Delimiter: "/", + Prefix: workspace.BookkeepingDir, + }) + + if _, err := iter.Next(ctx); err != nil { + if errors.Is(err, io.EOF) { + return true, nil + } + // io.EOF is expected if the bucket is empty + // but all other errors are not. + return false, fmt.Errorf("list bucket: %w", err) + } + + // If we get here, iter.Next succeeded, + // so the bucket is not empty. + return false, nil +} diff --git a/pkg/backend/filestate/state_test.go b/pkg/backend/filestate/state_test.go new file mode 100644 index 000000000000..daafc5956516 --- /dev/null +++ b/pkg/backend/filestate/state_test.go @@ -0,0 +1,61 @@ +package filestate + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gocloud.dev/blob/memblob" +) + +func TestIsStateEmpty(t *testing.T) { + t.Parallel() + + tests := []struct { + desc string + // List of files that exist in the bucket. + files []string + + // Whether the state is considered empty. + empty bool + }{ + { + desc: "empty", + empty: true, + }, + { + desc: "non-state files", + files: []string{ + "foo", + "bar", + }, + empty: true, + }, + { + desc: "state files", + files: []string{ + ".pulumi/stacks/foo.json", + ".pulumi/stacks/bar.json", + }, + empty: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.desc, func(t *testing.T) { + t.Parallel() + + b := memblob.OpenBucket(nil) + ctx := context.Background() + for _, f := range tt.files { + require.NoError(t, b.WriteAll(ctx, f, []byte{}, nil)) + } + + got, err := isStateEmpty(ctx, b) + require.NoError(t, err) + assert.Equal(t, tt.empty, got) + }) + } +} diff --git a/pkg/backend/filestate/store.go b/pkg/backend/filestate/store.go index cf736d09e8fe..faf96bf8b135 100644 --- a/pkg/backend/filestate/store.go +++ b/pkg/backend/filestate/store.go @@ -15,12 +15,14 @@ package filestate import ( + "errors" "fmt" "path/filepath" "strings" "github.com/pulumi/pulumi/sdk/v3/go/common/encoding" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil" "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) @@ -73,6 +75,207 @@ type referenceStore interface { // ParseReference parses a localBackendReference from a string. ParseReference(ref string) (*localBackendReference, error) + + // ValidateReference verifies that the provided reference is valid + // returning an error if it is not. + ValidateReference(*localBackendReference) error +} + +// projectReferenceStore is a referenceStore that stores stack +// information with the new project-based layout. +// +// This is version 1 of the stack storage format. +type projectReferenceStore struct { + bucket Bucket + + // currentProject is a thread-safe way to get the current project. + currentProject func() *workspace.Project +} + +var _ referenceStore = (*projectReferenceStore)(nil) + +func newProjectReferenceStore(bucket Bucket, currentProject func() *workspace.Project) *projectReferenceStore { + return &projectReferenceStore{ + bucket: bucket, + currentProject: currentProject, + } +} + +// newReference builds a new localBackendReference with the provided arguments. +// This DOES NOT modify the underlying storage. +func (p *projectReferenceStore) newReference(project, name tokens.Name) *localBackendReference { + return &localBackendReference{ + name: name, + project: project, + store: p, + currentProject: p.currentProject, + } +} + +func (p *projectReferenceStore) StackBasePath(ref *localBackendReference) string { + contract.Requiref(ref.project != "", "ref.project", "must not be empty") + return filepath.Join(StacksDir, fsutil.NamePath(ref.project), fsutil.NamePath(ref.name)) +} + +func (p *projectReferenceStore) HistoryDir(stack *localBackendReference) string { + contract.Requiref(stack.project != "", "ref.project", "must not be empty") + return filepath.Join(HistoriesDir, fsutil.NamePath(stack.project), fsutil.NamePath(stack.name)) +} + +func (p *projectReferenceStore) BackupDir(stack *localBackendReference) string { + contract.Requiref(stack.project != "", "ref.project", "must not be empty") + return filepath.Join(BackupsDir, fsutil.NamePath(stack.project), fsutil.NamePath(stack.name)) +} + +func (p *projectReferenceStore) ParseReference(stackRef string) (*localBackendReference, error) { + // We accept the following forms: + // + // 1. + // 2. / + // 3. // + // + // org-name must always be "organization". + // This matches the behavior of the Pulumi Service storage backend. + if stackRef == "" { + return nil, errors.New("stack name must not be empty") + } + + var name, project, org string + split := strings.Split(stackRef, "/") // guaranteed to have at least one element + switch len(split) { + case 1: + name = split[0] + case 2: + org = split[0] + name = split[1] + case 3: + org = split[0] + project = split[1] + name = split[2] + } + + // If the provided stack name didn't include the org or project, + // infer them from the local environment. + if org == "" { + // Filestate organization MUST always be "organization" + org = "organization" + } + + if org != "organization" { + return nil, errors.New("organization name must be 'organization'") + } + + if project == "" { + currentProject := p.currentProject() + if currentProject == nil { + return nil, fmt.Errorf("if you're using the --stack flag, " + + "pass the fully qualified name (organization/project/stack)") + } + + project = currentProject.Name.String() + } + + if len(project) > 100 { + return nil, errors.New("project names are limited to 100 characters") + } + + if project != "" && !tokens.IsName(project) { + return nil, fmt.Errorf( + "project names may only contain alphanumerics, hyphens, underscores, and periods: %s", + project) + } + + if !tokens.IsName(name) || len(name) > 100 { + return nil, fmt.Errorf( + "stack names are limited to 100 characters and may only contain alphanumeric, hyphens, underscores, or periods: %s", + name) + } + + return p.newReference(tokens.Name(project), tokens.Name(name)), nil +} + +func (p *projectReferenceStore) ValidateReference(ref *localBackendReference) error { + if ref.project == "" { + return fmt.Errorf("bad stack reference, project was not set") + } + return nil +} + +func (p *projectReferenceStore) ListProjects() ([]tokens.Name, error) { + path := StacksDir + + files, err := listBucket(p.bucket, path) + if err != nil { + return nil, fmt.Errorf("error listing stacks: %w", err) + } + + projects := make([]tokens.Name, 0, len(files)) + for _, file := range files { + if !file.IsDir { + continue // ignore files + } + + projName := objectName(file) + if !tokens.IsName(projName) { + // If this isn't a valid Name + // it won't be a project directory, + // so skip it. + continue + } + + projects = append(projects, tokens.Name(projName)) + } + + return projects, nil +} + +func (p *projectReferenceStore) ListReferences() ([]*localBackendReference, error) { + // The first level of the bucket is the project name. + // The second level of the bucket is the stack name. + path := StacksDir + + projects, err := p.ListProjects() + if err != nil { + return nil, err + } + + var stacks []*localBackendReference + for _, projName := range projects { + // TODO: Could we improve the efficiency here by firstly making listBucket return an enumerator not + // eagerly collecting all keys into a slice, and secondly by getting listBucket to return all + // descendent items not just the immediate children. We could then do the necessary splitting by + // file paths here to work out project names. + projectFiles, err := listBucket(p.bucket, filepath.Join(path, projName.String())) + if err != nil { + return nil, fmt.Errorf("error listing stacks: %w", err) + } + + for _, projectFile := range projectFiles { + // Can ignore directories at this level + if projectFile.IsDir { + continue + } + + objName := objectName(projectFile) + // Skip files without valid extensions (e.g., *.bak files). + ext := filepath.Ext(objName) + // But accept gzip compression + if ext == encoding.GZIPExt { + objName = strings.TrimSuffix(objName, encoding.GZIPExt) + ext = filepath.Ext(objName) + } + + if _, has := encoding.Marshalers[ext]; !has { + continue + } + + // Read in this stack's information. + name := objName[:len(objName)-len(ext)] + stacks = append(stacks, p.newReference(projName, tokens.Name(name))) + } + } + + return stacks, nil } // legacyReferenceStore is a referenceStore that stores stack @@ -103,14 +306,17 @@ func (p *legacyReferenceStore) newReference(name tokens.Name) *localBackendRefer } func (p *legacyReferenceStore) StackBasePath(ref *localBackendReference) string { + contract.Requiref(ref.project == "", "ref.project", "must be empty") return filepath.Join(StacksDir, fsutil.NamePath(ref.name)) } func (p *legacyReferenceStore) HistoryDir(stack *localBackendReference) string { + contract.Requiref(stack.project == "", "ref.project", "must be empty") return filepath.Join(HistoriesDir, fsutil.NamePath(stack.name)) } func (p *legacyReferenceStore) BackupDir(stack *localBackendReference) string { + contract.Requiref(stack.project == "", "ref.project", "must be empty") return filepath.Join(BackupsDir, fsutil.NamePath(stack.name)) } @@ -123,6 +329,13 @@ func (p *legacyReferenceStore) ParseReference(stackRef string) (*localBackendRef return p.newReference(tokens.Name(stackRef)), nil } +func (p *legacyReferenceStore) ValidateReference(ref *localBackendReference) error { + if ref.project != "" { + return fmt.Errorf("bad stack reference, project was set") + } + return nil +} + func (p *legacyReferenceStore) ListReferences() ([]*localBackendReference, error) { files, err := listBucket(p.bucket, StacksDir) if err != nil { diff --git a/pkg/backend/filestate/store_test.go b/pkg/backend/filestate/store_test.go index 33c1ff94639a..afc1e032b9a3 100644 --- a/pkg/backend/filestate/store_test.go +++ b/pkg/backend/filestate/store_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gocloud.dev/blob/memblob" @@ -41,6 +42,82 @@ func TestLegacyReferenceStore_referencePaths(t *testing.T) { assert.Equal(t, ".pulumi/backups/foo", ref.BackupDir()) } +func TestProjectReferenceStore_referencePaths(t *testing.T) { + t.Parallel() + + bucket := memblob.OpenBucket(nil) + store := newProjectReferenceStore(bucket, func() *workspace.Project { + return &workspace.Project{Name: "test"} + }) + + ref, err := store.ParseReference("organization/myproject/mystack") + require.NoError(t, err) + + assert.Equal(t, ".pulumi/stacks/myproject/mystack", ref.StackBasePath()) + assert.Equal(t, ".pulumi/history/myproject/mystack", ref.HistoryDir()) + assert.Equal(t, ".pulumi/backups/myproject/mystack", ref.BackupDir()) +} + +func TestProjectReferenceStore_ParseReference(t *testing.T) { + t.Parallel() + + bucket := memblob.OpenBucket(nil) + store := newProjectReferenceStore(bucket, func() *workspace.Project { + return &workspace.Project{Name: "currentProject"} + }) + + tests := []struct { + desc string + give string + + fqname tokens.QName + name tokens.Name + project tokens.Name + str string + }{ + { + desc: "simple", + give: "foo", + fqname: "organization/currentProject/foo", + name: "foo", + project: "currentProject", + str: "foo", + // truncated because project name is the same as current project + }, + { + desc: "organization", + give: "organization/foo", + fqname: "organization/currentProject/foo", + name: "foo", + project: "currentProject", + str: "foo", + }, + { + desc: "fully qualified", + give: "organization/project/foo", + fqname: "organization/project/foo", + name: "foo", + project: "project", + str: "organization/project/foo", // doesn't match current project + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.desc, func(t *testing.T) { + t.Parallel() + + ref, err := store.ParseReference(tt.give) + require.NoError(t, err) + + assert.Equal(t, tt.fqname, ref.FullyQualifiedName()) + assert.Equal(t, tt.name, ref.Name()) + assert.Equal(t, tt.project, ref.Project()) + assert.Equal(t, tt.str, ref.String()) + }) + } +} + func TestLegacyReferenceStore_ParseReference_errors(t *testing.T) { t.Parallel() @@ -73,6 +150,69 @@ func TestLegacyReferenceStore_ParseReference_errors(t *testing.T) { } } +func TestProjectReferenceStore_ParseReference_errors(t *testing.T) { + t.Parallel() + + bucket := memblob.OpenBucket(nil) + store := newProjectReferenceStore(bucket, func() *workspace.Project { + return nil // current project is not set + }) + + tests := []struct { + desc string + give string + wantErr string + }{ + { + desc: "empty", + wantErr: "must not be empty", + }, + { + desc: "bad organization", + give: "foo/bar/baz", + wantErr: "organization name must be 'organization'", + }, + { + desc: "long project name", + give: "organization/" + strings.Repeat("a", 101) + "/foo", + wantErr: "project names are limited to 100 characters", + }, + { + desc: "long project stack name", + give: "organization/foo/" + strings.Repeat("a", 101), + wantErr: "stack names are limited to 100 characters", + }, + { + desc: "no current project", + give: "organization/foo", + wantErr: "pass the fully qualified name", + }, + { + desc: "invalid project name", + give: "organization/foo:bar/baz", + wantErr: "may only contain alphanumeric", + }, + { + desc: "invalid stack name", + give: "organization/foo/baz:qux", + wantErr: "may only contain alphanumeric", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.desc, func(t *testing.T) { + t.Parallel() + + require.NotEmpty(t, tt.wantErr, + "bad test case: wantErr must be non-empty") + + _, err := store.ParseReference(tt.give) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + func TestLegacyReferenceStore_ListReferences(t *testing.T) { t.Parallel() @@ -149,3 +289,109 @@ func TestLegacyReferenceStore_ListReferences(t *testing.T) { }) } } + +func TestProjectReferenceStore_List(t *testing.T) { + t.Parallel() + + tests := []struct { + desc string + + // List of file paths relative to the storage root + // that should exist before ListReferences is called. + files []string + + // List of fully-qualified stack names that should be returned + // by ListReferences. + stacks []tokens.QName + + // List of project names that should be returned by ListProjects. + projects []tokens.Name + }{ + { + desc: "empty", + stacks: []tokens.QName{}, + projects: []tokens.Name{}, + }, + { + desc: "json", + files: []string{ + ".pulumi/stacks/proj/foo.json", + }, + stacks: []tokens.QName{"organization/proj/foo"}, + projects: []tokens.Name{"proj"}, + }, + { + desc: "gzipped", + files: []string{ + ".pulumi/stacks/foo/bar.json.gz", + }, + stacks: []tokens.QName{"organization/foo/bar"}, + projects: []tokens.Name{"foo"}, + }, + { + desc: "multiple", + files: []string{ + ".pulumi/stacks/a/foo.json", + ".pulumi/stacks/b/bar.json.gz", + ".pulumi/stacks/c/baz.json", + }, + stacks: []tokens.QName{ + "organization/a/foo", + "organization/b/bar", + "organization/c/baz", + }, + projects: []tokens.Name{"a", "b", "c"}, + }, + { + desc: "extraneous files and directories", + files: []string{ + ".pulumi/stacks/a/foo.json", + ".pulumi/stacks/foo.json", + ".pulumi/stacks/bar/baz/qux.json", // nested too deep + ".pulumi/stacks/a b/c.json", // bad project name + }, + stacks: []tokens.QName{"organization/a/foo"}, + projects: []tokens.Name{"a", "bar"}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.desc, func(t *testing.T) { + t.Parallel() + + bucket := memblob.OpenBucket(nil) + store := newProjectReferenceStore(bucket, func() *workspace.Project { + return &workspace.Project{Name: "test"} + }) + + ctx := context.Background() + for _, f := range tt.files { + require.NoError(t, bucket.WriteAll(ctx, f, []byte{}, nil)) + } + + t.Run("Projects", func(t *testing.T) { + t.Parallel() + + projects, err := store.ListProjects() + require.NoError(t, err) + + assert.Equal(t, tt.projects, projects) + }) + + t.Run("References", func(t *testing.T) { + t.Parallel() + + refs, err := store.ListReferences() + require.NoError(t, err) + + got := make([]tokens.QName, len(refs)) + for i, ref := range refs { + got[i] = ref.FullyQualifiedName() + } + + assert.Equal(t, tt.stacks, got) + }) + }) + } +} diff --git a/pkg/cmd/pulumi/new_acceptance_test.go b/pkg/cmd/pulumi/new_acceptance_test.go index 9897f6f10cfe..64fcdab6640d 100644 --- a/pkg/cmd/pulumi/new_acceptance_test.go +++ b/pkg/cmd/pulumi/new_acceptance_test.go @@ -149,7 +149,8 @@ func TestCreatingProjectWithPulumiBackendURL(t *testing.T) { proj := loadProject(t, tempdir) assert.Equal(t, defaultProjectName, proj.Name.String()) // Expect the stack directory to have a checkpoint file for the stack. - _, err = os.Stat(filepath.Join(fileStateDir, workspace.BookkeepingDir, workspace.StackDir, stackName+".json")) + _, err = os.Stat(filepath.Join( + fileStateDir, workspace.BookkeepingDir, workspace.StackDir, defaultProjectName, stackName+".json")) assert.NoError(t, err) b, err = currentBackend(ctx, nil, display.Options{}) diff --git a/tests/config_test.go b/tests/config_test.go index ab9278a79da7..a5f9ba571488 100644 --- a/tests/config_test.go +++ b/tests/config_test.go @@ -71,10 +71,14 @@ func TestConfigCommands(t *testing.T) { // check that the nested config does not exist because we didn't use path _, stderr := e.RunCommandExpectError("pulumi", "config", "get", "outer") - assert.Equal(t, "error: configuration key 'outer' not found for stack 'test'", strings.Trim(stderr, "\r\n")) + assert.Equal(t, + "error: configuration key 'outer' not found for stack 'test'", + strings.Trim(stderr, "\r\n")) _, stderr = e.RunCommandExpectError("pulumi", "config", "get", "myList") - assert.Equal(t, "error: configuration key 'myList' not found for stack 'test'", strings.Trim(stderr, "\r\n")) + assert.Equal(t, + "error: configuration key 'myList' not found for stack 'test'", + strings.Trim(stderr, "\r\n")) // set the nested config using --path e.RunCommand("pulumi", "config", "set-all", "--path", diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index a6b48d9978ae..5074ffc3f871 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -531,7 +531,7 @@ func TestDestroyStackRef(t *testing.T) { e.RunCommand("pulumi", "up", "--skip-preview", "--yes") e.CWD = os.TempDir() - e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "-s", "dev") + e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "-s", "organization/large_resource_js/dev") } //nolint:paralleltest // uses parallel programtest diff --git a/tests/stack_test.go b/tests/stack_test.go index 77dd04dda2b6..f29e0bc43436 100644 --- a/tests/stack_test.go +++ b/tests/stack_test.go @@ -367,7 +367,7 @@ func TestStackBackups(t *testing.T) { const stackName = "imulup" // Get the path to the backup directory for this project. - backupDir, err := getStackProjectBackupDir(e, stackName) + backupDir, err := getStackProjectBackupDir(e, "stack_outputs", stackName) assert.NoError(t, err, "getting stack project backup path") defer func() { if !t.Failed() { @@ -560,8 +560,8 @@ func TestLocalStateLocking(t *testing.T) { // stackFileFormatAsserters returns a function to assert that the current file // format is for gzip and plain formats respectively. -func stackFileFormatAsserters(t *testing.T, e *ptesting.Environment, stackName string) (func(), func()) { - stacksDir := filepath.Join(".pulumi", "stacks") +func stackFileFormatAsserters(t *testing.T, e *ptesting.Environment, projectName, stackName string) (func(), func()) { + stacksDir := filepath.Join(".pulumi", "stacks", projectName) pathStack := filepath.Join(stacksDir, stackName+".json") pathStackGzip := pathStack + ".gz" pathStackBak := pathStack + ".bak" @@ -622,7 +622,7 @@ func TestLocalStateGzip(t *testing.T) { //nolint:paralleltest e.RunCommand("yarn", "install") e.RunCommand("pulumi", "up", "--non-interactive", "--yes", "--skip-preview") - assertGzipFileFormat, assertPlainFileFormat := stackFileFormatAsserters(t, e, stackName) + assertGzipFileFormat, assertPlainFileFormat := stackFileFormatAsserters(t, e, "stack_dependencies", stackName) switchGzipOff := func() { e.Setenv(filestate.PulumiFilestateGzipEnvVar, "0") } switchGzipOn := func() { e.Setenv(filestate.PulumiFilestateGzipEnvVar, "1") } pulumiUp := func() { e.RunCommand("pulumi", "up", "--non-interactive", "--yes", "--skip-preview") } @@ -691,10 +691,11 @@ func assertBackupStackFile(t *testing.T, stackName string, file os.DirEntry, bef assert.True(t, parsedTime < after, "False: %v < %v", parsedTime, after) } -func getStackProjectBackupDir(e *ptesting.Environment, stackName string) (string, error) { +func getStackProjectBackupDir(e *ptesting.Environment, projectName, stackName string) (string, error) { return filepath.Join(e.RootPath, workspace.BookkeepingDir, workspace.BackupDir, + projectName, stackName, ), nil }