Skip to content

Commit

Permalink
Fix import resources with provider default tags (#4169)
Browse files Browse the repository at this point in the history 
We have special logic around applying default provider tags to
resources. This logic only applied to the `Check` call which means it
was not applied when you were importing resources. This PR extends that
logic to also run during the `Read` call by utilizing
`TransformOutputs`.

While it is true that `TransformOutputs` also runs during `Create` &
`Update`
this is a side effect that I think is ok. From my understanding `tags`
and `tagsAll`
should always be equal. If we have an additional place where we make
sure they are equal
it shouldn't harm anything.

I've added tests (see `testTagsPulumiLifecycle`) which test the complete
lifecycle of a pulumi program

1. `Up` with both provider `defaultTags`/`ignoreTags` and resource level
`tags`
    1a. Run validations on result
2. `Refresh` with no changes
3. `Import` using the resource option. Ensures resource can be
successfully imported
3a. Allows for a hook to be run prior to import being run. e.g. Add tags
remotely
4. `Import` using the CLI. Ensures resources can be successfully
imported
4a. Allows for a hook to be run prior to import being run. e.g. Add tags
remotely
5. `Refresh` with no changes

fix #4030, fix #4080, fix #3311
  • Loading branch information
@corymhall
corymhall committed Jul 23, 2024
1 parent 794e177 commit 21194dc
Show file tree
Hide file tree
Showing 12 changed files with 899 additions and 48 deletions.
26 changes: 25 additions & 1 deletion examples/examples_go_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ func (st tagsState) validateStateResult(phase int) func(
return func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
for k, v := range stack.Outputs {
switch k {
case "bucket-name", "legacy-bucket-name", "appconfig-app-arn":
case "bucket-name", "legacy-bucket-name", "appconfig-app-arn", "appconfig-env-arn", "get-appconfig-env":
continue
}

Expand All @@ -251,12 +251,18 @@ func (st tagsState) validateStateResult(phase int) func(
t.Logf("key=%s tags are as expected: %v", k, actualTagsJSON)

if k == "bucket" {
// TODO: uncomment when https://github.com/pulumi/pulumi-aws/issues/4258 is fixed
// getTags := stack.Outputs["get-bucket"].(string)
// assert.Equal(t, v.(string), getTags)
bucketName := stack.Outputs["bucket-name"].(string)
st.assertTagsEqualWithRetry(t,
fetchBucketTags(bucketName),
"bad bucket tags")
}
if k == "legacy-bucket" {
// TODO: uncomment when https://github.com/pulumi/pulumi-aws/issues/4258 is fixed
// getTags := stack.Outputs["get-legacy-bucket"].(string)
// assert.Equal(t, v.(string), getTags)
bucketName := stack.Outputs["legacy-bucket-name"].(string)
st.assertTagsEqualWithRetry(t,
fetchBucketTags(bucketName),
Expand All @@ -268,10 +274,28 @@ func (st tagsState) validateStateResult(phase int) func(
fetchAppConfigTags(arn),
"bad appconfig app tags")
}
if k == "appconfig-env" {
getTags := stack.Outputs["get-appconfig-env"].(string)
isEqual(t, v.(string), getTags)
arn := stack.Outputs["appconfig-env-arn"].(string)
st.assertTagsEqualWithRetry(t,
fetchAppConfigTags(arn),
"bad appconfig app tags")
}
}
}
}

func isEqual(t *testing.T, a, b string) {
if a == "null" {
a = "{}"
}
if b == "null" {
b = "{}"
}
assert.Equal(t, a, b)
}

func fetchBucketTags(awsBucket string) tagsFetcher {
return func() (map[string]string, error) {
sess := session.Must(session.NewSessionWithOptions(session.Options{
Expand Down
29 changes: 29 additions & 0 deletions examples/tags-combinations-go/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,41 @@ func main() {
return err
}

env, err := appconfig.NewEnvironment(ctx, "testappconfigenv"+testIdent, &appconfig.EnvironmentArgs{
ApplicationId: app.ID(),
Tags: tagsMap,
}, pulumi.Provider(p))
if err != nil {
return err
}

getEnv, err := appconfig.GetEnvironment(ctx, "get-testappconfigenv"+testIdent, env.ID(), &appconfig.EnvironmentState{}, pulumi.Provider(p))
if err != nil {
return err
}

// TODO: uncomment when https://github.com/pulumi/pulumi-aws/issues/4258 is fixed
// refresh doesn't work for `forceDelete` & `acl`
// getBucket, err := s3.GetBucketV2(ctx, "get-bucketv2"+testIdent, bucket.ID(), &s3.BucketV2State{}, pulumi.Provider(p), pulumi.IgnoreChanges([]string{"forceDestroy", "acl"}))
// if err != nil {
// return err
// }
// getLegacyBucket, err := s3.GetBucket(ctx, "get-legacybucket"+testIdent, legacyBucket.ID(), &s3.BucketState{}, pulumi.Provider(p), pulumi.IgnoreChanges([]string{"forceDestroy", "acl"}))
// if err != nil {
// return err
// }

ctx.Export("bucket", exportTags(bucket.Tags))
// ctx.Export("get-bucket", exportTags(getBucket.Tags))
ctx.Export("legacy-bucket", exportTags(legacyBucket.Tags))
// ctx.Export("get-legacy-bucket", exportTags(getLegacyBucket.Tags))
ctx.Export("bucket-name", bucket.Bucket)
ctx.Export("legacy-bucket-name", legacyBucket.Bucket)
ctx.Export("appconfig-app", exportTags(app.Tags))
ctx.Export("appconfig-app-arn", app.Arn)
ctx.Export("appconfig-env", exportTags(env.Tags))
ctx.Export("get-appconfig-env", exportTags(getEnv.Tags))
ctx.Export("appconfig-env-arn", env.Arn)

return nil
})
Expand Down
29 changes: 29 additions & 0 deletions examples/tags-combinations-go/step1/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,41 @@ func main() {
return err
}

env, err := appconfig.NewEnvironment(ctx, "testappconfigenv"+testIdent, &appconfig.EnvironmentArgs{
ApplicationId: app.ID(),
Tags: tagsMap,
}, pulumi.Provider(p))
if err != nil {
return err
}

getEnv, err := appconfig.GetEnvironment(ctx, "get-testappconfigenv"+testIdent, env.ID(), &appconfig.EnvironmentState{}, pulumi.Provider(p))
if err != nil {
return err
}

// TODO: uncomment when https://github.com/pulumi/pulumi-aws/issues/4258 is fixed
// refresh doesn't work for `forceDelete` & `acl`
// getBucket, err := s3.GetBucketV2(ctx, "get-bucketv2"+testIdent, bucket.ID(), &s3.BucketV2State{}, pulumi.Provider(p))
// if err != nil {
// return err
// }
// getLegacyBucket, err := s3.GetBucket(ctx, "get-legacybucket"+testIdent, legacyBucket.ID(), &s3.BucketState{}, pulumi.Provider(p))
// if err != nil {
// return err
// }

ctx.Export("bucket", exportTags(bucket.Tags))
// ctx.Export("get-bucket", exportTags(getBucket.Tags))
ctx.Export("legacy-bucket", exportTags(legacyBucket.Tags))
// ctx.Export("get-legacy-bucket", exportTags(getLegacyBucket.Tags))
ctx.Export("bucket-name", bucket.Bucket)
ctx.Export("legacy-bucket-name", legacyBucket.Bucket)
ctx.Export("appconfig-app", exportTags(app.Tags))
ctx.Export("appconfig-app-arn", app.Arn)
ctx.Export("appconfig-env", exportTags(env.Tags))
ctx.Export("get-appconfig-env", exportTags(getEnv.Tags))
ctx.Export("appconfig-env-arn", env.Arn)

return nil
})
Expand Down
6 changes: 3 additions & 3 deletions provider/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ module github.com/pulumi/pulumi-aws/provider/v6
go 1.22.5

require (
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/config v1.27.26
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11
github.com/aws/aws-sdk-go-v2/service/appconfig v1.31.3
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.33.3
github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.23.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.54
github.com/hashicorp/terraform-plugin-sdk/v2 v2.34.0
Expand Down Expand Up @@ -65,7 +68,6 @@ require (
github.com/armon/go-radix v1.0.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go v1.54.18 // indirect
github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.26 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.7 // indirect
Expand All @@ -81,7 +83,6 @@ require (
github.com/aws/aws-sdk-go-v2/service/amplify v1.23.3 // indirect
github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.3 // indirect
github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.3 // indirect
github.com/aws/aws-sdk-go-v2/service/appconfig v1.31.3 // indirect
github.com/aws/aws-sdk-go-v2/service/appfabric v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/appflow v1.43.3 // indirect
github.com/aws/aws-sdk-go-v2/service/appintegrations v1.27.3 // indirect
Expand Down Expand Up @@ -229,7 +230,6 @@ require (
github.com/aws/aws-sdk-go-v2/service/rekognition v1.43.2 // indirect
github.com/aws/aws-sdk-go-v2/service/resourceexplorer2 v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/resourcegroups v1.24.3 // indirect
github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.23.3 // indirect
github.com/aws/aws-sdk-go-v2/service/rolesanywhere v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 // indirect
github.com/aws/aws-sdk-go-v2/service/route53domains v1.25.3 // indirect
Expand Down
25 changes: 2 additions & 23 deletions provider/provider_python_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,12 @@
package provider

import (
"bytes"
"context"
"fmt"
"path/filepath"
"strings"
"testing"
"time"

"github.com/pulumi/pulumi/pkg/v3/testing/integration"
"github.com/stretchr/testify/require"
)

func TestRegress3196(t *testing.T) {
Expand Down Expand Up @@ -61,7 +57,6 @@ func TestRegress3887(t *testing.T) {

// Make sure that importing an AWS targetGroup succeeds.
func TestRegress2534(t *testing.T) {
ctx := context.Background()
ptest := pulumiTest(t, filepath.Join("test-programs", "regress-2534"))
upResult := ptest.Up()
targetGroupArn := upResult.Outputs["targetGroupArn"].Value.(string)
Expand All @@ -71,24 +66,8 @@ func TestRegress2534(t *testing.T) {
workdir := workspace.WorkDir()
t.Logf("workdir = %s", workdir)

exec := func(args ...string) {
var env []string
for k, v := range workspace.GetEnvVars() {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
stdin := bytes.NewReader([]byte{})
var arguments []string
arguments = append(arguments, args...)
arguments = append(arguments, "-s", ptest.CurrentStack().Name())
s1, s2, code, err := workspace.PulumiCommand().Run(ctx, workdir, stdin, nil, nil, env, arguments...)
t.Logf("import stdout: %s", s1)
t.Logf("import stderr: %s", s2)
t.Logf("code=%v", code)
require.NoError(t, err)
}

exec("import", "aws:lb/targetGroup:TargetGroup", "newtg", targetGroupArn, "--yes")
exec("state", "unprotect", strings.ReplaceAll(targetGroupUrn, "::test", "::newtg"), "--yes")
execPulumi(t, ptest, workdir, "import", "aws:lb/targetGroup:TargetGroup", "newtg", targetGroupArn, "--yes")
execPulumi(t, ptest, workdir, "state", "unprotect", strings.ReplaceAll(targetGroupUrn, "::test", "::newtg"), "--yes")
}

func getPythonBaseOptions(t *testing.T) integration.ProgramTestOptions {
Expand Down
20 changes: 20 additions & 0 deletions provider/provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
package provider

import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
Expand Down Expand Up @@ -34,6 +36,24 @@ func getEnvRegion(t *testing.T) string {
return envRegion
}

func execPulumi(t *testing.T, ptest *pulumitest.PulumiTest, workdir string, args ...string) {
ctx := context.Background()
var env []string
workspace := ptest.CurrentStack().Workspace()
for k, v := range workspace.GetEnvVars() {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
stdin := bytes.NewReader([]byte{})
var arguments []string
arguments = append(arguments, args...)
arguments = append(arguments, "-s", ptest.CurrentStack().Name())
s1, s2, code, err := workspace.PulumiCommand().Run(ctx, workdir, stdin, nil, nil, env, arguments...)
t.Logf("stdout: %s", s1)
t.Logf("stderr: %s", s2)
t.Logf("code=%v", code)
require.NoError(t, err)
}

type testProviderUpgradeOptions struct {
baselineVersion string
linkNodeSDK bool
Expand Down

0 comments on commit 21194dc

Please sign in to comment.