diff --git a/.github/workflows/download-pulumi-cron.yml b/.github/workflows/download-pulumi-cron.yml index 39cf5e1edfc9..75f57321b5c1 100644 --- a/.github/workflows/download-pulumi-cron.yml +++ b/.github/workflows/download-pulumi-cron.yml @@ -94,27 +94,6 @@ jobs: run: | echo "Expected version ${{ steps.vars.outputs.expected-version }} but found ${{ steps.vars.outputs.installed-version }}" exit 1 - windows-winget-install: - name: Install Pulumi with WinGet on Windows - runs-on: windows-latest - steps: - - name: Install WinGet CLI - shell: powershell - run: | - try { winget --help } catch { Add-AppxPackage -Path https://aka.ms/getwinget } - - name: Install Pulumi Using Winget - run: winget install pulumi - - name: Pulumi Version Details - id: vars - shell: bash - run: | - echo "::set-output name=installed-version::$(pulumi version)" - echo "::set-output name=expected-version::v$(curl -sS https://www.pulumi.com/latest-version)" - - name: Error if incorrect version found - if: ${{ steps.vars.outputs.expected-version != steps.vars.outputs.installed-version }} - run: | - echo "Expected version ${{ steps.vars.outputs.expected-version }} but found ${{ steps.vars.outputs.installed-version }}" - exit 1 windows-direct-install: name: Install Pulumi via script on Windows runs-on: windows-latest diff --git a/changelog/pending/20221121--backend-service--allows-the-service-to-opt-into-bandwidth-optimized-diff-protocol.yaml b/changelog/pending/20221121--backend-service--allows-the-service-to-opt-into-bandwidth-optimized-diff-protocol.yaml new file mode 100644 index 000000000000..f929c8632edf --- /dev/null +++ b/changelog/pending/20221121--backend-service--allows-the-service-to-opt-into-bandwidth-optimized-diff-protocol.yaml @@ -0,0 +1,4 @@ +changes: +- type: feat + scope: backend/service + description: Allows the service to opt into a bandwidth-optimized DIFF protocol for storing checkpoints. Previously this required setting the PULUMI_OPTIMIZED_CHECKPOINT_PATCH env variable on the client. This env variable is now deprecated. diff --git a/changelog/pending/20221128--programgen-nodejs--add-between-and.yaml b/changelog/pending/20221128--programgen-nodejs--add-between-and.yaml new file mode 100644 index 000000000000..912903f06b7a --- /dev/null +++ b/changelog/pending/20221128--programgen-nodejs--add-between-and.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: programgen/nodejs + description: Add `.` between `?` and `[`. diff --git a/changelog/pending/20221128--programgen-nodejs--fix-capitalization-when-generating-fs-readdirsync.yaml b/changelog/pending/20221128--programgen-nodejs--fix-capitalization-when-generating-fs-readdirsync.yaml new file mode 100644 index 000000000000..94dafdbf717c --- /dev/null +++ b/changelog/pending/20221128--programgen-nodejs--fix-capitalization-when-generating-fs-readdirsync.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: programgen/nodejs + description: Fix capitalization when generating `fs.readdirSync`. diff --git a/changelog/pending/20221129--programgen-go--convert-the-result-of-immediate-invokes-to-ouputs-when-necessary.yaml b/changelog/pending/20221129--programgen-go--convert-the-result-of-immediate-invokes-to-ouputs-when-necessary.yaml new file mode 100644 index 000000000000..9985d3106368 --- /dev/null +++ b/changelog/pending/20221129--programgen-go--convert-the-result-of-immediate-invokes-to-ouputs-when-necessary.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: programgen/go + description: Convert the result of immediate invokes to ouputs when necessary. diff --git a/pkg/backend/httpstate/backend.go b/pkg/backend/httpstate/backend.go index b3a80586431b..44d17efc3d14 100644 --- a/pkg/backend/httpstate/backend.go +++ b/pkg/backend/httpstate/backend.go @@ -18,6 +18,7 @@ import ( "context" cryptorand "crypto/rand" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -29,6 +30,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" opentracing "github.com/opentracing/opentracing-go" @@ -119,6 +121,7 @@ type cloudBackend struct { url string client *client.Client currentProject *workspace.Project + capabilities func(context.Context) capabilities } // Assert we implement the backend.Backend and backend.SpecificDeploymentExporter interfaces. @@ -139,11 +142,15 @@ func New(d diag.Sink, cloudURL string) (Backend, error) { currentProject = nil } + client := client.NewClient(cloudURL, apiToken, d) + capabilities := detectCapabilities(d, client) + return &cloudBackend{ d: d, url: cloudURL, - client: client.NewClient(cloudURL, apiToken, d), + client: client, currentProject: currentProject, + capabilities: capabilities, }, nil } @@ -767,6 +774,10 @@ func (b *cloudBackend) GetStack(ctx context.Context, stackRef backend.StackRefer return nil, err } + // GetStack is typically the initial call to a series of calls to the backend. Although logically unrelated, + // this is a good time to start detecting capabilities so that capability request is not on the critical path. + go b.capabilities(ctx) + stack, err := b.client.GetStack(ctx, stackID) if err != nil { // If this was a 404, return nil, nil as per this method's contract. @@ -1753,3 +1764,57 @@ func (c httpstateBackendClient) GetStackResourceOutputs( ctx context.Context, name string) (resource.PropertyMap, error) { return backend.NewBackendClient(c.backend).GetStackResourceOutputs(ctx, name) } + +// Represents feature-detected capabilities of the service the backend is connected to. +type capabilities struct { + // If non-nil, indicates that delta checkpoint updates are supported. + deltaCheckpointUpdates *apitype.DeltaCheckpointUploadsConfigV1 +} + +// Builds a lazy wrapper around doDetectCapabilities. +func detectCapabilities(d diag.Sink, client *client.Client) func(ctx context.Context) capabilities { + var once sync.Once + var caps capabilities + done := make(chan struct{}) + get := func(ctx context.Context) capabilities { + once.Do(func() { + caps = doDetectCapabilities(ctx, d, client) + close(done) + }) + <-done + return caps + } + return get +} + +func doDetectCapabilities(ctx context.Context, d diag.Sink, client *client.Client) capabilities { + resp, err := client.GetCapabilities(ctx) + if err != nil { + d.Warningf(diag.Message("" /*urn*/, "failed to get capabilities: %v"), err) + return capabilities{} + } + caps, err := decodeCapabilities(resp.Capabilities) + if err != nil { + d.Warningf(diag.Message("" /*urn*/, "failed to decode capabilities: %v"), err) + return capabilities{} + } + return caps +} + +func decodeCapabilities(wireLevel []apitype.APICapabilityConfig) (capabilities, error) { + var parsed capabilities + for _, entry := range wireLevel { + switch entry.Capability { + case apitype.DeltaCheckpointUploads: + var cap apitype.DeltaCheckpointUploadsConfigV1 + if err := json.Unmarshal(entry.Configuration, &cap); err != nil { + msg := "decoding DeltaCheckpointUploadsConfigV1 returned %w" + return capabilities{}, fmt.Errorf(msg, err) + } + parsed.deltaCheckpointUpdates = &cap + default: + continue + } + } + return parsed, nil +} diff --git a/pkg/backend/httpstate/client/api_endpoints.go b/pkg/backend/httpstate/client/api_endpoints.go index 718ca9723ec1..ebfc88eb0dc2 100644 --- a/pkg/backend/httpstate/client/api_endpoints.go +++ b/pkg/backend/httpstate/client/api_endpoints.go @@ -78,6 +78,8 @@ func init() { routes.Path(path).Methods(method).Name(name) } + addEndpoint("GET", "/api/capabilities", "getCapabilities") + addEndpoint("GET", "/api/user", "getCurrentUser") addEndpoint("GET", "/api/user/stacks", "listUserStacks") addEndpoint("GET", "/api/stacks/{orgName}", "listOrganizationStacks") diff --git a/pkg/backend/httpstate/client/client.go b/pkg/backend/httpstate/client/client.go index eaeb483a1450..3b41d9192cc3 100644 --- a/pkg/backend/httpstate/client/client.go +++ b/pkg/backend/httpstate/client/client.go @@ -49,6 +49,9 @@ type Client struct { apiOrgs []string diag diag.Sink client restClient + + // If true, do not probe the backend with GET /api/capabilities and assume no capabilities. + DisableCapabilityProbing bool } // newClient creates a new Pulumi API client with the given URL and API token. It is a variable instead of a regular @@ -321,7 +324,7 @@ func (pc *Client) GetLatestConfiguration(ctx context.Context, stackID StackIdent func (pc *Client) DoesProjectExist(ctx context.Context, owner string, projectName string) (bool, error) { if err := pc.restCall(ctx, "HEAD", getProjectPath(owner, projectName), nil, nil, nil); err != nil { // If this was a 404, return false - project not found. - if errResp, ok := err.(*apitype.ErrorResponse); ok && errResp.Code == http.StatusNotFound { + if is404(err) { return false, nil } @@ -1070,3 +1073,32 @@ func (pc *Client) GetDeploymentUpdates(ctx context.Context, stack StackIdentifie } return resp, nil } + +func (pc *Client) GetCapabilities(ctx context.Context) (*apitype.CapabilitiesResponse, error) { + if pc.DisableCapabilityProbing { + return &apitype.CapabilitiesResponse{}, nil + } + + var resp apitype.CapabilitiesResponse + err := pc.restCall(ctx, http.MethodGet, "/api/capabilities", nil, nil, &resp) + if is404(err) { + // The client continues to support legacy backends. They do not support /api/capabilities and are + // assumed here to have no additional capabilities. + return &apitype.CapabilitiesResponse{}, nil + } + if err != nil { + return nil, fmt.Errorf("querying capabilities failed: %w", err) + } + return &resp, nil +} + +func is404(err error) bool { + if err == nil { + return false + } + var errResp *apitype.ErrorResponse + if errors.As(err, &errResp) && errResp.Code == http.StatusNotFound { + return true + } + return false +} diff --git a/pkg/backend/httpstate/client/client_test.go b/pkg/backend/httpstate/client/client_test.go index 6e6d95018032..30eaef55b35d 100644 --- a/pkg/backend/httpstate/client/client_test.go +++ b/pkg/backend/httpstate/client/client_test.go @@ -179,3 +179,46 @@ func TestPatchUpdateCheckpointVerbatimPreservesIndent(t *testing.T) { assert.Equal(t, string(indented), string(request.UntypedDeployment)) } + +func TestGetCapabilities(t *testing.T) { + t.Parallel() + t.Run("legacy-service-404", func(t *testing.T) { + t.Parallel() + s := newMockServer(404, "NOT FOUND") + defer s.Close() + + c := newMockClient(s) + resp, err := c.GetCapabilities(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Empty(t, resp.Capabilities) + }) + t.Run("updated-service-with-delta-checkpoint-capability", func(t *testing.T) { + t.Parallel() + cfg := apitype.DeltaCheckpointUploadsConfigV1{ + CheckpointCutoffSizeBytes: 1024 * 1024 * 4, + } + cfgJSON, err := json.Marshal(cfg) + require.NoError(t, err) + actualResp := apitype.CapabilitiesResponse{ + Capabilities: []apitype.APICapabilityConfig{{ + Version: 3, + Capability: apitype.DeltaCheckpointUploads, + Configuration: json.RawMessage(cfgJSON), + }}, + } + respJSON, err := json.Marshal(actualResp) + require.NoError(t, err) + s := newMockServer(200, string(respJSON)) + defer s.Close() + + c := newMockClient(s) + resp, err := c.GetCapabilities(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Capabilities, 1) + assert.Equal(t, apitype.DeltaCheckpointUploads, resp.Capabilities[0].Capability) + assert.Equal(t, `{"checkpointCutoffSizeBytes":4194304}`, + string(resp.Capabilities[0].Configuration)) + }) +} diff --git a/pkg/backend/httpstate/diffs.go b/pkg/backend/httpstate/diffs.go index af13601b0095..19a40ca6af8f 100644 --- a/pkg/backend/httpstate/diffs.go +++ b/pkg/backend/httpstate/diffs.go @@ -33,8 +33,6 @@ import ( type deploymentDiffState struct { lastSavedDeployment json.RawMessage sequenceNumber int - noChecksums bool - strictMode bool minimalDiffSize int } @@ -44,8 +42,11 @@ type deploymentDiff struct { deploymentDelta json.RawMessage } -func newDeploymentDiffState() *deploymentDiffState { - return &deploymentDiffState{sequenceNumber: 1} +func newDeploymentDiffState(minimalDiffSize int) *deploymentDiffState { + return &deploymentDiffState{ + sequenceNumber: 1, + minimalDiffSize: minimalDiffSize, + } } func (dds *deploymentDiffState) SequenceNumber() int { @@ -62,14 +63,10 @@ func (dds *deploymentDiffState) ShouldDiff(new *apitype.UntypedDeployment) bool if !dds.CanDiff() { return false } - small := dds.minimalDiffSize - if small == 0 { - small = 1024 * 32 - } - if len(dds.lastSavedDeployment) < small { + if len(dds.lastSavedDeployment) < dds.minimalDiffSize { return false } - if len(new.Deployment) < small { + if len(new.Deployment) < dds.minimalDiffSize { return false } return true @@ -99,13 +96,11 @@ func (dds *deploymentDiffState) Diff(ctx context.Context, var checkpointHash string checkpointHashReady := &sync.WaitGroup{} - if !dds.noChecksums { - checkpointHashReady.Add(1) - go func() { - defer checkpointHashReady.Done() - checkpointHash = dds.computeHash(childCtx, after) - }() - } + checkpointHashReady.Add(1) + go func() { + defer checkpointHashReady.Done() + checkpointHash = dds.computeHash(childCtx, after) + }() delta, err := dds.computeEdits(childCtx, string(before), string(after)) if err != nil { diff --git a/pkg/backend/httpstate/snapshot.go b/pkg/backend/httpstate/snapshot.go index 3892391e5c11..a2801a4216b9 100644 --- a/pkg/backend/httpstate/snapshot.go +++ b/pkg/backend/httpstate/snapshot.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "os" "github.com/pulumi/pulumi/pkg/v3/backend" "github.com/pulumi/pulumi/pkg/v3/backend/httpstate/client" @@ -26,7 +25,6 @@ import ( "github.com/pulumi/pulumi/pkg/v3/resource/stack" "github.com/pulumi/pulumi/pkg/v3/secrets" "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/logging" ) @@ -82,9 +80,6 @@ func (persister *cloudSnapshotPersister) Save(snapshot *deploy.Snapshot) error { return err } if err := persister.saveDiff(ctx, diff, token); err != nil { - if differ.strictMode { - return err - } if logging.V(3) { logging.V(3).Infof("ignoring error saving checkpoint "+ "with PatchUpdateCheckpointDelta, falling back to "+ @@ -139,17 +134,10 @@ func (cb *cloudBackend) newSnapshotPersister(ctx context.Context, update client. sm: sm, } - if cmdutil.IsTruthy(os.Getenv("PULUMI_OPTIMIZED_CHECKPOINT_PATCH")) { - p.deploymentDiffState = newDeploymentDiffState() - - if cmdutil.IsTruthy(os.Getenv("PULUMI_OPTIMIZED_CHECKPOINT_PATCH_STRICT")) { - p.deploymentDiffState.strictMode = true - } - - if cmdutil.IsTruthy(os.Getenv("PULUMI_OPTIMIZED_CHECKPOINT_PATCH_NO_CHECKSUMS")) { - p.deploymentDiffState.noChecksums = true - } + caps := cb.capabilities(ctx) + deltaCaps := caps.deltaCheckpointUpdates + if deltaCaps != nil { + p.deploymentDiffState = newDeploymentDiffState(deltaCaps.CheckpointCutoffSizeBytes) } - return p } diff --git a/pkg/backend/httpstate/snapshot_test.go b/pkg/backend/httpstate/snapshot_test.go index 0e57e5fd7f1c..1d49b6b13afa 100644 --- a/pkg/backend/httpstate/snapshot_test.go +++ b/pkg/backend/httpstate/snapshot_test.go @@ -94,8 +94,18 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { } newMockServer := func() *httptest.Server { - return httptest.NewServer( - http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/api/capabilities": + resp := apitype.CapabilitiesResponse{Capabilities: []apitype.APICapabilityConfig{{ + Capability: apitype.DeltaCheckpointUploads, + Configuration: json.RawMessage(`{"checkpointCutoffSizeBytes":1}`), + }}} + err := json.NewEncoder(rw).Encode(resp) + assert.NoError(t, err) + return + case "/api/stacks/owner/project/stack/update/update-id/checkpointverbatim", + "/api/stacks/owner/project/stack/update/update-id/checkpointdelta": lastRequest = req rw.WriteHeader(200) message := `{}` @@ -107,7 +117,10 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { _, err = rw.Write([]byte(message)) assert.NoError(t, err) req.Body = io.NopCloser(bytes.NewBuffer(rbytes)) - })) + default: + panic(fmt.Sprintf("Path not supported: %v", req.URL.Path)) + } + })) } newMockTokenSource := func() tokenSourceCapability { @@ -126,8 +139,6 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { UpdateKind: apitype.UpdateUpdate, UpdateID: updateID, }, newMockTokenSource(), nil) - persister.deploymentDiffState = newDeploymentDiffState() - persister.deploymentDiffState.minimalDiffSize = 1 return persister } diff --git a/pkg/cmd/pulumi/new.go b/pkg/cmd/pulumi/new.go index 91dab7d70f56..4112fe05a0f9 100644 --- a/pkg/cmd/pulumi/new.go +++ b/pkg/cmd/pulumi/new.go @@ -823,32 +823,22 @@ func chooseTemplate(templates []workspace.Template, opts display.Options) (works // Customize the prompt a little bit (and disable color since it doesn't match our scheme). surveycore.DisableColor = true - var selectedOption workspace.Template - - for { - options, optionToTemplateMap := templatesToOptionArrayAndMap(templates, true) - nopts := len(options) - pageSize := optimalPageSize(optimalPageSizeOpts{nopts: nopts}) - message := fmt.Sprintf("\rPlease choose a template (%d/%d shown):\n", pageSize, nopts) - message = opts.Color.Colorize(colors.SpecPrompt + message + colors.Reset) - - var option string - if err := survey.AskOne(&survey.Select{ - Message: message, - Options: options, - PageSize: pageSize, - }, &option, surveyIcons(opts.Color)); err != nil { - return workspace.Template{}, errors.New(chooseTemplateErr) - } - - var has bool - selectedOption, has = optionToTemplateMap[option] - if has { - break - } + options, optionToTemplateMap := templatesToOptionArrayAndMap(templates, true) + nopts := len(options) + pageSize := optimalPageSize(optimalPageSizeOpts{nopts: nopts}) + message := fmt.Sprintf("\rPlease choose a template (%d/%d shown):\n", pageSize, nopts) + message = opts.Color.Colorize(colors.SpecPrompt + message + colors.Reset) + + var option string + if err := survey.AskOne(&survey.Select{ + Message: message, + Options: options, + PageSize: pageSize, + }, &option, surveyIcons(opts.Color)); err != nil { + return workspace.Template{}, errors.New(chooseTemplateErr) } - return selectedOption, nil + return optionToTemplateMap[option], nil } // parseConfig parses the config values passed via command line flags. diff --git a/pkg/cmd/pulumi/policy_new.go b/pkg/cmd/pulumi/policy_new.go index ad253a867c00..4be30af35481 100644 --- a/pkg/cmd/pulumi/policy_new.go +++ b/pkg/cmd/pulumi/policy_new.go @@ -149,6 +149,9 @@ func runNewPolicyPack(ctx context.Context, args newPolicyArgs) error { return err } } + if template.Errored() { + return fmt.Errorf("template '%s' is currently broken: %w", template.Name, template.Error) + } // Do a dry run, if we're not forcing files to be overwritten. if !args.force { @@ -366,15 +369,26 @@ func policyTemplatesToOptionArrayAndMap( // Build the array and map. var options []string + var brokenOptions []string nameToTemplateMap := make(map[string]workspace.PolicyPackTemplate) for _, template := range templates { + // If template is broken, indicate it in the project description. + if template.Errored() { + template.Description = brokenTemplateDescription + } + // Create the option string that combines the name, padding, and description. option := fmt.Sprintf(fmt.Sprintf("%%%ds %%s", -maxNameLength), template.Name, template.Description) - // Add it to the array and map. - options = append(options, option) nameToTemplateMap[option] = template + if template.Errored() { + brokenOptions = append(brokenOptions, option) + } else { + options = append(options, option) + } } + // After sorting the options, add the broken templates to the end sort.Strings(options) + options = append(options, brokenOptions...) return options, nameToTemplateMap } diff --git a/pkg/codegen/go/gen_program.go b/pkg/codegen/go/gen_program.go index bbff50318e9f..edbbec1d1cf8 100644 --- a/pkg/codegen/go/gen_program.go +++ b/pkg/codegen/go/gen_program.go @@ -664,9 +664,7 @@ func (g *generator) genResource(w io.Writer, r *pcl.Resource) { if len(r.Inputs) > 0 { g.Fgenf(w, "&%s.%sArgs{\n", modOrAlias, typ) for _, attr := range r.Inputs { - g.Fgenf(w, "%s: ", strings.Title(attr.Name)) - g.Fgenf(w, "%.v,\n", attr.Value) - + g.Fgenf(w, "%s: %.v,\n", strings.Title(attr.Name), attr.Value) } g.Fprint(w, "}") } else { diff --git a/pkg/codegen/go/gen_program_expressions.go b/pkg/codegen/go/gen_program_expressions.go index b651afb06d22..a116d3eb3332 100644 --- a/pkg/codegen/go/gen_program_expressions.go +++ b/pkg/codegen/go/gen_program_expressions.go @@ -579,9 +579,11 @@ func (g *generator) genScopeTraversalExpression( _, isInput = schemaType.(*schema.InputType) } - if resource, ok := expr.Parts[0].(*pcl.Resource); ok { + var sourceIsPlain bool + switch root := expr.Parts[0].(type) { + case *pcl.Resource: isInput = false - if _, ok := pcl.GetSchemaForType(resource.InputType); ok { + if _, ok := pcl.GetSchemaForType(root.InputType); ok { // convert .id into .ID() last := expr.Traversal[len(expr.Traversal)-1] if attr, ok := last.(hcl.TraverseAttr); ok && attr.Name == "id" { @@ -589,29 +591,39 @@ func (g *generator) genScopeTraversalExpression( expr.Traversal = expr.Traversal[:len(expr.Traversal)-1] } } + case *pcl.LocalVariable: + if root, ok := root.Definition.Value.(*model.FunctionCallExpression); ok && !pcl.IsOutputVersionInvokeCall(root) { + sourceIsPlain = true + } } // TODO if it's an array type, we need a lowering step to turn []string -> pulumi.StringArray if isInput { - argType := g.argumentTypeName(expr, expr.Type(), isInput) - if strings.HasSuffix(argType, "Array") { + argTypeName := g.argumentTypeName(expr, expr.Type(), isInput) + if strings.HasSuffix(argTypeName, "Array") { destTypeName := g.argumentTypeName(expr, destType, isInput) - if argType != destTypeName { + // `argTypeName` == `destTypeName` and `argTypeName` ends with `Array`, we + // know that `destType` is an outputty type. If the source is plain (and thus + // not outputty), then the types can never line up and we will need a + // conversion helper method. + if argTypeName != destTypeName || sourceIsPlain { // use a helper to transform prompt arrays into inputty arrays var helper *promptToInputArrayHelper - if h, ok := g.arrayHelpers[argType]; ok { + if h, ok := g.arrayHelpers[argTypeName]; ok { helper = h } else { // helpers are emitted at the end in the postamble step helper = &promptToInputArrayHelper{ - destType: argType, + destType: argTypeName, } - g.arrayHelpers[argType] = helper + g.arrayHelpers[argTypeName] = helper } + // Wrap the emitted expression in a call to the generated helper function. g.Fgenf(w, "%s(", helper.getFnName()) defer g.Fgenf(w, ")") } } else { + // Wrap the emitted expression in a type conversion. g.Fgenf(w, "%s(", g.argumentTypeName(expr, expr.Type(), isInput)) defer g.Fgenf(w, ")") } diff --git a/pkg/codegen/nodejs/gen_program_expressions.go b/pkg/codegen/nodejs/gen_program_expressions.go index 9ef51257bde5..94f9bc9a8438 100644 --- a/pkg/codegen/nodejs/gen_program_expressions.go +++ b/pkg/codegen/nodejs/gen_program_expressions.go @@ -429,7 +429,7 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC case "readFile": g.Fgenf(w, "fs.readFileSync(%v)", expr.Args[0]) case "readDir": - g.Fgenf(w, "fs.readDirSync(%v)", expr.Args[0]) + g.Fgenf(w, "fs.readdirSync(%v)", expr.Args[0]) case "secret": g.Fgenf(w, "pulumi.secret(%v)", expr.Args[0]) case "split": @@ -594,8 +594,20 @@ func (g *generator) genRelativeTraversal(w io.Writer, traversal hcl.Traversal, p contract.Failf("unexpected traversal part of type %T (%v)", part, part.SourceRange()) } + var indexPrefix string if model.IsOptionalType(model.GetTraversableType(parts[i])) { g.Fgen(w, "?") + // `expr?[expr]` is not valid typescript, since it looks like a ternary + // operator. + // + // Typescript solves this by inserting a `.` in before the `[`: `expr?.[expr]` + // + // We need to do the same when generating index based expressions. + indexPrefix = "." + } + + genIndex := func(inner string, value interface{}) { + g.Fgenf(w, "%s["+inner+"]", indexPrefix, value) } switch key.Type() { @@ -604,13 +616,13 @@ func (g *generator) genRelativeTraversal(w io.Writer, traversal hcl.Traversal, p if isLegalIdentifier(keyVal) { g.Fgenf(w, ".%s", keyVal) } else { - g.Fgenf(w, "[%q]", keyVal) + genIndex("%q", keyVal) } case cty.Number: idx, _ := key.AsBigFloat().Int64() - g.Fgenf(w, "[%d]", idx) + genIndex("%d", idx) default: - g.Fgenf(w, "[%q]", key.AsString()) + genIndex("%q", key.AsString()) } } } diff --git a/pkg/codegen/testing/test/program_driver.go b/pkg/codegen/testing/test/program_driver.go index 6b7669f2a20c..7dfc2c122939 100644 --- a/pkg/codegen/testing/test/program_driver.go +++ b/pkg/codegen/testing/test/program_driver.go @@ -66,13 +66,12 @@ var PulumiPulumiProgramTests = []ProgramTest{ { Directory: "aws-s3-folder", Description: "AWS S3 Folder", - ExpectNYIDiags: allProgLanguages.Except("go"), - SkipCompile: allProgLanguages.Except("dotnet"), + ExpectNYIDiags: codegen.NewStringSet("dotnet", "python"), + SkipCompile: codegen.NewStringSet("go", "python"), // Blocked on python: TODO[pulumi/pulumi#8062]: Re-enable this test. // Blocked on go: // TODO[pulumi/pulumi#8064] // TODO[pulumi/pulumi#8065] - // Blocked on nodejs: TODO[pulumi/pulumi#8063] }, { Directory: "aws-eks", @@ -81,14 +80,11 @@ var PulumiPulumiProgramTests = []ProgramTest{ { Directory: "aws-fargate", Description: "AWS Fargate", - - // TODO[pulumi/pulumi#8440] - SkipCompile: codegen.NewStringSet("go"), }, { Directory: "aws-s3-logging", Description: "AWS S3 with logging", - SkipCompile: allProgLanguages.Except("python").Except("dotnet"), + SkipCompile: codegen.NewStringSet("go"), // Blocked on nodejs: TODO[pulumi/pulumi#8068] // Flaky in go: TODO[pulumi/pulumi#8123] }, diff --git a/pkg/codegen/testing/test/testdata/aws-fargate-pp/go/aws-fargate.go b/pkg/codegen/testing/test/testdata/aws-fargate-pp/go/aws-fargate.go index a5c974d4eafb..a180575357c9 100644 --- a/pkg/codegen/testing/test/testdata/aws-fargate-pp/go/aws-fargate.go +++ b/pkg/codegen/testing/test/testdata/aws-fargate-pp/go/aws-fargate.go @@ -85,7 +85,7 @@ func main() { return err } webLoadBalancer, err := elasticloadbalancingv2.NewLoadBalancer(ctx, "webLoadBalancer", &elasticloadbalancingv2.LoadBalancerArgs{ - Subnets: subnets.Ids, + Subnets: toPulumiStringArray(subnets.Ids), SecurityGroups: pulumi.StringArray{ webSecurityGroup.ID(), }, @@ -153,7 +153,7 @@ func main() { TaskDefinition: appTask.Arn, NetworkConfiguration: &ecs.ServiceNetworkConfigurationArgs{ AssignPublicIp: pulumi.Bool(true), - Subnets: subnets.Ids, + Subnets: toPulumiStringArray(subnets.Ids), SecurityGroups: pulumi.StringArray{ webSecurityGroup.ID(), }, @@ -175,3 +175,10 @@ func main() { return nil }) } +func toPulumiStringArray(arr []string) pulumi.StringArray { + var pulumiArr pulumi.StringArray + for _, v := range arr { + pulumiArr = append(pulumiArr, pulumi.String(v)) + } + return pulumiArr +} diff --git a/pkg/codegen/testing/test/testdata/aws-s3-folder-pp/nodejs/aws-s3-folder.ts b/pkg/codegen/testing/test/testdata/aws-s3-folder-pp/nodejs/aws-s3-folder.ts index d7863011e199..9b969dd944d8 100644 --- a/pkg/codegen/testing/test/testdata/aws-s3-folder-pp/nodejs/aws-s3-folder.ts +++ b/pkg/codegen/testing/test/testdata/aws-s3-folder-pp/nodejs/aws-s3-folder.ts @@ -9,7 +9,7 @@ const siteBucket = new aws.s3.Bucket("siteBucket", {website: { const siteDir = "www"; // For each file in the directory, create an S3 object stored in `siteBucket` const files: aws.s3.BucketObject[] = []; -for (const range of fs.readDirSync(siteDir).map((v, k) => ({key: k, value: v}))) { +for (const range of fs.readdirSync(siteDir).map((v, k) => ({key: k, value: v}))) { files.push(new aws.s3.BucketObject(`files-${range.key}`, { bucket: siteBucket.id, key: range.value, diff --git a/pkg/codegen/testing/test/testdata/aws-s3-logging-pp/nodejs/aws-s3-logging.ts b/pkg/codegen/testing/test/testdata/aws-s3-logging-pp/nodejs/aws-s3-logging.ts index 51840abf7b34..68b8675231d8 100644 --- a/pkg/codegen/testing/test/testdata/aws-s3-logging-pp/nodejs/aws-s3-logging.ts +++ b/pkg/codegen/testing/test/testdata/aws-s3-logging-pp/nodejs/aws-s3-logging.ts @@ -5,4 +5,4 @@ const logs = new aws.s3.Bucket("logs", {}); const bucket = new aws.s3.Bucket("bucket", {loggings: [{ targetBucket: logs.bucket, }]}); -export const targetBucket = bucket.loggings.apply(loggings => loggings?[0]?.targetBucket); +export const targetBucket = bucket.loggings.apply(loggings => loggings?.[0]?.targetBucket); diff --git a/sdk/go/common/workspace/templates.go b/sdk/go/common/workspace/templates.go index b146084c3b54..2955d0667bd5 100644 --- a/sdk/go/common/workspace/templates.go +++ b/sdk/go/common/workspace/templates.go @@ -184,7 +184,11 @@ func (repo TemplateRepository) PolicyTemplates() ([]PolicyPackTemplate, error) { template, err := LoadPolicyPackTemplate(filepath.Join(path, name)) if err != nil && !errors.Is(err, fs.ErrNotExist) { - return nil, err + logging.V(2).Infof( + "Failed to load template %s: %s", + name, err.Error(), + ) + result = append(result, PolicyPackTemplate{Name: name, Error: err}) } else if err == nil { result = append(result, template) } @@ -217,6 +221,12 @@ type PolicyPackTemplate struct { Dir string // The directory containing PulumiPolicy.yaml. Name string // The name of the template. Description string // Description of the template. + Error error // Non-nil if the template is broken. +} + +// Errored returns if the template has an error +func (t PolicyPackTemplate) Errored() bool { + return t.Error != nil } // cleanupLegacyTemplateDir deletes an existing ~/.pulumi/templates directory if it isn't a git repository.