diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 05e6ced76a41..dc1999452d80 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,4 +1,15 @@ ### Improvements +- [codegen/go] Chunk the `pulumiTypes.go` file to reduce max file size. + [#10666](https://github.com/pulumi/pulumi/pull/10666) + ### Bug Fixes +- Fix invalid resource type on `pulumi convert` to Go + [#10670](https://github.com/pulumi/pulumi/pull/10670) + +- [auto/nodejs] `onOutput` is now called incrementally as the + underyling Pulumi process produces data, instead of being called + once at the end of the process execution. This restores behavior + that regressed since 3.39.0. + [#10678](https://github.com/pulumi/pulumi/pull/10678) diff --git a/pkg/backend/filestate/backend.go b/pkg/backend/filestate/backend.go index d1e2dc985a34..22d0f01652f1 100644 --- a/pkg/backend/filestate/backend.go +++ b/pkg/backend/filestate/backend.go @@ -291,7 +291,8 @@ func (b *localBackend) ValidateStackName(stackName string) error { validNameRegex := regexp.MustCompile("^[A-Za-z0-9_.-]{1,100}$") if !validNameRegex.MatchString(stackName) { - return errors.New("stack names may only contain alphanumeric, hyphens, underscores, or periods") + return errors.New( + "stack names are limited to 100 characters and may only contain alphanumeric, hyphens, underscores, or periods") } return nil diff --git a/pkg/cmd/pulumi/convert.go b/pkg/cmd/pulumi/convert.go index c578dd43f56f..db16219f2cbf 100644 --- a/pkg/cmd/pulumi/convert.go +++ b/pkg/cmd/pulumi/convert.go @@ -46,8 +46,7 @@ func newConvertCmd() *cobra.Command { Short: "Convert Pulumi programs from YAML into other supported languages", Long: "Convert Pulumi programs from YAML into other supported languages.\n" + "\n" + - "The YAML program to convert will default to the manifest in the current working directory.\n" + - "You may also specify '-f' for the file path or '-d' for the directory path containing the manifests.\n", + "The YAML program to convert will default to the manifest in the current working directory.\n", Run: cmdutil.RunResultFunc(func(cmd *cobra.Command, args []string) result.Result { var projectGenerator projectGeneratorFunc diff --git a/pkg/codegen/go/gen.go b/pkg/codegen/go/gen.go index 2e668b2bca5f..fdc810c98dbb 100644 --- a/pkg/codegen/go/gen.go +++ b/pkg/codegen/go/gen.go @@ -3692,52 +3692,34 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error } // Types - if len(pkg.types) > 0 { - hasOutputs, importsAndAliases := false, map[string]string{} - for _, t := range pkg.types { - pkg.getImports(t, importsAndAliases) - hasOutputs = hasOutputs || pkg.detailsForType(t).hasOutputs() - } - - sortedKnownTypes := make([]schema.Type, 0, len(knownTypes)) - for k := range knownTypes { - sortedKnownTypes = append(sortedKnownTypes, k) - } - sort.Slice(sortedKnownTypes, func(i, j int) bool { - return sortedKnownTypes[i].String() < sortedKnownTypes[j].String() - }) - - collectionTypes := map[string]*nestedTypeInfo{} - for _, t := range sortedKnownTypes { - pkg.collectNestedCollectionTypes(collectionTypes, t) - } - - // All collection types have Outputs - if len(collectionTypes) > 0 { - hasOutputs = true - } + sortedKnownTypes := make([]schema.Type, 0, len(knownTypes)) + for k := range knownTypes { + sortedKnownTypes = append(sortedKnownTypes, k) + } + sort.Slice(sortedKnownTypes, func(i, j int) bool { + return sortedKnownTypes[i].String() < sortedKnownTypes[j].String() + }) - var goImports []string - if hasOutputs { - goImports = []string{"context", "reflect"} - importsAndAliases["github.com/pulumi/pulumi/sdk/v3/go/pulumi"] = "" + for types, i := pkg.types, 0; len(types) > 0; i++ { + // 500 types corresponds to approximately 5M or 40_000 lines of code. + const chunkSize = 500 + chunk := types + if len(chunk) > chunkSize { + chunk = chunk[:chunkSize] } + types = types[len(chunk):] buffer := &bytes.Buffer{} - pkg.genHeader(buffer, goImports, importsAndAliases) - - for _, t := range pkg.types { - if err := pkg.genType(buffer, t); err != nil { - return nil, err - } - delete(knownTypes, t) + err := generateTypes(buffer, pkg, chunk, sortedKnownTypes) + if err != nil { + return nil, err } - types := pkg.genNestedCollectionTypes(buffer, collectionTypes) - - pkg.genTypeRegistrations(buffer, pkg.types, types...) - - setFile(path.Join(mod, "pulumiTypes.go"), buffer.String()) + typePath := "pulumiTypes" + if i != 0 { + typePath = fmt.Sprintf("%s%d", typePath, i) + } + setFile(path.Join(mod, typePath+".go"), buffer.String()) } // Utilities @@ -3771,6 +3753,43 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error return files, nil } +func generateTypes(w io.Writer, pkg *pkgContext, types []*schema.ObjectType, knownTypes []schema.Type) error { + hasOutputs, importsAndAliases := false, map[string]string{} + for _, t := range types { + pkg.getImports(t, importsAndAliases) + hasOutputs = hasOutputs || pkg.detailsForType(t).hasOutputs() + } + + collectionTypes := map[string]*nestedTypeInfo{} + for _, t := range knownTypes { + pkg.collectNestedCollectionTypes(collectionTypes, t) + } + + // All collection types have Outputs + if len(collectionTypes) > 0 { + hasOutputs = true + } + + var goImports []string + if hasOutputs { + goImports = []string{"context", "reflect"} + importsAndAliases["github.com/pulumi/pulumi/sdk/v3/go/pulumi"] = "" + } + + pkg.genHeader(w, goImports, importsAndAliases) + + for _, t := range types { + if err := pkg.genType(w, t); err != nil { + return err + } + } + + typeNames := pkg.genNestedCollectionTypes(w, collectionTypes) + + pkg.genTypeRegistrations(w, types, typeNames...) + return nil +} + func allResourcesAreOverlays(resources []*schema.Resource) bool { for _, r := range resources { if !r.IsOverlay { diff --git a/pkg/codegen/go/gen_program.go b/pkg/codegen/go/gen_program.go index 3fc54b9400b8..ea87e20ea99d 100644 --- a/pkg/codegen/go/gen_program.go +++ b/pkg/codegen/go/gen_program.go @@ -939,6 +939,7 @@ func (g *generator) getModOrAlias(pkg, mod, originalMod string) string { return path.Base(info.ImportBasePath) } } + mod = strings.Split(mod, "/")[0] return mod } diff --git a/pkg/codegen/go/gen_program_expressions.go b/pkg/codegen/go/gen_program_expressions.go index 5abbab6f301c..5fc8124694a3 100644 --- a/pkg/codegen/go/gen_program_expressions.go +++ b/pkg/codegen/go/gen_program_expressions.go @@ -184,8 +184,9 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC "Unsafe enum conversions from type %s not implemented yet: %s => %s", from.Type(), from, to)) } - enumTag := fmt.Sprintf("%s.%s", - tokenToModule(to.Token), tokenToName(to.Token)) + pkg, mod, typ, _ := pcl.DecomposeToken(to.Token, to.SyntaxNode().Range()) + mod = g.getModOrAlias(pkg, mod, mod) + enumTag := fmt.Sprintf("%s.%s", mod, typ) if isOutput { g.Fgenf(w, "%.v.ApplyT(func(x *%[3]s) %[2]s { return %[2]s(*x) }).(%[2]sOutput)", diff --git a/sdk/nodejs/automation/cmd.ts b/sdk/nodejs/automation/cmd.ts index 76896938e0b7..01eaea543187 100644 --- a/sdk/nodejs/automation/cmd.ts +++ b/sdk/nodejs/automation/cmd.ts @@ -56,14 +56,23 @@ export async function runPulumiCmd( const env = { ...process.env, ...additionalEnv }; try { - const { stdout, stderr, exitCode } = await execa("pulumi", args, { env, cwd }); + const proc = execa("pulumi", args, { env, cwd }); + + if (onOutput && proc.stdout) { + proc.stdout!.on("data", (data: any) => { + if (data && data.toString) { + data = data.toString(); + } + onOutput(data); + }); + } + + const { stdout, stderr, exitCode } = await proc; const commandResult = new CommandResult(stdout, stderr, exitCode); if (exitCode !== 0) { throw createCommandError(commandResult); } - if (onOutput) { - onOutput(stdout); - } + return commandResult; } catch (err) { const error = err as Error; diff --git a/sdk/nodejs/tests/automation/cmd.spec.ts b/sdk/nodejs/tests/automation/cmd.spec.ts index 1eb7cb2345c4..38f60b34c4fd 100644 --- a/sdk/nodejs/tests/automation/cmd.spec.ts +++ b/sdk/nodejs/tests/automation/cmd.spec.ts @@ -1,4 +1,4 @@ -// Copyright 2016-2021, Pulumi Corporation. +// Copyright 2016-2022, Pulumi Corporation. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,17 +13,19 @@ // limitations under the License. import * as assert from "assert"; -import * as sinon from "sinon"; import { runPulumiCmd } from "../../automation"; import { asyncTest } from "../util"; describe("automation/cmd", () => { it("calls onOutput when provided to runPulumiCmd", asyncTest(async () => { - const spy = sinon.spy(); - await runPulumiCmd(["version"], ".", {}, spy); - - assert.ok(spy.calledOnce); - assert.strictEqual(spy.firstCall.firstArg, spy.lastCall.lastArg); + let output = ""; + let numCalls = 0; + await runPulumiCmd(["--help"], ".", {}, (data: string) => { + output += data; + numCalls += 1; + }); + assert.ok(numCalls > 1, `expected numCalls > 1, got ${numCalls}`); + assert.match(output, new RegExp("Usage[:]")); + assert.match(output, new RegExp("[-][-]verbose")); })); }); - diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 332130c07b46..929c1f960995 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -82,7 +82,8 @@ func TestStackTagValidation(t *testing.T) { stdout, stderr := e.RunCommandExpectError("pulumi", "stack", "init", "invalid name (spaces, parens, etc.)") assert.Equal(t, "", stdout) - assert.Contains(t, stderr, "stack names may only contain alphanumeric, hyphens, underscores, or periods") + assert.Contains(t, stderr, + "stack names are limited to 100 characters and may only contain alphanumeric, hyphens, underscores, or periods") }) t.Run("Error_DescriptionLength", func(t *testing.T) {