Merge remote-tracking branch 'origin/master' into dixler/10668/patch-…

…state-once

CHANGELOG_PENDING.md

@@ -1,4 +1,15 @@
 ### Improvements
 
+- [codegen/go] Chunk the `pulumiTypes.go` file to reduce max file size.
+  [#10666](https://github.com/pulumi/pulumi/pull/10666)
+
 ### Bug Fixes
 
+- Fix invalid resource type on `pulumi convert` to Go
+  [#10670](https://github.com/pulumi/pulumi/pull/10670)
+
+- [auto/nodejs] `onOutput` is now called incrementally as the
+  underyling Pulumi process produces data, instead of being called
+  once at the end of the process execution. This restores behavior
+  that regressed since 3.39.0.
+  [#10678](https://github.com/pulumi/pulumi/pull/10678)

pkg/backend/filestate/backend.go

@@ -291,7 +291,8 @@ func (b *localBackend) ValidateStackName(stackName string) error {
 
 	validNameRegex := regexp.MustCompile("^[A-Za-z0-9_.-]{1,100}$")
 	if !validNameRegex.MatchString(stackName) {
-		return errors.New("stack names may only contain alphanumeric, hyphens, underscores, or periods")
+		return errors.New(
+			"stack names are limited to 100 characters and may only contain alphanumeric, hyphens, underscores, or periods")
 	}
 
 	return nil

pkg/cmd/pulumi/convert.go

@@ -46,8 +46,7 @@ func newConvertCmd() *cobra.Command {
 		Short: "Convert Pulumi programs from YAML into other supported languages",
 		Long: "Convert Pulumi programs from YAML into other supported languages.\n" +
 			"\n" +
-			"The YAML program to convert will default to the manifest in the current working directory.\n" +
-			"You may also specify '-f' for the file path or '-d' for the directory path containing the manifests.\n",
+			"The YAML program to convert will default to the manifest in the current working directory.\n",
 		Run: cmdutil.RunResultFunc(func(cmd *cobra.Command, args []string) result.Result {
 
 			var projectGenerator projectGeneratorFunc

pkg/codegen/go/gen.go

@@ -3692,52 +3692,34 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error
 		}
 
 		// Types
-		if len(pkg.types) > 0 {
-			hasOutputs, importsAndAliases := false, map[string]string{}
-			for _, t := range pkg.types {
-				pkg.getImports(t, importsAndAliases)
-				hasOutputs = hasOutputs || pkg.detailsForType(t).hasOutputs()
-			}
-
-			sortedKnownTypes := make([]schema.Type, 0, len(knownTypes))
-			for k := range knownTypes {
-				sortedKnownTypes = append(sortedKnownTypes, k)
-			}
-			sort.Slice(sortedKnownTypes, func(i, j int) bool {
-				return sortedKnownTypes[i].String() < sortedKnownTypes[j].String()
-			})
-
-			collectionTypes := map[string]*nestedTypeInfo{}
-			for _, t := range sortedKnownTypes {
-				pkg.collectNestedCollectionTypes(collectionTypes, t)
-			}
-
-			// All collection types have Outputs
-			if len(collectionTypes) > 0 {
-				hasOutputs = true
-			}
+		sortedKnownTypes := make([]schema.Type, 0, len(knownTypes))
+		for k := range knownTypes {
+			sortedKnownTypes = append(sortedKnownTypes, k)
+		}
+		sort.Slice(sortedKnownTypes, func(i, j int) bool {
+			return sortedKnownTypes[i].String() < sortedKnownTypes[j].String()
+		})
 
-			var goImports []string
-			if hasOutputs {
-				goImports = []string{"context", "reflect"}
-				importsAndAliases["github.com/pulumi/pulumi/sdk/v3/go/pulumi"] = ""
+		for types, i := pkg.types, 0; len(types) > 0; i++ {
+			// 500 types corresponds to approximately 5M or 40_000 lines of code.
+			const chunkSize = 500
+			chunk := types
+			if len(chunk) > chunkSize {
+				chunk = chunk[:chunkSize]
 			}
+			types = types[len(chunk):]
 
 			buffer := &bytes.Buffer{}
-			pkg.genHeader(buffer, goImports, importsAndAliases)
-
-			for _, t := range pkg.types {
-				if err := pkg.genType(buffer, t); err != nil {
-					return nil, err
-				}
-				delete(knownTypes, t)
+			err := generateTypes(buffer, pkg, chunk, sortedKnownTypes)
+			if err != nil {
+				return nil, err
 			}
 
-			types := pkg.genNestedCollectionTypes(buffer, collectionTypes)
-
-			pkg.genTypeRegistrations(buffer, pkg.types, types...)
-
-			setFile(path.Join(mod, "pulumiTypes.go"), buffer.String())
+			typePath := "pulumiTypes"
+			if i != 0 {
+				typePath = fmt.Sprintf("%s%d", typePath, i)
+			}
+			setFile(path.Join(mod, typePath+".go"), buffer.String())
 		}
 
 		// Utilities
@@ -3771,6 +3753,43 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error
 	return files, nil
 }
 
+func generateTypes(w io.Writer, pkg *pkgContext, types []*schema.ObjectType, knownTypes []schema.Type) error {
+	hasOutputs, importsAndAliases := false, map[string]string{}
+	for _, t := range types {
+		pkg.getImports(t, importsAndAliases)
+		hasOutputs = hasOutputs || pkg.detailsForType(t).hasOutputs()
+	}
+
+	collectionTypes := map[string]*nestedTypeInfo{}
+	for _, t := range knownTypes {
+		pkg.collectNestedCollectionTypes(collectionTypes, t)
+	}
+
+	// All collection types have Outputs
+	if len(collectionTypes) > 0 {
+		hasOutputs = true
+	}
+
+	var goImports []string
+	if hasOutputs {
+		goImports = []string{"context", "reflect"}
+		importsAndAliases["github.com/pulumi/pulumi/sdk/v3/go/pulumi"] = ""
+	}
+
+	pkg.genHeader(w, goImports, importsAndAliases)
+
+	for _, t := range types {
+		if err := pkg.genType(w, t); err != nil {
+			return err
+		}
+	}
+
+	typeNames := pkg.genNestedCollectionTypes(w, collectionTypes)
+
+	pkg.genTypeRegistrations(w, types, typeNames...)
+	return nil
+}
+
 func allResourcesAreOverlays(resources []*schema.Resource) bool {
 	for _, r := range resources {
 		if !r.IsOverlay {

pkg/codegen/go/gen_program.go

@@ -939,6 +939,7 @@ func (g *generator) getModOrAlias(pkg, mod, originalMod string) string {
 			return path.Base(info.ImportBasePath)
 		}
 	}
+	mod = strings.Split(mod, "/")[0]
 	return mod
 }
 

pkg/codegen/go/gen_program_expressions.go

@@ -184,8 +184,9 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC
 					"Unsafe enum conversions from type %s not implemented yet: %s => %s",
 					from.Type(), from, to))
 			}
-			enumTag := fmt.Sprintf("%s.%s",
-				tokenToModule(to.Token), tokenToName(to.Token))
+			pkg, mod, typ, _ := pcl.DecomposeToken(to.Token, to.SyntaxNode().Range())
+			mod = g.getModOrAlias(pkg, mod, mod)
+			enumTag := fmt.Sprintf("%s.%s", mod, typ)
 			if isOutput {
 				g.Fgenf(w,
 					"%.v.ApplyT(func(x *%[3]s) %[2]s { return %[2]s(*x) }).(%[2]sOutput)",

sdk/nodejs/automation/cmd.ts

@@ -56,14 +56,23 @@ export async function runPulumiCmd(
     const env = { ...process.env, ...additionalEnv };
 
     try {
-        const { stdout, stderr, exitCode } = await execa("pulumi", args, { env, cwd });
+        const proc = execa("pulumi", args, { env, cwd });
+
+        if (onOutput && proc.stdout) {
+            proc.stdout!.on("data", (data: any) => {
+                if (data && data.toString) {
+                    data = data.toString();
+                }
+                onOutput(data);
+            });
+        }
+
+        const { stdout, stderr, exitCode } = await proc;
         const commandResult = new CommandResult(stdout, stderr, exitCode);
         if (exitCode !== 0) {
             throw createCommandError(commandResult);
         }
-        if (onOutput) {
-            onOutput(stdout);
-        }
+
         return commandResult;
     } catch (err) {
         const error = err as Error;

sdk/nodejs/tests/automation/cmd.spec.ts

@@ -1,4 +1,4 @@
-// Copyright 2016-2021, Pulumi Corporation.
+// Copyright 2016-2022, Pulumi Corporation.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,17 +13,19 @@
 // limitations under the License.
 
 import * as assert from "assert";
-import * as sinon from "sinon";
 import { runPulumiCmd } from "../../automation";
 import { asyncTest } from "../util";
 
 describe("automation/cmd", () => {
     it("calls onOutput when provided to runPulumiCmd", asyncTest(async () => {
-        const spy = sinon.spy();
-        await runPulumiCmd(["version"], ".", {}, spy);
-
-        assert.ok(spy.calledOnce);
-        assert.strictEqual(spy.firstCall.firstArg, spy.lastCall.lastArg);
+        let output = "";
+        let numCalls = 0;
+        await runPulumiCmd(["--help"], ".", {}, (data: string) => {
+            output += data;
+            numCalls += 1;
+        });
+        assert.ok(numCalls > 1, `expected numCalls > 1, got ${numCalls}`);
+        assert.match(output, new RegExp("Usage[:]"));
+        assert.match(output, new RegExp("[-][-]verbose"));
     }));
 });
-

tests/integration/integration_test.go

@@ -82,7 +82,8 @@ func TestStackTagValidation(t *testing.T) {
 
 		stdout, stderr := e.RunCommandExpectError("pulumi", "stack", "init", "invalid name (spaces, parens, etc.)")
 		assert.Equal(t, "", stdout)
-		assert.Contains(t, stderr, "stack names may only contain alphanumeric, hyphens, underscores, or periods")
+		assert.Contains(t, stderr,
+			"stack names are limited to 100 characters and may only contain alphanumeric, hyphens, underscores, or periods")
 	})
 
 	t.Run("Error_DescriptionLength", func(t *testing.T) {