Add ability to define a globalEnv key in turbo.json (#1950)

This simplifies the globalDependencies key, so it can only
be responsible for file declarations and not need a microsyntax for env
vars ($ prefixes).

cli/internal/fs/testdata/invalid-global-env/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "test-repo"
+}

cli/internal/fs/testdata/invalid-global-env/turbo.json

@@ -0,0 +1,11 @@
+{
+  // Both global declarations with duplicates
+  "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"],
+  // some invalid values
+  "globalEnv": ["FOO", "BAZ", "$QUX"],
+  "pipeline": {
+    "task1": {
+      "dependsOn": ["$A"]
+    }
+  }
+}

cli/internal/fs/testdata/legacy-env/turbo.json

@@ -1,5 +1,8 @@
 // mocked test comment
 {
+  // Both global declarations with duplicates and with
+  "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"],
+  "globalEnv": ["FOO", "BAZ", "QUX"],
   "pipeline": {
     // Only legacy declaration
     "task1": {

cli/internal/fs/turbo_json.go

@@ -20,17 +20,26 @@ const (
 
 var defaultOutputs = []string{"dist/**/*", "build/**/*"}
 
-// TurboJSON is the root turborepo configuration
-type TurboJSON struct {
+type rawTurboJSON struct {
 	// Global root filesystem dependencies
 	GlobalDependencies []string `json:"globalDependencies,omitempty"`
+	// Global env
+	GlobalEnv []string `json:"globalEnv,omitempty"`
 	// Pipeline is a map of Turbo pipeline entries which define the task graph
 	// and cache behavior on a per task or per package-task basis.
 	Pipeline Pipeline
 	// Configuration options when interfacing with the remote cache
 	RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"`
 }
 
+// TurboJSON is the root turborepo configuration
+type TurboJSON struct {
+	GlobalDeps         []string
+	GlobalEnv          []string
+	Pipeline           Pipeline
+	RemoteCacheOptions RemoteCacheOptions
+}
+
 // RemoteCacheOptions is a struct for deserializing .remoteCache of configFile
 type RemoteCacheOptions struct {
 	TeamID    string `json:"teamId,omitempty"`
@@ -107,10 +116,13 @@ func readTurboJSON(path turbopath.AbsolutePath) (*TurboJSON, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	err = jsonc.Unmarshal(data, &turboJSON)
+
 	if err != nil {
 		return nil, err
 	}
+
 	return turboJSON, nil
 }
 
@@ -193,3 +205,42 @@ func (c *TaskDefinition) UnmarshalJSON(data []byte) error {
 	c.OutputMode = rawPipeline.OutputMode
 	return nil
 }
+
+// UnmarshalJSON deserializes TurboJSON objects into struct
+func (c *TurboJSON) UnmarshalJSON(data []byte) error {
+	raw := &rawTurboJSON{}
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+
+	envVarDependencies := make(util.Set)
+	globalFileDependencies := make(util.Set)
+
+	for _, value := range raw.GlobalEnv {
+		if strings.HasPrefix(value, envPipelineDelimiter) {
+			// Hard error to help people specify this correctly during migration.
+			// TODO: Remove this error after we have run summary.
+			return fmt.Errorf("You specified \"%s\" in the \"env\" key. You should not prefix your environment variables with \"%s\"", value, envPipelineDelimiter)
+		}
+
+		envVarDependencies.Add(value)
+	}
+
+	for _, value := range raw.GlobalDependencies {
+		if strings.HasPrefix(value, envPipelineDelimiter) {
+			envVarDependencies.Add(strings.TrimPrefix(value, envPipelineDelimiter))
+		} else {
+			globalFileDependencies.Add(value)
+		}
+	}
+
+	// turn the set into an array and assign to the TurboJSON struct fields.
+	c.GlobalEnv = envVarDependencies.UnsafeListOfStrings()
+	c.GlobalDeps = globalFileDependencies.UnsafeListOfStrings()
+
+	// copy these over, we don't need any changes here.
+	c.Pipeline = raw.Pipeline
+	c.RemoteCacheOptions = raw.RemoteCacheOptions
+
+	return nil
+}

cli/internal/fs/turbo_json_test.go

@@ -169,6 +169,23 @@ func Test_ReadTurboConfig_InvalidEnvDeclarations2(t *testing.T) {
 	assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr)
 }
 
+func Test_ReadTurboConfig_InvalidGlobalEnvDeclarations(t *testing.T) {
+	testDir := getTestDir(t, "invalid-global-env")
+
+	packageJSONPath := testDir.Join("package.json")
+	rootPackageJSON, pkgJSONReadErr := ReadPackageJSON(packageJSONPath)
+
+	if pkgJSONReadErr != nil {
+		t.Fatalf("invalid parse: %#v", pkgJSONReadErr)
+	}
+
+	_, turboJSONReadErr := ReadTurboConfig(testDir, rootPackageJSON)
+
+	expectedErrorMsg := "turbo.json: You specified \"$QUX\" in the \"env\" key. You should not prefix your environment variables with \"$\""
+
+	assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr)
+}
+
 func Test_ReadTurboConfig_EnvDeclarations(t *testing.T) {
 	testDir := getTestDir(t, "legacy-env")
 
@@ -186,7 +203,6 @@ func Test_ReadTurboConfig_EnvDeclarations(t *testing.T) {
 	}
 
 	pipeline := turboJSON.Pipeline
-
 	assert.EqualValues(t, sortedArray(pipeline["task1"].EnvVarDependencies), sortedArray([]string{"A"}))
 	assert.EqualValues(t, sortedArray(pipeline["task2"].EnvVarDependencies), sortedArray([]string{"A"}))
 	assert.EqualValues(t, sortedArray(pipeline["task3"].EnvVarDependencies), sortedArray([]string{"A"}))
@@ -197,6 +213,10 @@ func Test_ReadTurboConfig_EnvDeclarations(t *testing.T) {
 	assert.EqualValues(t, sortedArray(pipeline["task9"].EnvVarDependencies), sortedArray([]string{"A"}))
 	assert.EqualValues(t, sortedArray(pipeline["task10"].EnvVarDependencies), sortedArray([]string{"A"}))
 	assert.EqualValues(t, sortedArray(pipeline["task11"].EnvVarDependencies), sortedArray([]string{"A", "B"}))
+
+	// check global env vars also
+	assert.EqualValues(t, sortedArray([]string{"FOO", "BAR", "BAZ", "QUX"}), sortedArray(turboJSON.GlobalEnv))
+	assert.EqualValues(t, sortedArray([]string{"somefile.txt"}), sortedArray(turboJSON.GlobalDeps))
 }
 
 // Helpers

cli/internal/run/global_hash.go

@@ -23,43 +23,36 @@ var _defaultEnvVars = []string{
 	"VERCEL_ANALYTICS_ID",
 }
 
-func calculateGlobalHash(rootpath turbopath.AbsolutePath, rootPackageJSON *fs.PackageJSON, pipeline fs.Pipeline, externalGlobalDependencies []string, packageManager *packagemanager.PackageManager, logger hclog.Logger, env []string) (string, error) {
-	// Calculate the global hash
-	globalDeps := make(util.Set)
-
+func calculateGlobalHash(rootpath turbopath.AbsolutePath, rootPackageJSON *fs.PackageJSON, pipeline fs.Pipeline, envVarDependencies []string, globalFileDependencies []string, packageManager *packagemanager.PackageManager, logger hclog.Logger, env []string) (string, error) {
+	// Calculate env var dependencies
 	globalHashableEnvNames := []string{}
 	globalHashableEnvPairs := []string{}
-	// Calculate global file and env var dependencies
 	for _, builtinEnvVar := range _defaultEnvVars {
 		globalHashableEnvNames = append(globalHashableEnvNames, builtinEnvVar)
 		globalHashableEnvPairs = append(globalHashableEnvPairs, fmt.Sprintf("%v=%v", builtinEnvVar, os.Getenv(builtinEnvVar)))
 	}
-	if len(externalGlobalDependencies) > 0 {
-		var globs []string
-		for _, v := range externalGlobalDependencies {
-			if strings.HasPrefix(v, "$") {
-				trimmed := strings.TrimPrefix(v, "$")
-				globalHashableEnvNames = append(globalHashableEnvNames, trimmed)
-				globalHashableEnvPairs = append(globalHashableEnvPairs, fmt.Sprintf("%v=%v", trimmed, os.Getenv(trimmed)))
-			} else {
-				globs = append(globs, v)
-			}
-		}
 
-		if len(globs) > 0 {
-			ignores, err := packageManager.GetWorkspaceIgnores(rootpath)
-			if err != nil {
-				return "", err
-			}
+	// Calculate global env var dependencies
+	for _, v := range envVarDependencies {
+		globalHashableEnvNames = append(globalHashableEnvNames, v)
+		globalHashableEnvPairs = append(globalHashableEnvPairs, fmt.Sprintf("%v=%v", v, os.Getenv(v)))
+	}
+
+	// Calculate global file dependencies
+	globalDeps := make(util.Set)
+	if len(globalFileDependencies) > 0 {
+		ignores, err := packageManager.GetWorkspaceIgnores(rootpath)
+		if err != nil {
+			return "", err
+		}
 
-			f, err := globby.GlobFiles(rootpath.ToStringDuringMigration(), globs, ignores)
-			if err != nil {
-				return "", err
-			}
+		f, err := globby.GlobFiles(rootpath.ToStringDuringMigration(), globalFileDependencies, ignores)
+		if err != nil {
+			return "", err
+		}
 
-			for _, val := range f {
-				globalDeps.Add(val)
-			}
+		for _, val := range f {
+			globalDeps.Add(val)
 		}
 	}
 

cli/internal/run/run.go

@@ -237,7 +237,8 @@ func (r *run) run(ctx gocontext.Context, targets []string) error {
 		r.base.RepoRoot,
 		rootPackageJSON,
 		pipeline,
-		turboJSON.GlobalDependencies,
+		turboJSON.GlobalEnv,
+		turboJSON.GlobalDeps,
 		pkgDepGraph.PackageManager,
 		r.base.Logger,
 		os.Environ(),