fix: copy turbo.json when running turbo prune (#633)

cli/internal/prune/prune.go

@@ -212,6 +212,13 @@ func (c *PruneCommand) Run(args []string) int {
 				return 1
 			}
 		}
+		// We only need to actually copy turbo.json into "full" folder since it isn't needed for installation in docker
+		if fs.FileExists("turbo.json") {
+			if err := fs.CopyFile("turbo.json", filepath.Join(pruneOptions.cwd, "out", "full", "turbo.json"), fs.DirPermissions); err != nil {
+				c.logError(c.Config.Logger, "", fmt.Errorf("failed to copy root turbo.json: %w", err))
+				return 1
+			}
+		}
 
 		if err := fs.CopyFile("package.json", filepath.Join(pruneOptions.cwd, "out", "full", "package.json"), fs.DirPermissions); err != nil {
 			c.logError(c.Config.Logger, "", fmt.Errorf("failed to copy root package.json: %w", err))
@@ -229,6 +236,14 @@ func (c *PruneCommand) Run(args []string) int {
 				return 1
 			}
 		}
+
+		if fs.FileExists("turbo.json") {
+			if err := fs.CopyFile("turbo.json", filepath.Join(pruneOptions.cwd, "out", "turbo.json"), fs.DirPermissions); err != nil {
+				c.logError(c.Config.Logger, "", fmt.Errorf("failed to copy root turbo.json: %w", err))
+				return 1
+			}
+		}
+
 		if err := fs.CopyFile("package.json", filepath.Join(pruneOptions.cwd, "out", "package.json"), fs.DirPermissions); err != nil {
 			c.logError(c.Config.Logger, "", fmt.Errorf("failed to copy root package.json: %w", err))
 			return 1
@@ -249,42 +264,50 @@ func (c *PruneCommand) Run(args []string) int {
 		c.logError(c.Config.Logger, "", fmt.Errorf("failed to write sub-lockfile: %w", err))
 		return 1
 	}
-	// because of yarn being yarn, we need to inject lines in between each block of YAML to make it "valid" syml
-	f, err := os.Open(filepath.Join(filepath.Join(pruneOptions.cwd, "out", "yarn.lock")))
-	if err != nil {
-		c.logError(c.Config.Logger, "", fmt.Errorf("failed to massage lockfile: %w", err))
-	}
-	defer f.Close()
 
-	output, err := os.Create(filepath.Join(pruneOptions.cwd, "out", "yarn-tmp.lock"))
-	writer := bufio.NewWriter(output)
+	tmpGeneratedLockfile, err := os.Create(filepath.Join(pruneOptions.cwd, "out", "yarn-tmp.lock"))
+	tmpGeneratedLockfileWriter := bufio.NewWriter(tmpGeneratedLockfile)
 	if err != nil {
 		c.logError(c.Config.Logger, "", fmt.Errorf("failed create tempory lockfile: %w", err))
+		return 1
 	}
-	defer output.Close()
 
 	if ctx.Backend.Name == "nodejs-yarn" {
-		writer.WriteString("# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile v1\n\n")
+		tmpGeneratedLockfileWriter.WriteString("# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile v1\n\n")
 	} else {
-		writer.WriteString("# This file is generated by running \"yarn install\" inside your project.\n# Manual changes might be lost - proceed with caution!\n\n__metadata:\nversion: 5\ncacheKey: 8\n\n")
+		tmpGeneratedLockfileWriter.WriteString("# This file is generated by running \"yarn install\" inside your project.\n# Manual changes might be lost - proceed with caution!\n\n__metadata:\nversion: 5\ncacheKey: 8\n\n")
 	}
 
-	scan := bufio.NewScanner(f)
+	// because of yarn being yarn, we need to inject lines in between each block of YAML to make it "valid" SYML
+	generatedLockfile, err := os.Open(filepath.Join(filepath.Join(pruneOptions.cwd, "out", "yarn.lock")))
+	if err != nil {
+		c.logError(c.Config.Logger, "", fmt.Errorf("failed to massage lockfile: %w", err))
+		return 1
+	}
+
+	scan := bufio.NewScanner(generatedLockfile)
 	buf := make([]byte, 0, 1024*1024)
 	scan.Buffer(buf, 10*1024*1024)
 	for scan.Scan() {
 		line := scan.Text() //Writing to Stdout
 		if !strings.HasPrefix(line, " ") {
-			writer.WriteString(fmt.Sprintf("\n%v\n", strings.ReplaceAll(line, "'", "\"")))
+			tmpGeneratedLockfileWriter.WriteString(fmt.Sprintf("\n%v\n", strings.ReplaceAll(line, "'", "\"")))
 		} else {
-			writer.WriteString(fmt.Sprintf("%v\n", strings.ReplaceAll(line, "'", "\"")))
+			tmpGeneratedLockfileWriter.WriteString(fmt.Sprintf("%v\n", strings.ReplaceAll(line, "'", "\"")))
 		}
 	}
-	writer.Flush() // make sure to flush the log write before we start saving it.
+	// Make sure to flush the log write before we start saving it.
+	tmpGeneratedLockfileWriter.Flush()
 
+	// Close the files before we rename them
+	tmpGeneratedLockfile.Close()
+	generatedLockfile.Close()
+
+	// Rename the file
 	err = os.Rename(filepath.Join(pruneOptions.cwd, "out", "yarn-tmp.lock"), filepath.Join(pruneOptions.cwd, "out", "yarn.lock"))
 	if err != nil {
 		c.logError(c.Config.Logger, "", fmt.Errorf("failed finalize lockfile: %w", err))
+		return 1
 	}
 	return 0
 }

cli/scripts/e2e/e2e.ts

@@ -67,7 +67,7 @@ function runSmokeTests<T>(
       options.cwd ? " from " + options.cwd : ""
     } `,
     async () => {
-      const results = repo.turbo("run", ["test", "--stream", "-vvv"], options);
+      const results = repo.turbo("run", ["test", "--stream"], options);
       assert.equal(0, results.exitCode, "exit code should be 0");
       const commandOutput = getCommandOutputAsArray(results);
       const hash = getHashFromOutput(commandOutput, "c#test");
@@ -95,32 +95,36 @@ function runSmokeTests<T>(
       });
 
       const sinceCommandOutput = getCommandOutputAsArray(
-        repo.turbo("run", ["test", "--since=main", "--stream", "-vvv"], options)
+        repo.turbo("run", ["test", "--since=main", "--stream"], options)
       );
 
-      assert.equal(
+      assert.fixture(
         `• Packages changed since main: a`,
         sinceCommandOutput[0],
         "Calculates changed packages (--since)"
       );
-      assert.equal(
+      assert.fixture(
         `• Packages in scope: a`,
         sinceCommandOutput[1],
         "Packages in scope"
       );
-      assert.equal(
+      assert.fixture(
         `• Running test in 1 packages`,
         sinceCommandOutput[2],
         "Runs only in changed packages"
       );
-      assert.ok(
-        sinceCommandOutput[3].startsWith(`a:test: cache miss, executing`),
+      assert.fixture(
+        sinceCommandOutput[3],
+        `a:test: cache miss, executing ${getHashFromOutput(
+          sinceCommandOutput,
+          "a#test"
+        )}`,
         "Cache miss in changed package"
       );
 
       // Check cache hit after another run
       const sinceCommandSecondRunOutput = getCommandOutputAsArray(
-        repo.turbo("run", ["test", "--since=main", "--stream", "-vvv"], options)
+        repo.turbo("run", ["test", "--since=main", "--stream"], options)
       );
       assert.equal(
         `• Packages changed since main: a`,
@@ -138,10 +142,13 @@ function runSmokeTests<T>(
         "Runs only in changed packages after a second run"
       );
 
-      assert.ok(
-        sinceCommandSecondRunOutput[3].startsWith(
-          `a:test: cache hit, replaying output`
-        ),
+      assert.fixture(
+        sinceCommandSecondRunOutput[3],
+        `a:test: cache hit, replaying output ${getHashFromOutput(
+          sinceCommandSecondRunOutput,
+          "a#test"
+        )}`,
+
         "Cache hit in changed package after a second run"
       );
 
@@ -154,12 +161,12 @@ function runSmokeTests<T>(
         repo.turbo("run", ["test", "--stream"], options)
       );
 
-      assert.equal(
+      assert.fixture(
         `• Packages in scope: a, b, c`,
         commandOnceBHasChangedOutput[0],
         "After running, changing source of b, and running `turbo run test` again, should print `Packages in scope: a, b, c`"
       );
-      assert.equal(
+      assert.fixture(
         `• Running test in 3 packages`,
         commandOnceBHasChangedOutput[1],
         "After running, changing source of b, and running `turbo run test` again, should print `Running in 3 packages`"
@@ -184,6 +191,77 @@ function runSmokeTests<T>(
       );
     }
   );
+
+  if (npmClient === "yarn") {
+    // Test `turbo prune --scope=a`
+    // @todo refactor with other package managers
+    suite(
+      `${npmClient} + turbo prune ${
+        options.cwd ? " from " + options.cwd : ""
+      } `,
+      async () => {
+        const pruneCommandOutput = getCommandOutputAsArray(
+          repo.turbo("prune", ["--scope=a"], options)
+        );
+        assert.fixture(pruneCommandOutput[1], " - Added a");
+        assert.fixture(pruneCommandOutput[2], " - Added b");
+
+        let files = [];
+        assert.not.throws(() => {
+          files = repo.globbySync("out/**/*", {
+            cwd: options.cwd ?? repo.root,
+          });
+        }, `Could not read generated \`out\` directory after \`turbo prune\``);
+        const expected = [
+          "out/package.json",
+          "out/turbo.json",
+          "out/yarn.lock",
+          "out/packages/a/build.js",
+          "out/packages/a/lint.js",
+          "out/packages/a/package.json",
+          "out/packages/a/test.js",
+          "out/packages/b/build.js",
+          "out/packages/b/lint.js",
+          "out/packages/b/package.json",
+          "out/packages/b/test.js",
+        ];
+        for (const file of expected) {
+          assert.ok(
+            files.includes(file),
+            `Expected file ${file} to be generated`
+          );
+        }
+        const install = repo.run("install", ["--frozen-lockfile"], {
+          cwd: options.cwd
+            ? path.join(options.cwd, "out")
+            : path.join(repo.root, "out"),
+        });
+        assert.is(
+          install.exitCode,
+          0,
+          "Expected yarn install --frozen-lockfile to succeed"
+        );
+      }
+    );
+  }
+}
+
+type PackageManager = "yarn" | "pnpm" | "npm" | "berry";
+
+// getLockfileForPackageManager returns the name of the lockfile for the given package manager
+function getLockfileForPackageManager(ws: PackageManager) {
+  switch (ws) {
+    case "yarn":
+      return "yarn.lock";
+    case "pnpm":
+      return "pnpm-lock.yaml";
+    case "npm":
+      return "package-lock.json";
+    case "berry":
+      return "yarn.lock";
+    default:
+      throw new Error(`Unknown package manager: ${ws}`);
+  }
 }
 
 function getCommandOutputAsArray(
@@ -202,7 +280,7 @@ function getHashFromOutput(lines: string[], taskId: string): string {
 
 function getCachedDirForHash(repo: Monorepo, hash: string): string {
   return path.join(
-    repo.subdir ? repo.subdir + "/" : "",
+    repo.subdir ? repo.subdir : ".",
     "node_modules",
     ".cache",
     "turbo",

cli/scripts/monorepo.ts

@@ -1,5 +1,6 @@
 import execa from "execa";
 import fsNormal from "fs";
+import globby from "globby";
 import fs from "fs-extra";
 import os from "os";
 import path from "path";
@@ -349,6 +350,24 @@ fs.copyFileSync(
     return fs.readFileSync(path.join(this.root, filepath), "utf-8");
   }
 
+  readdirSync(filepath) {
+    return fs.readdirSync(path.join(this.root, filepath), "utf-8");
+  }
+
+  globbySync(
+    patterns: string | readonly string[],
+    options?: globby.GlobbyOptions
+  ) {
+    return globby.sync(patterns, { cwd: this.root, ...options });
+  }
+
+  async globby(
+    patterns: string | readonly string[],
+    options?: globby.GlobbyOptions
+  ) {
+    return await globby(patterns, { cwd: this.root, ...options });
+  }
+
   cleanup() {
     fs.rmSync(this.root, { recursive: true });
   }

cli/scripts/package.json

@@ -9,6 +9,7 @@
     "faker": "^5.1.0",
     "find-up": "^5.0.0",
     "fs-extra": "^9.1.0",
+    "globby": "11.1.0",
     "ngraph.generators": "^19.3.0",
     "parse-package-name": "^0.1.0",
     "shelljs": "^0.8.4",