Automatically deepen clones to find correct commits (#495)

* Automatically deepen clones to find correct commits

This fixes #494 - where we would get the wrong commit
that added a file if we were operating in a shallow
repository that didn't contain the actual commit -
by modifying `getCommitThatAddsFile` so that it
automatically deepens a shallow clone until it finds the
_real_ commit that added a file.

* Switch to a bulk getCommitsThatAddFiles

Instead of running multiple getCommitThatAddsFile calls in parallel
and trying to figure out how to do shallow-clone-deepening safely
in such an environment, we move to a bulk getCommitsThatAddFiles
call that itself retrieves commit information for all the files
simultaneously and safely deepens the clone as necessary.

* Make `isRepoShallow` work on older versions of Git

Changesets' CircleCI build is using Git 2.11, which
doesn't support `rev-parse --is-shallow-repository`.

Testing for the existence of `.git/shallow` is a good
workaround for older versions of Git, but in case
of future `.git` folder changes, we'll use `--is-shallow-repository`
if present.

* Move logic to appropriate path

Move the recalculation of the `remaining` commits to the only control path
that actually uses it to clarify the logic.

* Tweak changesets

Co-authored-by: Mateusz Burzyński <mateuszburzynski@gmail.com>

.changeset/six-roses-hammer.md

@@ -0,0 +1,5 @@
+---
+"@changesets/git": minor
+---
+
+Automatically deepen shallow clones in order to determine the correct commit at which changesets were added.

.changeset/some-other-changeset.md

@@ -0,0 +1,6 @@
+---
+"@changesets/git": minor
+---
+
+Deprecate the `getCommitThatAddsFile` function. It's replaced with a bulk `getCommitsThatAddFiles` operation which will safely deepen a
+shallow repo whilst processing multiple filenames simultaneously.

.changeset/yet-another-core-changeset.md

@@ -0,0 +1,5 @@
+---
+"@changesets/cli": minor
+---
+
+Automatically deepen shallow clones in order to determine the correct commit at which changesets were added. This helps Git-based changelog generators to always link to the correct commit. From now on it's not required to configure `fetch-depth: 0` for your `actions/checkout` when using [Changesets GitHub action](https://github.com/changesets/action).

packages/apply-release-plan/src/index.ts

@@ -20,21 +20,41 @@ import versionPackage from "./version-package";
 import createVersionCommit from "./createVersionCommit";
 import getChangelogEntry from "./get-changelog-entry";
 
-async function getCommitThatAddsChangeset(changesetId: string, cwd: string) {
-  let commit = await git.getCommitThatAddsFile(
-    `.changeset/${changesetId}.md`,
-    cwd
-  );
-  if (commit) {
-    return commit;
+function stringDefined(s: string | undefined): s is string {
+  return !!s;
+}
+async function getCommitsThatAddChangesets(
+  changesetIds: string[],
+  cwd: string
+) {
+  const paths = changesetIds.map(id => `.changeset/${id}.md`);
+  const commits = await git.getCommitsThatAddFiles(paths, cwd);
+
+  if (commits.every(stringDefined)) {
+    // We have commits for all files
+    return commits;
   }
-  let commitForOldChangeset = await git.getCommitThatAddsFile(
-    `.changeset/${changesetId}/changes.json`,
+
+  // Some files didn't exist. Try legacy filenames instead
+  const missingIds = changesetIds
+    .map((id, i) => (commits[i] ? undefined : id))
+    .filter(stringDefined);
+
+  const legacyPaths = missingIds.map(id => `.changeset/${id}/changes.json`);
+  const commitsForLegacyPaths = await git.getCommitsThatAddFiles(
+    legacyPaths,
     cwd
   );
-  if (commitForOldChangeset) {
-    return commitForOldChangeset;
-  }
+
+  // Fill in the blanks in the array of commits
+  changesetIds.forEach((id, i) => {
+    if (!commits[i]) {
+      const missingIndex = missingIds.indexOf(id);
+      commits[i] = commitsForLegacyPaths[missingIndex];
+    }
+  });
+
+  return commits;
 }
 
 export default async function applyReleasePlan(
@@ -200,12 +220,14 @@ async function getNewChangelogEntry(
     }
   }
 
-  let moddedChangesets = await Promise.all(
-    changesets.map(async cs => ({
-      ...cs,
-      commit: await getCommitThatAddsChangeset(cs.id, cwd)
-    }))
+  let commits = await getCommitsThatAddChangesets(
+    changesets.map(cs => cs.id),
+    cwd
   );
+  let moddedChangesets = changesets.map((cs, i) => ({
+    ...cs,
+    commit: commits[i]
+  }));
 
   return Promise.all(
     releasesWithPackage.map(async release => {

packages/cli/src/commands/version/version.test.ts

@@ -47,6 +47,8 @@ git.add.mockImplementation(() => Promise.resolve(true));
 // @ts-ignore
 git.commit.mockImplementation(() => Promise.resolve(true));
 // @ts-ignore
+git.getCommitsThatAddFiles.mockImplementation(() => Promise.resolve([]));
+// @ts-ignore
 git.tag.mockImplementation(() => Promise.resolve(true));
 
 const simpleChangeset: NewChangeset = {

packages/git/package.json

@@ -15,6 +15,7 @@
     "spawndamnit": "^2.0.0"
   },
   "devDependencies": {
+    "file-url": "^3.0.0",
     "fixturez": "^1.1.0"
   }
 }

packages/git/src/index.test.ts

@@ -1,8 +1,10 @@
 import fixtures from "fixturez";
 import spawn from "spawndamnit";
+import fileUrl from "file-url";
 
 import {
   getCommitThatAddsFile,
+  getCommitsThatAddFiles,
   getChangedFilesSince,
   add,
   commit,
@@ -19,6 +21,16 @@ async function getCurrentCommit(cwd: string) {
   return cmd.stdout.toString().trim();
 }
 
+async function getCurrentCommitShort(cwd: string) {
+  const cmd = await spawn("git", ["rev-parse", "--short", "HEAD"], { cwd });
+  return cmd.stdout.toString().trim();
+}
+
+async function getCommitCount(cwd: string) {
+  const cmd = await spawn("git", ["rev-list", "--count", "HEAD"], { cwd });
+  return parseInt(cmd.stdout.toString(), 10);
+}
+
 describe("git", () => {
   let cwd: string;
   beforeEach(async () => {
@@ -185,20 +197,156 @@ describe("git", () => {
     });
   });
 
-  describe("getCommitThatAddsFile", () => {
+  describe("getCommitsThatAddFiles", () => {
     it("should commit a file and get the hash of that commit", async () => {
       await add("packages/pkg-b/package.json", cwd);
       await commit("added packageB package.json", cwd);
-      const head = await spawn("git", ["rev-parse", "--short", "HEAD"], {
+      const headSha = await getCurrentCommitShort(cwd);
+
+      const commitHash = await getCommitsThatAddFiles(
+        ["packages/pkg-b/package.json"],
         cwd
-      });
+      );
+
+      expect(commitHash).toEqual([headSha]);
+    });
+
+    // We have replaced the single-file version with a multi-file version
+    // which will correctly run all the file retrieves in parallel, safely
+    // deepening a shallow clone as necessary.  We've deprecated the
+    // single-file version and can remove it in a major release.
+    it("exposes a deprecated single-file call", async () => {
+      await add("packages/pkg-b/package.json", cwd);
+      await commit("added packageB package.json", cwd);
+      const headSha = await getCurrentCommitShort(cwd);
 
       const commitHash = await getCommitThatAddsFile(
         "packages/pkg-b/package.json",
         cwd
       );
 
-      expect(commitHash).toEqual(head.stdout.toString().trim());
+      expect(commitHash).toEqual(headSha);
+    });
+
+    describe("with shallow clone", () => {
+      // We will add these three well-known files
+      // over multiple commits, then test looking up
+      // the commits at which they were added.
+      const file1 = ".changeset/quick-lions-devour.md";
+      const file2 = "packages/pkg-a/index.js";
+      const file3 = "packages/pkg-b/index.js";
+
+      // Roughly how many commits will the deepening algorithm
+      // deepen each time?  We use this to set up test data to
+      // check that the deepens the clone but doesn't need to *fully* unshallow
+      // the clone.
+      const shallowCloneDeepeningAmount = 50;
+
+      /**
+       * Creates a number of empty commits; this is useful to ensure
+       * that a particular commit doesn't make it into a shallow clone.
+       */
+      async function createDummyCommits(count: number) {
+        for (let i = 0; i < count; i++) {
+          await commit("dummy commit", cwd);
+        }
+      }
+
+      async function addFileAndCommit(file: string) {
+        await add(file, cwd);
+        await commit(`add file ${file}`, cwd);
+        const commitSha = await getCurrentCommitShort(cwd);
+        return commitSha;
+      }
+
+      async function createShallowClone(depth: number): Promise<string> {
+        // Make a 1-commit-deep shallow clone of this repo
+        const cloneDir = f.temp();
+        await spawn(
+          "git",
+          // Note: a file:// URL is needed in order to make a shallow clone of
+          // a local repo
+          ["clone", "--depth", depth.toString(), fileUrl(cwd), "."],
+          {
+            cwd: cloneDir
+          }
+        );
+        return cloneDir;
+      }
+
+      it("reads the SHA of a file-add without deepening if commit already included in the shallow clone", async () => {
+        // We create a repo that we shallow-clone;
+        // the commit we're going to scan for is the latest commit,
+        // so will be in the shallow clone immediately without deepening
+        await createDummyCommits(10);
+        const originalCommit = await addFileAndCommit(file1);
+
+        const clone = await createShallowClone(5);
+
+        // This file was added in the head commit, so will definitely be in our
+        // 1-commit clone.
+        const commits = await getCommitsThatAddFiles([file1], clone);
+        expect(commits).toEqual([originalCommit]);
+
+        // We should not need to have deepened the clone for this
+        expect(await getCommitCount(clone)).toEqual(5);
+      });
+
+      it("reads the SHA of a file-add even if not already included in the shallow clone", async () => {
+        // We're going to create a repo where the commit we're looking for isn't
+        // in the shallow clone, so we'll need to deepen it to locate it.
+        await createDummyCommits((shallowCloneDeepeningAmount * 2) / 3);
+        const originalCommit = await addFileAndCommit(file2);
+        await createDummyCommits((shallowCloneDeepeningAmount * 2) / 3);
+
+        const clone = await createShallowClone(5);
+
+        // Finding this commit will require deepening the clone until it appears.
+        const commit = await getCommitThatAddsFile(file2, clone);
+        expect(commit).toEqual(originalCommit);
+
+        // It should not have completely unshallowed the clone; just enough.
+        const originalRepoDepth = await getCommitCount(cwd);
+        expect(await getCommitCount(clone)).toBeGreaterThan(5);
+        expect(await getCommitCount(clone)).toBeLessThan(originalRepoDepth);
+      });
+
+      it("reads the SHA of a file-add even if the first commit of a repo", async () => {
+        // Finding this commit will require deepening the clone right to the start
+        // of the repo history, and coping with a commit that has no parent.
+        const originalCommit = await addFileAndCommit(file3);
+        await createDummyCommits(shallowCloneDeepeningAmount * 2);
+        const clone = await createShallowClone(5);
+
+        // Finding this commit will require fully deepening the repo
+        const commit = await getCommitThatAddsFile(file3, clone);
+        expect(commit).toEqual(originalCommit);
+
+        // We should have fully deepened
+        const originalRepoDepth = await getCommitCount(cwd);
+        expect(await getCommitCount(clone)).toEqual(originalRepoDepth);
+      });
+
+      it("can return SHAs for multiple files including return blanks for missing files", async () => {
+        // We want to ensure that we can retrieve SHAs for multiple files at the same time,
+        // and also that requesting missing files doesn't affect the location of commits
+        // for the files that succeed.
+        await createDummyCommits(shallowCloneDeepeningAmount);
+        const originalCommit1 = await addFileAndCommit(file1);
+        await createDummyCommits(shallowCloneDeepeningAmount);
+        const originalCommit2 = await addFileAndCommit(file2);
+
+        const nonExistentFile = "this-file-does-not-exist";
+
+        const clone = await createShallowClone(5);
+
+        const commits = await getCommitsThatAddFiles(
+          [file1, nonExistentFile, file2],
+          clone
+        );
+
+        expect(commits).toEqual([originalCommit1, undefined, originalCommit2]);
+      });
     });
   });