feat(ci): add github util to get previous commit sha

.github/workflows/ci.yml

@@ -184,7 +184,6 @@ jobs:
         env:
           GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }}
           GITHUB_PUSH_BASE_SHA: ${{ github.event.base }}
-          GITHUB_PULL_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
           GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
@@ -204,6 +203,7 @@ jobs:
         with:
           args: .
         env:
+          GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }}
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
           GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
 
@@ -222,6 +222,7 @@ jobs:
         with:
           args: .
         env:
+          GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }}
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
           GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
 

ggshield/cmd/iac/scan/ci.py

@@ -11,8 +11,12 @@
     update_context,
 )
 from ggshield.core.config import Config
-from ggshield.core.git_hooks.ci import collect_commit_range_from_ci_env
+from ggshield.core.git_hooks.ci import (
+    collect_commit_range_from_ci_env,
+    get_previous_commit_from_ci_env,
+)
 from ggshield.core.text_utils import display_warning
+from ggshield.core.utils import EMPTY_SHA
 
 
 @click.command()
@@ -44,13 +48,30 @@ def scan_ci_cmd(
         return display_iac_scan_all_result(ctx, directory, result)
 
     config: Config = ctx.obj["config"]
-    commit_list, _ = collect_commit_range_from_ci_env(config.user_config.verbose)
-    reference, current_ref = commit_list[0], commit_list[-1]
 
-    # If we failed to fetch a current reference, we set it to HEAD
-    if len(commit_list) < 2 or not current_ref:
+    new_commits, _ = collect_commit_range_from_ci_env(config.user_config.verbose)
+    current_ref = new_commits[-1]
+    reference, _ = get_previous_commit_from_ci_env(config.user_config.verbose)
+
+    # If no sha provided, consider repo was empty before
+    if reference == EMPTY_SHA or reference == "null":
+        reference = None
+
+    # If we fail to fetch current state, we set it to HEAD
+    if current_ref == EMPTY_SHA or current_ref == "null":
         current_ref = "HEAD"
 
+    if config.user_config.verbose:
+        if len(new_commits) > 0:
+            click.echo("List of new commits: ")
+            for commit in new_commits:
+                click.echo(f"   {commit}")
+
+        click.echo(
+            f"Comparing commit {current_ref} to commit {reference}",
+            err=True,
+        )
+
     result = iac_scan_diff(
         ctx, directory, reference, current_ref=current_ref, include_staged=True
     )

ggshield/cmd/sca/scan/ci.py

@@ -18,8 +18,12 @@
 )
 from ggshield.core.config import Config
 from ggshield.core.errors import handle_exception
-from ggshield.core.git_hooks.ci import collect_commit_range_from_ci_env
+from ggshield.core.git_hooks.ci import (
+    collect_commit_range_from_ci_env,
+    get_previous_commit_from_ci_env,
+)
 from ggshield.core.git_shell import check_git_dir
+from ggshield.core.utils import EMPTY_SHA
 from ggshield.sca.collection.collection import (
     SCAScanAllVulnerabilityCollection,
     SCAScanDiffVulnerabilityCollection,
@@ -64,15 +68,39 @@ def scan_ci_cmd(
             return output_handler.process_scan_all_result(scan)
 
         check_git_dir()
-        commit_count = len(
-            collect_commit_range_from_ci_env(config.user_config.verbose)[0]
-        )
+
+        new_commits, _ = collect_commit_range_from_ci_env(config.user_config.verbose)
+        previous_commit, _ = get_previous_commit_from_ci_env(config.user_config.verbose)
+
+        # If no sha provided, consider repo was empty before
+        if previous_commit == EMPTY_SHA or previous_commit == "null":
+            previous_commit = None
+
+        # If we fail to fetch current state, we set it to HEAD
+        if len(new_commits) == 0:
+            current_state = "HEAD"
+        else:
+            current_state = new_commits[-1]
+
+        if current_state == EMPTY_SHA or current_state == "null":
+            current_state = "HEAD"
+
         if config.user_config.verbose:
-            click.echo(f"Commits to scan: {commit_count}", err=True)
+            if len(new_commits) > 0:
+                click.echo("List of new commits: ")
+                for commit in new_commits:
+                    click.echo(f"   {commit}")
+
+            click.echo(
+                f"Comparing commit {current_state} to commit {previous_commit}",
+                err=True,
+            )
+
         result = sca_scan_diff(
             ctx=ctx,
             directory=directory,
-            previous_ref=f"HEAD~{commit_count}" if commit_count > 0 else "HEAD",
+            previous_ref=previous_commit,
+            current_ref=current_state,
         )
         scan = SCAScanDiffVulnerabilityCollection(id=str(directory), result=result)
         return output_handler.process_scan_diff_result(scan)

ggshield/core/git_hooks/ci/__init__.py

@@ -0,0 +1,10 @@
+from .commit_range import collect_commit_range_from_ci_env
+from .previous_commit import get_previous_commit_from_ci_env
+from .supported_ci import SupportedCI
+
+
+__all__ = [
+    "SupportedCI",
+    "collect_commit_range_from_ci_env",
+    "get_previous_commit_from_ci_env",
+]

ggshield/core/git_hooks/ci.py → ggshield/core/git_hooks/ci/commit_range.py

@@ -1,5 +1,4 @@
 import os
-from enum import Enum
 from typing import List, Tuple
 
 import click
@@ -8,16 +7,12 @@
 from ggshield.core.git_shell import get_list_commit_SHA
 from ggshield.core.utils import EMPTY_SHA
 
-
-class SupportedCI(Enum):
-    GITLAB = "GITLAB"
-    TRAVIS = "TRAVIS"
-    CIRCLECI = "CIRCLECI"
-    JENKINS = "JENKINS HOME"
-    GITHUB = "GITHUB ACTIONS"
-    BITBUCKET = "BITBUCKET PIPELINES"
-    DRONE = "DRONE"
-    AZURE = "AZURE PIPELINES"
+from .previous_commit import (
+    github_pull_request_previous_commit_sha,
+    github_push_previous_commit_sha,
+    gitlab_push_previous_commit_sha,
+)
+from .supported_ci import SupportedCI
 
 
 def collect_commit_range_from_ci_env(
@@ -161,7 +156,7 @@ def circle_ci_range(verbose: bool) -> List[str]:  # pragma: no cover
 
 
 def gitlab_ci_range(verbose: bool) -> List[str]:  # pragma: no cover
-    before_sha = os.getenv("CI_COMMIT_BEFORE_SHA")
+    before_sha = gitlab_push_previous_commit_sha()
     commit_sha = os.getenv("CI_COMMIT_SHA", "HEAD")
     merge_request_target_branch = os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
 
@@ -197,9 +192,9 @@ def gitlab_ci_range(verbose: bool) -> List[str]:  # pragma: no cover
 
 
 def github_actions_range(verbose: bool) -> List[str]:  # pragma: no cover
-    push_before_sha = os.getenv("GITHUB_PUSH_BEFORE_SHA")
+    push_before_sha = github_push_previous_commit_sha()
     push_base_sha = os.getenv("GITHUB_PUSH_BASE_SHA")
-    pull_req_base_sha = os.getenv("GITHUB_PULL_BASE_SHA")
+    pull_req_base_sha = github_pull_request_previous_commit_sha()
     default_branch = os.getenv("GITHUB_DEFAULT_BRANCH")
     head_sha = os.getenv("GITHUB_SHA", "HEAD")
 
@@ -213,13 +208,13 @@ def github_actions_range(verbose: bool) -> List[str]:  # pragma: no cover
             err=True,
         )
 
-    if push_before_sha and push_before_sha != EMPTY_SHA:
-        commit_list = get_list_commit_SHA(f"{push_before_sha}...")
+    if pull_req_base_sha and pull_req_base_sha != EMPTY_SHA:
+        commit_list = get_list_commit_SHA(f"{pull_req_base_sha}..")
         if commit_list:
             return commit_list
 
-    if pull_req_base_sha and pull_req_base_sha != EMPTY_SHA:
-        commit_list = get_list_commit_SHA(f"{pull_req_base_sha}..")
+    if push_before_sha and push_before_sha != EMPTY_SHA:
+        commit_list = get_list_commit_SHA(f"{push_before_sha}...")
         if commit_list:
             return commit_list
 

ggshield/core/git_hooks/ci/previous_commit.py

@@ -0,0 +1,163 @@
+import os
+from typing import Optional, Tuple
+
+import click
+
+from ggshield.core.errors import UnexpectedError
+from ggshield.core.git_shell import get_list_commit_SHA
+from ggshield.core.utils import EMPTY_SHA
+
+from .supported_ci import SupportedCI
+
+
+def get_previous_commit_from_ci_env(
+    verbose: bool,
+) -> Tuple[str, SupportedCI]:
+    """
+    Returns the previous HEAD sha of the targeted branch.
+    Returns EMPTY_SHA if there was no commit before.
+    """
+    if os.getenv("GITLAB_CI"):
+        ci_mode = SupportedCI.GITLAB
+        commit_sha = gitlav_previous_commit_sha(verbose)
+
+    elif os.getenv("GITHUB_ACTIONS"):
+        ci_mode = SupportedCI.GITHUB
+        commit_sha = github_previous_commit_sha(verbose)
+
+    elif os.getenv("TRAVIS"):
+        ci_mode = SupportedCI.TRAVIS
+        raise NotImplementedError("Not implemented for Travis.")
+
+    elif os.getenv("JENKINS_HOME") or os.getenv("JENKINS_URL"):
+        ci_mode = SupportedCI.JENKINS
+        raise NotImplementedError("Not implemented for Jenkins.")
+
+    elif os.getenv("CIRCLECI"):
+        ci_mode = SupportedCI.CIRCLECI
+        raise NotImplementedError("Not implemented for CircleCI.")
+
+    elif os.getenv("BITBUCKET_COMMIT"):
+        ci_mode = SupportedCI.BITBUCKET
+        raise NotImplementedError("Not implemented for BitBucket.")
+
+    elif os.getenv("DRONE"):
+        ci_mode = SupportedCI.DRONE
+        raise NotImplementedError("Not implemented for Drone.")
+
+    elif os.getenv("BUILD_BUILDID"):
+        ci_mode = SupportedCI.AZURE
+        raise NotImplementedError("Not implemented for Azure.")
+
+    else:
+        raise UnexpectedError(
+            f"Current CI is not detected or supported."
+            f" Supported CIs: {', '.join([ci.value for ci in SupportedCI])}."
+        )
+    return (commit_sha, ci_mode)
+
+
+def github_previous_commit_sha(verbose: bool) -> str:
+    push_before_sha = github_push_previous_commit_sha()
+    pull_req_base_sha = github_pull_request_previous_commit_sha()
+
+    if verbose:
+        click.echo(
+            f"github_push_before_sha: {push_before_sha}\n"
+            f"github_pull_base_sha: {pull_req_base_sha}\n",
+            err=True,
+        )
+
+    # This one has to be checked before the push one
+    # because this one is only populated in case of PR
+    # whereas push_before_sha can be populated in PR in case of
+    # push force event in a PR
+    if pull_req_base_sha:
+        return pull_req_base_sha
+
+    if push_before_sha:
+        return push_before_sha
+
+    raise UnexpectedError(
+        "Unable to get previous commit. Please submit an issue with the following info:\n"
+        "  Repository URL: <Fill if public>\n"
+        f"github_push_before_sha: {push_before_sha}\n"
+        f"github_pull_base_sha: {pull_req_base_sha}\n"
+    )
+
+
+def github_push_previous_commit_sha() -> Optional[str]:
+    return os.getenv("GITHUB_PUSH_BEFORE_SHA")
+
+
+def github_pull_request_previous_commit_sha() -> Optional[str]:
+    targeted_branch = os.getenv("GITHUB_BASE_REF")
+
+    # Not in a pull request workflow
+    if targeted_branch is None:
+        return None
+
+    # The branch is not directly available in CI env
+    # We need to get commits through remotes
+    last_target_commit = get_list_commit_SHA(
+        f"remotes/origin/{targeted_branch}", max_count=1
+    )
+
+    # Unable to find a commit on this branch
+    # Consider it empty
+    if len(last_target_commit) == 0:
+        return None
+
+    return last_target_commit[0]
+
+
+def gitlav_previous_commit_sha(verbose: bool) -> str:
+    push_before_sha = gitlab_push_previous_commit_sha()
+    merge_req_base_sha = gitlab_merge_request_previous_commit_sha()
+
+    if verbose:
+        click.echo(
+            f"gitlab_push_before_sha: {push_before_sha}\n"
+            f"gitlab_merge_base_sha: {merge_req_base_sha}\n",
+            err=True,
+        )
+
+    # Always EMPTY_SHA in MR pipeline according with
+    # https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+    if push_before_sha and push_before_sha is not EMPTY_SHA:
+        return push_before_sha
+
+    if merge_req_base_sha:
+        return merge_req_base_sha
+
+    raise UnexpectedError(
+        "Unable to get previous commit. Please submit an issue with the following info:\n"
+        "  Repository URL: <Fill if public>\n"
+        f"gitlab_push_before_sha: {push_before_sha}\n"
+        f"gitlab_pull_base_sha: {merge_req_base_sha}\n"
+    )
+
+
+def gitlab_push_previous_commit_sha() -> Optional[str]:
+    return os.getenv("CI_COMMIT_BEFORE_SHA")
+
+
+def gitlab_merge_request_previous_commit_sha() -> Optional[str]:
+    targeted_branch = os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
+
+    # Not in a pull request workflow
+    if targeted_branch is None:
+        return None
+
+    # The branch is not directly available in CI env
+    # We need to get commits through remotes
+    last_target_commit = get_list_commit_SHA(
+        f"remotes/origin/{targeted_branch}", max_count=1
+    )
+
+    # Unable to find a commit on this branch
+    # Consider it empty
+    if len(last_target_commit) == 0:
+        return None
+
+    return last_target_commit[0]

ggshield/core/git_hooks/ci/supported_ci.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class SupportedCI(Enum):
+    GITLAB = "GITLAB"
+    TRAVIS = "TRAVIS"
+    CIRCLECI = "CIRCLECI"
+    JENKINS = "JENKINS HOME"
+    GITHUB = "GITHUB ACTIONS"
+    BITBUCKET = "BITBUCKET PIPELINES"
+    DRONE = "DRONE"
+    AZURE = "AZURE PIPELINES"