Fix submodule deepening

assets/deepen_shallow_clone_until_ref_is_found_then_check_out

@@ -0,0 +1,50 @@
+#!/bin/bash
+# vim: set ft=sh
+
+set -e
+
+readonly max_depth=128
+
+declare depth="$1"
+readonly ref="$2"
+readonly tagflag="$3"
+
+# A shallow clone may not contain the Git commit $ref:
+# 1. The depth of the shallow clone is measured backwards from the latest
+#    commit on the given head (master or branch), and in the meantime there may
+#    have been more than $depth commits pushed on top of our $ref.
+# 2. If there's a path filter (`paths`/`ignore_paths`), then there may be more
+#    than $depth such commits pushed to the head (master or branch) on top of
+#    $ref that are not affecting the filtered paths.
+#
+# In either case we try to deepen the shallow clone until we find $ref, reach
+# the max depth of the repo, or give up after a given depth and resort to deep
+# clone.
+
+git_dir="$(git rev-parse --git-dir)"
+readonly git_dir
+
+while ! git checkout -q "$ref" &>/dev/null; do
+  # once the depth of a shallow clone reaches the max depth of the origin
+  # repo, Git silenty turns it into a deep clone
+  if [ ! -e "$git_dir"/shallow ]; then
+    echo "Reached max depth of the origin repo while deepening the shallow clone, it's a deep clone now"
+    break
+  fi
+
+  echo "Could not find ref ${ref} in a shallow clone of depth ${depth}"
+
+  (( depth *= 2 ))
+
+  if [ "$depth" -gt "$max_depth" ]; then
+    echo "Reached depth threshold ${max_depth}, falling back to deep clone..."
+    git fetch --unshallow origin $tagflag
+
+    break
+  fi
+
+  echo "Deepening the shallow clone to depth ${depth}..."
+  git fetch --depth "$depth" origin $tagflag
+done
+
+git checkout -q "$ref"

assets/in

@@ -79,10 +79,11 @@ cd $destination
 
 git fetch origin refs/notes/*:refs/notes/* $tagflag
 
-concourse_git_resource__depth="$depth" \
-  "$bin_dir"/deepen_shallow_clone_until_ref_is_found "$ref" "$tagflag"
-
-git checkout -q $ref
+if [ "$depth" -gt 0 ]; then
+  "$bin_dir"/deepen_shallow_clone_until_ref_is_found_then_check_out "$depth" "$ref" "$tagflag"
+else
+  git checkout -q "$ref"
+fi
 
 invalid_key() {
   echo "Invalid GPG key in: ${commit_verification_keys}"
@@ -141,23 +142,26 @@ if [ "$submodules" != "none" ]; then
   fi
 
   for submodule in $submodules; do
-    # remember submodule update config
-    update_conf_was_set=1
-    if update_conf="$(git config --get "submodule.${submodule}.update")"; then
-	    update_conf_was_set=0
+    if [ "$depth" -gt 0 ]; then
+      # remember submodule update config
+      update_conf_was_set=0
+      if update_conf="$(git config --get "submodule.${submodule}.update")"; then
+        update_conf_was_set=1
+      fi
+
+      # temporarily set to our shallow clone deepening shell script
+      git config "submodule.${submodule}.update" "!$bin_dir/deepen_shallow_clone_until_ref_is_found_then_check_out $depth"
     fi
 
-    # temporarily set to our shallow clone deepening shell script
-    git config "submodule.${submodule}.update" "!$bin_dir"/deepen_shallow_clone_until_ref_is_found
-
-    concourse_git_resource__depth="$depth" \
-      git submodule update --no-fetch $depthflag $submodule_parameters $submodule
+    git submodule update --no-fetch $depthflag $submodule_parameters $submodule
 
-    # restore submodule update config
-    if [ "$update_conf_was_set" != 0 ]; then
-      git config "submodule.${submodule}.update" "$update_conf"
-    else
-      git config --unset "submodule.${submodule}.update"
+    if [ "$depth" -gt 0 ]; then
+      # restore submodule update config
+      if [ "$update_conf_was_set" != 0 ]; then
+        git config "submodule.${submodule}.update" "$update_conf"
+      else
+        git config --unset "submodule.${submodule}.update"
+      fi
     fi
   done
 fi

test/get.sh

@@ -145,7 +145,7 @@ it_returns_list_of_tags_in_metadata() {
   "
 }
 
-it_can_use_submodlues_without_perl_warning() {
+it_can_use_submodules_without_perl_warning() {
   local repo=$(init_repo_with_submodule | cut -d "," -f1)
   local dest=$TMPDIR/destination
 
@@ -250,25 +250,51 @@ it_honors_the_depth_flag_for_submodules() {
   local submodule_folder=$(echo $repo_with_submodule_info | cut -d "," -f2)
   local submodule_name=$(basename $submodule_folder)
   local project_last_commit_id=$(git -C $project_folder rev-parse HEAD)
+  local submodule_last_commit_id=$(git -C $project_folder/$submodule_name rev-parse HEAD)
 
-  local dest_all=$TMPDIR/destination_all
-  local dest_one=$TMPDIR/destination_one
+  local dest_all_depth0=$TMPDIR/destination_all_depth0
 
   get_uri_with_submodules_all \
-  "file://"$project_folder 1 $dest_all |  jq -e "
+  "file://"$project_folder 0 $dest_all_depth0 |  jq -e "
     .version == {ref: $(echo $project_last_commit_id | jq -R .)}
   "
 
-  test "$(git -C $project_folder rev-parse HEAD)" = $project_last_commit_id
-  test "$(git -C $dest_all/$submodule_name rev-list --all --count)" = 1
+  test "$(git -C $dest_all_depth0 rev-parse HEAD)" = $project_last_commit_id
+  test "$(git -C $dest_all_depth0/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id
+  test "$(git -C $dest_all_depth0/$submodule_name rev-list --all --count)" \> 1
+
+  local dest_one_depth0=$TMPDIR/destination_one_depth0
 
   get_uri_with_submodules_at_depth \
-  "file://"$project_folder 1 $submodule_name $dest_one |  jq -e "
+  "file://"$project_folder 0 $submodule_name $dest_one_depth0 |  jq -e "
     .version == {ref: $(echo $project_last_commit_id | jq -R .)}
   "
 
-  test "$(git -C $project_folder rev-parse HEAD)" = $project_last_commit_id
-  test "$(git -C $dest_one/$submodule_name rev-list --all --count)" = 1
+  test "$(git -C $dest_one_depth0 rev-parse HEAD)" = $project_last_commit_id
+  test "$(git -C $dest_all_depth0/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id
+  test "$(git -C $dest_one_depth0/$submodule_name rev-list --all --count)" \> 1
+
+  local dest_all_depth1=$TMPDIR/destination_all_depth1
+
+  get_uri_with_submodules_all \
+  "file://"$project_folder 1 $dest_all_depth1 |  jq -e "
+    .version == {ref: $(echo $project_last_commit_id | jq -R .)}
+  "
+
+  test "$(git -C $dest_all_depth1 rev-parse HEAD)" = $project_last_commit_id
+  test "$(git -C $dest_all_depth1/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id
+  test "$(git -C $dest_all_depth1/$submodule_name rev-list --all --count)" = 1
+
+  local dest_one_depth1=$TMPDIR/destination_one_depth1
+
+  get_uri_with_submodules_at_depth \
+  "file://"$project_folder 1 $submodule_name $dest_one_depth1 |  jq -e "
+    .version == {ref: $(echo $project_last_commit_id | jq -R .)}
+  "
+
+  test "$(git -C $dest_one_depth1 rev-parse HEAD)" = $project_last_commit_id
+  test "$(git -C $dest_all_depth1/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id
+  test "$(git -C $dest_one_depth1/$submodule_name rev-list --all --count)" = 1
 }
 
 it_falls_back_to_deep_clone_of_submodule_if_ref_not_found() {
@@ -309,6 +335,26 @@ it_falls_back_to_deep_clone_of_submodule_if_ref_not_found() {
   grep "Reached depth threshold 128, falling back to deep clone..." <$TMPDIR/stderr
 }
 
+it_fails_if_the_ref_cannot_be_found_while_deepening_a_submodule() {
+  local repo_with_submodule_info=$(init_repo_with_submodule)
+  local main_repo=${repo_with_submodule_info%,*}
+  local submodule_repo=${repo_with_submodule_info#*,}
+  local submodule_name=${submodule_repo##*/}
+  local submodule_last_commit_id=$(git -C "$submodule_repo" rev-parse HEAD)
+
+  git -C "$submodule_repo" reset --hard HEAD^ >/dev/null
+
+  local dest=$TMPDIR/destination
+
+  output=$(get_uri_with_submodules_all "file://$main_repo" 1 $dest 2>&1) \
+    && exit_code=$? || exit_code=$?
+
+  echo $output $exit_code
+  test "${exit_code}" \!= 0
+  echo "${output}" | grep "Reached max depth of the origin repo while deepening the shallow clone, it's a deep clone now"
+  echo "${output}" | grep "fatal: reference is not a tree: $submodule_last_commit_id"
+}
+
 # the submodule incremental deepening depends on overwriting the update method
 # of the submodule, so we should test if it's properly restored
 it_preserves_the_submodule_update_method() {
@@ -317,7 +363,6 @@ it_preserves_the_submodule_update_method() {
   local submodule_repo=${repo_with_submodule_info#*,}
   local submodule_name=${submodule_repo##*/}
   local main_repo_last_commit_id=$(git -C $main_repo rev-parse HEAD)
-  local submodule_repo_last_commit_id=$(git -C $submodule_repo rev-parse HEAD)
 
   local dest=$TMPDIR/destination
 
@@ -332,9 +377,14 @@ it_preserves_the_submodule_update_method() {
   rm -rf "$dest"
 
 
-  get_uri_with_submodules_and_git_config \
-    "file://$main_repo" $dest "submodule.${submodule_name}.update" "merge" \
-    | jq -e "
+  git -C "$main_repo" config --file .gitmodules --replace-all "submodule.${submodule_name}.update" merge
+  git -C "$main_repo" add .gitmodules
+  git -C "$main_repo" commit -m 'Add .gitmodules' >/dev/null
+
+  local main_repo_last_commit_id=$(git -C $main_repo rev-parse HEAD)
+  local submodule_repo_last_commit_id=$(git -C $submodule_repo rev-parse HEAD)
+
+  get_uri_with_submodules_all "file://$main_repo" 1 $dest | jq -e "
     .version == {ref: $(echo $main_repo_last_commit_id | jq -R .)}
   "
 
@@ -673,13 +723,14 @@ run it_omits_empty_branch_in_metadata
 run it_returns_branch_in_metadata
 run it_omits_empty_tags_in_metadata
 run it_returns_list_of_tags_in_metadata
-run it_can_use_submodlues_without_perl_warning
+run it_can_use_submodules_without_perl_warning
 run it_honors_the_depth_flag
 run it_can_get_from_url_at_depth_at_ref
 run it_falls_back_to_deep_clone_if_ref_not_found
 run it_does_not_enter_an_infinite_loop_if_the_ref_cannot_be_found_and_depth_is_set
 run it_honors_the_depth_flag_for_submodules
 run it_falls_back_to_deep_clone_of_submodule_if_ref_not_found
+run it_fails_if_the_ref_cannot_be_found_while_deepening_a_submodule
 run it_preserves_the_submodule_update_method
 run it_honors_the_parameter_flags_for_submodules
 run it_can_get_and_set_git_config

test/helpers.sh

@@ -489,23 +489,6 @@ get_uri_with_submodules_all() {
   }" | ${resource_dir}/in "$3" | tee /dev/stderr
 }
 
-get_uri_with_submodules_and_git_config() {
-  jq -n "{
-    source: {
-      uri: $(echo $1 | jq -R .),
-      git_config: [
-        {
-          name: $(echo $3 | jq -R .),
-          value: $(echo $4 | jq -R .)
-        }
-      ]
-    },
-    params: {
-      submodules: \"all\",
-    }
-  }" | ${resource_dir}/in "$2" | tee /dev/stderr
-}
-
 get_uri_with_submodules_and_parameter_remote() {
   jq -n "{
     source: {