diff --git a/assets/deepen_shallow_clone_until_ref_is_found b/assets/deepen_shallow_clone_until_ref_is_found deleted file mode 100755 index 6c49ed38..00000000 --- a/assets/deepen_shallow_clone_until_ref_is_found +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# vim: set ft=sh - -set -e - -readonly max_depth=128 - -readonly ref="$1" -readonly tagflag="$2" - -# the concourse_git_resource__depth variable is exported by the 'in' script - -# A shallow clone may not contain the Git commit $ref: -# 1. The depth of the shallow clone is measured backwards from the latest -# commit on the given head (master or branch), and in the meantime there may -# have been more than $depth commits pushed on top of our $ref. -# 2. If there's a path filter (`paths`/`ignore_paths`), then there may be more -# than $depth such commits pushed to the head (master or branch) on top of -# $ref that are not affecting the filtered paths. -# -# In either case we try to deepen the shallow clone until we find $ref, reach -# the max depth of the repo, or give up after a given depth and resort to deep -# clone. -if [ "$concourse_git_resource__depth" -gt 0 ]; then - depth="$concourse_git_resource__depth" - git_dir="$(git rev-parse --git-dir)" - readonly git_dir - - while ! git checkout -q "$ref" &>/dev/null; do - # once the depth of a shallow clone reaches the max depth of the origin - # repo, Git silenty turns it into a deep clone - if [ ! -e "$git_dir"/shallow ]; then - echo "Reached max depth of the origin repo while deepening the shallow clone, it's a deep clone now" - break - fi - - echo "Could not find ref ${ref} in a shallow clone of depth ${depth}" - - (( depth *= 2 )) - - if [ "$depth" -gt "$max_depth" ]; then - echo "Reached depth threshold ${max_depth}, falling back to deep clone..." - git fetch --unshallow origin $tagflag - - break - fi - - echo "Deepening the shallow clone to depth ${depth}..." - git fetch --depth "$depth" origin $tagflag - done -fi diff --git a/assets/deepen_shallow_clone_until_ref_is_found_then_check_out b/assets/deepen_shallow_clone_until_ref_is_found_then_check_out new file mode 100755 index 00000000..881ff743 --- /dev/null +++ b/assets/deepen_shallow_clone_until_ref_is_found_then_check_out @@ -0,0 +1,50 @@ +#!/bin/bash +# vim: set ft=sh + +set -e + +readonly max_depth=128 + +declare depth="$1" +readonly ref="$2" +readonly tagflag="$3" + +# A shallow clone may not contain the Git commit $ref: +# 1. The depth of the shallow clone is measured backwards from the latest +# commit on the given head (master or branch), and in the meantime there may +# have been more than $depth commits pushed on top of our $ref. +# 2. If there's a path filter (`paths`/`ignore_paths`), then there may be more +# than $depth such commits pushed to the head (master or branch) on top of +# $ref that are not affecting the filtered paths. +# +# In either case we try to deepen the shallow clone until we find $ref, reach +# the max depth of the repo, or give up after a given depth and resort to deep +# clone. + +git_dir="$(git rev-parse --git-dir)" +readonly git_dir + +while ! git checkout -q "$ref" &>/dev/null; do + # once the depth of a shallow clone reaches the max depth of the origin + # repo, Git silenty turns it into a deep clone + if [ ! -e "$git_dir"/shallow ]; then + echo "Reached max depth of the origin repo while deepening the shallow clone, it's a deep clone now" + break + fi + + echo "Could not find ref ${ref} in a shallow clone of depth ${depth}" + + (( depth *= 2 )) + + if [ "$depth" -gt "$max_depth" ]; then + echo "Reached depth threshold ${max_depth}, falling back to deep clone..." + git fetch --unshallow origin $tagflag + + break + fi + + echo "Deepening the shallow clone to depth ${depth}..." + git fetch --depth "$depth" origin $tagflag +done + +git checkout -q "$ref" diff --git a/assets/in b/assets/in index 11780ac5..49edd56d 100755 --- a/assets/in +++ b/assets/in @@ -79,10 +79,11 @@ cd $destination git fetch origin refs/notes/*:refs/notes/* $tagflag -concourse_git_resource__depth="$depth" \ - "$bin_dir"/deepen_shallow_clone_until_ref_is_found "$ref" "$tagflag" - -git checkout -q $ref +if [ "$depth" -gt 0 ]; then + "$bin_dir"/deepen_shallow_clone_until_ref_is_found_then_check_out "$depth" "$ref" "$tagflag" +else + git checkout -q "$ref" +fi invalid_key() { echo "Invalid GPG key in: ${commit_verification_keys}" @@ -141,23 +142,26 @@ if [ "$submodules" != "none" ]; then fi for submodule in $submodules; do - # remember submodule update config - update_conf_was_set=1 - if update_conf="$(git config --get "submodule.${submodule}.update")"; then - update_conf_was_set=0 + if [ "$depth" -gt 0 ]; then + # remember submodule update config + update_conf_was_set=0 + if update_conf="$(git config --get "submodule.${submodule}.update")"; then + update_conf_was_set=1 + fi + + # temporarily set to our shallow clone deepening shell script + git config "submodule.${submodule}.update" "!$bin_dir/deepen_shallow_clone_until_ref_is_found_then_check_out $depth" fi - # temporarily set to our shallow clone deepening shell script - git config "submodule.${submodule}.update" "!$bin_dir"/deepen_shallow_clone_until_ref_is_found - - concourse_git_resource__depth="$depth" \ - git submodule update --no-fetch $depthflag $submodule_parameters $submodule + git submodule update --no-fetch $depthflag $submodule_parameters $submodule - # restore submodule update config - if [ "$update_conf_was_set" != 0 ]; then - git config "submodule.${submodule}.update" "$update_conf" - else - git config --unset "submodule.${submodule}.update" + if [ "$depth" -gt 0 ]; then + # restore submodule update config + if [ "$update_conf_was_set" != 0 ]; then + git config "submodule.${submodule}.update" "$update_conf" + else + git config --unset "submodule.${submodule}.update" + fi fi done fi diff --git a/test/get.sh b/test/get.sh index 9363e7c7..61878351 100755 --- a/test/get.sh +++ b/test/get.sh @@ -145,7 +145,7 @@ it_returns_list_of_tags_in_metadata() { " } -it_can_use_submodlues_without_perl_warning() { +it_can_use_submodules_without_perl_warning() { local repo=$(init_repo_with_submodule | cut -d "," -f1) local dest=$TMPDIR/destination @@ -250,25 +250,51 @@ it_honors_the_depth_flag_for_submodules() { local submodule_folder=$(echo $repo_with_submodule_info | cut -d "," -f2) local submodule_name=$(basename $submodule_folder) local project_last_commit_id=$(git -C $project_folder rev-parse HEAD) + local submodule_last_commit_id=$(git -C $project_folder/$submodule_name rev-parse HEAD) - local dest_all=$TMPDIR/destination_all - local dest_one=$TMPDIR/destination_one + local dest_all_depth0=$TMPDIR/destination_all_depth0 get_uri_with_submodules_all \ - "file://"$project_folder 1 $dest_all | jq -e " + "file://"$project_folder 0 $dest_all_depth0 | jq -e " .version == {ref: $(echo $project_last_commit_id | jq -R .)} " - test "$(git -C $project_folder rev-parse HEAD)" = $project_last_commit_id - test "$(git -C $dest_all/$submodule_name rev-list --all --count)" = 1 + test "$(git -C $dest_all_depth0 rev-parse HEAD)" = $project_last_commit_id + test "$(git -C $dest_all_depth0/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id + test "$(git -C $dest_all_depth0/$submodule_name rev-list --all --count)" \> 1 + + local dest_one_depth0=$TMPDIR/destination_one_depth0 get_uri_with_submodules_at_depth \ - "file://"$project_folder 1 $submodule_name $dest_one | jq -e " + "file://"$project_folder 0 $submodule_name $dest_one_depth0 | jq -e " .version == {ref: $(echo $project_last_commit_id | jq -R .)} " - test "$(git -C $project_folder rev-parse HEAD)" = $project_last_commit_id - test "$(git -C $dest_one/$submodule_name rev-list --all --count)" = 1 + test "$(git -C $dest_one_depth0 rev-parse HEAD)" = $project_last_commit_id + test "$(git -C $dest_all_depth0/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id + test "$(git -C $dest_one_depth0/$submodule_name rev-list --all --count)" \> 1 + + local dest_all_depth1=$TMPDIR/destination_all_depth1 + + get_uri_with_submodules_all \ + "file://"$project_folder 1 $dest_all_depth1 | jq -e " + .version == {ref: $(echo $project_last_commit_id | jq -R .)} + " + + test "$(git -C $dest_all_depth1 rev-parse HEAD)" = $project_last_commit_id + test "$(git -C $dest_all_depth1/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id + test "$(git -C $dest_all_depth1/$submodule_name rev-list --all --count)" = 1 + + local dest_one_depth1=$TMPDIR/destination_one_depth1 + + get_uri_with_submodules_at_depth \ + "file://"$project_folder 1 $submodule_name $dest_one_depth1 | jq -e " + .version == {ref: $(echo $project_last_commit_id | jq -R .)} + " + + test "$(git -C $dest_one_depth1 rev-parse HEAD)" = $project_last_commit_id + test "$(git -C $dest_all_depth1/$submodule_name rev-parse HEAD)" = $submodule_last_commit_id + test "$(git -C $dest_one_depth1/$submodule_name rev-list --all --count)" = 1 } it_falls_back_to_deep_clone_of_submodule_if_ref_not_found() { @@ -309,6 +335,26 @@ it_falls_back_to_deep_clone_of_submodule_if_ref_not_found() { grep "Reached depth threshold 128, falling back to deep clone..." <$TMPDIR/stderr } +it_fails_if_the_ref_cannot_be_found_while_deepening_a_submodule() { + local repo_with_submodule_info=$(init_repo_with_submodule) + local main_repo=${repo_with_submodule_info%,*} + local submodule_repo=${repo_with_submodule_info#*,} + local submodule_name=${submodule_repo##*/} + local submodule_last_commit_id=$(git -C "$submodule_repo" rev-parse HEAD) + + git -C "$submodule_repo" reset --hard HEAD^ >/dev/null + + local dest=$TMPDIR/destination + + output=$(get_uri_with_submodules_all "file://$main_repo" 1 $dest 2>&1) \ + && exit_code=$? || exit_code=$? + + echo $output $exit_code + test "${exit_code}" \!= 0 + echo "${output}" | grep "Reached max depth of the origin repo while deepening the shallow clone, it's a deep clone now" + echo "${output}" | grep "fatal: reference is not a tree: $submodule_last_commit_id" +} + # the submodule incremental deepening depends on overwriting the update method # of the submodule, so we should test if it's properly restored it_preserves_the_submodule_update_method() { @@ -317,7 +363,6 @@ it_preserves_the_submodule_update_method() { local submodule_repo=${repo_with_submodule_info#*,} local submodule_name=${submodule_repo##*/} local main_repo_last_commit_id=$(git -C $main_repo rev-parse HEAD) - local submodule_repo_last_commit_id=$(git -C $submodule_repo rev-parse HEAD) local dest=$TMPDIR/destination @@ -332,9 +377,14 @@ it_preserves_the_submodule_update_method() { rm -rf "$dest" - get_uri_with_submodules_and_git_config \ - "file://$main_repo" $dest "submodule.${submodule_name}.update" "merge" \ - | jq -e " + git -C "$main_repo" config --file .gitmodules --replace-all "submodule.${submodule_name}.update" merge + git -C "$main_repo" add .gitmodules + git -C "$main_repo" commit -m 'Add .gitmodules' >/dev/null + + local main_repo_last_commit_id=$(git -C $main_repo rev-parse HEAD) + local submodule_repo_last_commit_id=$(git -C $submodule_repo rev-parse HEAD) + + get_uri_with_submodules_all "file://$main_repo" 1 $dest | jq -e " .version == {ref: $(echo $main_repo_last_commit_id | jq -R .)} " @@ -673,13 +723,14 @@ run it_omits_empty_branch_in_metadata run it_returns_branch_in_metadata run it_omits_empty_tags_in_metadata run it_returns_list_of_tags_in_metadata -run it_can_use_submodlues_without_perl_warning +run it_can_use_submodules_without_perl_warning run it_honors_the_depth_flag run it_can_get_from_url_at_depth_at_ref run it_falls_back_to_deep_clone_if_ref_not_found run it_does_not_enter_an_infinite_loop_if_the_ref_cannot_be_found_and_depth_is_set run it_honors_the_depth_flag_for_submodules run it_falls_back_to_deep_clone_of_submodule_if_ref_not_found +run it_fails_if_the_ref_cannot_be_found_while_deepening_a_submodule run it_preserves_the_submodule_update_method run it_honors_the_parameter_flags_for_submodules run it_can_get_and_set_git_config diff --git a/test/helpers.sh b/test/helpers.sh index 9ed2a0ff..329583d4 100644 --- a/test/helpers.sh +++ b/test/helpers.sh @@ -489,23 +489,6 @@ get_uri_with_submodules_all() { }" | ${resource_dir}/in "$3" | tee /dev/stderr } -get_uri_with_submodules_and_git_config() { - jq -n "{ - source: { - uri: $(echo $1 | jq -R .), - git_config: [ - { - name: $(echo $3 | jq -R .), - value: $(echo $4 | jq -R .) - } - ] - }, - params: { - submodules: \"all\", - } - }" | ${resource_dir}/in "$2" | tee /dev/stderr -} - get_uri_with_submodules_and_parameter_remote() { jq -n "{ source: {