From a62caf6dfdc8bd5f5bd579d85b48817cd980145e Mon Sep 17 00:00:00 2001 From: Richard Godden Date: Mon, 19 Nov 2018 01:06:33 +0000 Subject: [PATCH 1/5] searches all branches when tag filter is provided and branch isnt Signed-off-by: Richard Godden --- assets/check | 16 ++++++++++++---- assets/in | 5 +++-- test/check.sh | 47 ++++++++++++++++++++++++++++++++++++++++++++++- test/get.sh | 2 +- test/helpers.sh | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 8 deletions(-) diff --git a/assets/check b/assets/check index 9dba1e82..40d469f7 100755 --- a/assets/check +++ b/assets/check @@ -33,17 +33,22 @@ configure_git_global "${git_config_payload}" destination=$TMPDIR/git-resource-repo-cache +tagflag="" +if [ -d $tag_filter ]; then + tagflag="--tags" +fi + if [ -d $destination ]; then cd $destination - git fetch + git fetch $tagflag git reset --hard FETCH_HEAD else branchflag="" if [ -n "$branch" ]; then - branchflag="--branch $branch" + branchflag="--single-branch --branch $branch" fi - git clone --single-branch $uri $branchflag $destination + git clone $uri $branchflag $destination cd $destination fi @@ -72,8 +77,11 @@ fi if [ -n "$tag_filter" ]; then { - if [ -n "$ref" ]; then + if [ -n "$ref" ] && [ -n "$branch" ]; then git tag --list "$tag_filter" --sort=creatordate --contains $ref + elif [ -n "$ref" ]; then + ref_for_search=$(sed 's#/#\\/#g' <<< $ref) + git tag --list "$tag_filter" --list --sort=creatordate | sed -ne "/$ref_for_search/,$ p" else git tag --list "$tag_filter" --sort=creatordate | tail -1 fi diff --git a/assets/in b/assets/in index 26fc6d70..82169e5e 100755 --- a/assets/in +++ b/assets/in @@ -54,7 +54,7 @@ fi branchflag="" if [ -n "$branch" ]; then - branchflag="--branch $branch" + branchflag="--single-branch --branch $branch" fi depthflag="" @@ -62,11 +62,12 @@ if test "$depth" -gt 0 2> /dev/null; then depthflag="--depth $depth" fi -git clone --single-branch $depthflag $uri $branchflag $destination +git clone $depthflag $uri $branchflag $destination cd $destination git fetch origin refs/notes/*:refs/notes/* +git fetch --tags # A shallow clone may not contain the Git commit $ref: # 1. The depth of the shallow clone is measured backwards from the latest diff --git a/test/check.sh b/test/check.sh index 469ca5b6..a589b8df 100755 --- a/test/check.sh +++ b/test/check.sh @@ -19,7 +19,7 @@ it_can_check_from_head_only_fetching_single_branch() { local cachedir="$TMPDIR/git-resource-repo-cache" - check_uri $repo | jq -e " + check_uri_with_branch $repo "master" | jq -e " . == [{ref: $(echo $ref | jq -R .)}] " @@ -451,6 +451,49 @@ it_can_check_with_tag_filter_with_cursor() { local ref12=$(make_annotated_tag $repo "3.0-production" "tag 6") local ref13=$(make_commit $repo) + x=$(check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging") + check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging" | jq -e ' + . == [{ref: "2.0-staging"}, {ref: "3.0-staging"}] + ' +} + +it_can_check_with_tag_filter_over_all_branches() { + local repo=$(init_repo) + local ref1=$(make_commit_to_branch $repo branch-a) + local ref2=$(make_annotated_tag $repo "1.0-staging" "a tag") + local ref3=$(make_commit_to_branch $repo branch-a) + local ref4=$(make_annotated_tag $repo "1.0-production" "another tag") + local ref5=$(make_commit_to_branch $repo branch-a) + local ref6=$(make_annotated_tag $repo "2.0-staging" "tag 3") + local ref7=$(make_commit_to_branch $repo branch-a) + local ref8=$(make_annotated_tag $repo "2.0-production" "tag 4") + local ref9=$(make_commit_to_branch $repo branch-a) + local ref10=$(make_annotated_tag $repo "3.0-staging" "tag 5") + local ref11=$(make_commit_to_branch $repo branch-a) + local ref12=$(make_annotated_tag $repo "3.0-production" "tag 6") + local ref13=$(make_commit_to_branch $repo branch-a) + + check_uri_with_tag_filter $repo "*-staging" | jq -e ' + . == [{ref: "3.0-staging"}] + ' +} + +it_can_check_with_tag_filter_over_all_branches_with_cursor() { + local repo=$(init_repo) + local ref1=$(make_commit_to_branch $repo branch-a) + local ref2=$(make_annotated_tag $repo "1.0-staging" "a tag") + local ref3=$(make_commit_to_branch $repo branch-a) + local ref4=$(make_annotated_tag $repo "1.0-production" "another tag") + local ref5=$(make_commit_to_branch $repo branch-a) + local ref6=$(make_annotated_tag $repo "2.0-staging" "tag 3") + local ref7=$(make_commit_to_branch $repo branch-a) + local ref8=$(make_annotated_tag $repo "2.0-production" "tag 4") + local ref9=$(make_commit_to_branch $repo branch-a) + local ref10=$(make_annotated_tag $repo "3.0-staging" "tag 5") + local ref11=$(make_commit_to_branch $repo branch-a) + local ref12=$(make_annotated_tag $repo "3.0-production" "tag 6") + local ref13=$(make_commit_to_branch $repo branch-a) + check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging" | jq -e ' . == [{ref: "2.0-staging"}, {ref: "3.0-staging"}] ' @@ -507,6 +550,8 @@ run it_clears_netrc_even_after_errors run it_can_check_empty_commits run it_can_check_with_tag_filter run it_can_check_with_tag_filter_with_cursor +run it_can_check_with_tag_filter_over_all_branches +run it_can_check_with_tag_filter_over_all_branches_with_cursor run it_can_check_with_tag_filter_with_bogus_ref run it_can_check_from_head_only_fetching_single_branch run it_can_check_and_set_git_config diff --git a/test/get.sh b/test/get.sh index 06d8d6c2..a405b04a 100755 --- a/test/get.sh +++ b/test/get.sh @@ -70,7 +70,7 @@ it_can_get_from_url_only_single_branch() { local ref=$(make_commit $repo) local dest=$TMPDIR/destination - get_uri $repo $dest | jq -e " + get_uri_with_branch $repo "master" $dest | jq -e " .version == {ref: $(echo $ref | jq -R .)} " diff --git a/test/helpers.sh b/test/helpers.sh index 67749ef0..329583d4 100644 --- a/test/helpers.sh +++ b/test/helpers.sh @@ -195,6 +195,15 @@ check_uri() { }" | ${resource_dir}/check | tee /dev/stderr } +check_uri_with_branch() { + jq -n "{ + source: { + uri: $(echo $1 | jq -R .), + branch: $(echo $2 | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + get_initial_ref() { local repo=$1 @@ -337,6 +346,19 @@ check_uri_with_tag_filter() { }" | ${resource_dir}/check | tee /dev/stderr } +check_uri_with_tag_filter_given_branch() { + local uri=$1 + local tag_filter=$2 + local branch=$3 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + tag_filter: $(echo $tag_filter | jq -R .), + branch: $(echo $branch | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + check_uri_with_tag_filter_from() { local uri=$1 local tag_filter=$2 @@ -394,6 +416,18 @@ get_uri() { }" | ${resource_dir}/in "$2" | tee /dev/stderr } +get_uri_with_branch() { + jq -n "{ + source: { + uri: $(echo $1 | jq -R .), + branch: $(echo $2 | jq -R .), + }, + params: { + short_ref_format: \"test-%s\" + } + }" | ${resource_dir}/in "$3" | tee /dev/stderr +} + get_uri_with_git_crypt_key() { local git_crypt_key_path=$(git_crypt_fixture_key_path) local git_crypt_key_base64_encoded=$(cat $git_crypt_key_path | base64) From 61f5bfae954e1a22b774813093cc060b0e8d9e14 Mon Sep 17 00:00:00 2001 From: rgodden Date: Tue, 20 Nov 2018 17:11:36 +0000 Subject: [PATCH 2/5] clarified sed commands --- assets/check | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/assets/check b/assets/check index 40d469f7..a0dd1324 100755 --- a/assets/check +++ b/assets/check @@ -75,13 +75,21 @@ if [ "$skip_ci_disabled" != "true" ]; then ci_skip="--grep \\[ci\\sskip\\] --grep \\[skip\\sci\\] --invert-grep" fi +replace_escape_chars() { + sed -e 's/[]\/$*.^[]/\\&/g' <<< $1 +} + +lines_including_and_after() { + local escaped_string=$(replace_escape_chars $1) + sed -ne "/$escaped_string/,$ p" +} + if [ -n "$tag_filter" ]; then { if [ -n "$ref" ] && [ -n "$branch" ]; then git tag --list "$tag_filter" --sort=creatordate --contains $ref elif [ -n "$ref" ]; then - ref_for_search=$(sed 's#/#\\/#g' <<< $ref) - git tag --list "$tag_filter" --list --sort=creatordate | sed -ne "/$ref_for_search/,$ p" + git tag --list "$tag_filter" --list --sort=creatordate | lines_including_and_after $ref else git tag --list "$tag_filter" --sort=creatordate | tail -1 fi From b8f845b350e20eac65b7ed799577f990ac78b0dd Mon Sep 17 00:00:00 2001 From: rgodden Date: Tue, 20 Nov 2018 17:13:51 +0000 Subject: [PATCH 3/5] added clone --single-branch back in for performance --- assets/check | 4 ++-- assets/in | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/assets/check b/assets/check index a0dd1324..36ba3db5 100755 --- a/assets/check +++ b/assets/check @@ -45,10 +45,10 @@ if [ -d $destination ]; then else branchflag="" if [ -n "$branch" ]; then - branchflag="--single-branch --branch $branch" + branchflag="--branch $branch" fi - git clone $uri $branchflag $destination + git clone --single-branch $uri $branchflag $destination $tagflag cd $destination fi diff --git a/assets/in b/assets/in index 82169e5e..ad42621e 100755 --- a/assets/in +++ b/assets/in @@ -39,6 +39,7 @@ submodule_recursive=$(jq -r '(.params.submodule_recursive // true)' < $payload) submodule_remote=$(jq -r '(.params.submodule_remote // false)' < $payload) commit_verification_key_ids=$(jq -r '(.source.commit_verification_key_ids // [])[]' < $payload) commit_verification_keys=$(jq -r '(.source.commit_verification_keys // [])[]' < $payload) +tag_filter=$(jq -r '.source.tag_filter // ""' < $payload) gpg_keyserver=$(jq -r '.source.gpg_keyserver // "hkp://ipv4.pool.sks-keyservers.net/"' < $payload) disable_git_lfs=$(jq -r '(.params.disable_git_lfs // false)' < $payload) clean_tags=$(jq -r '(.params.clean_tags // false)' < $payload) @@ -62,12 +63,16 @@ if test "$depth" -gt 0 2> /dev/null; then depthflag="--depth $depth" fi -git clone $depthflag $uri $branchflag $destination +tagflag="" +if [ -d $tag_filter ]; then + tagflag="--tags" +fi + +git clone $depthflag $uri $branchflag $destination $tagflag cd $destination git fetch origin refs/notes/*:refs/notes/* -git fetch --tags # A shallow clone may not contain the Git commit $ref: # 1. The depth of the shallow clone is measured backwards from the latest @@ -98,13 +103,13 @@ if [ "$depth" -gt 0 ]; then if [ "$d" -gt $max_depth ]; then echo "Reached depth threshold ${max_depth}, falling back to deep clone..." - git fetch --unshallow origin + git fetch --unshallow origin $tagflag break fi echo "Deepening the shallow clone to depth ${d}..." - git fetch --depth $d origin + git fetch --depth $d origin $tagflag done fi From 02c4547a943687f0791058de57a5d8f50dd4334f Mon Sep 17 00:00:00 2001 From: rgodden Date: Tue, 20 Nov 2018 17:20:28 +0000 Subject: [PATCH 4/5] fetching tags at all depths for now --- assets/in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/in b/assets/in index ad42621e..ab7593c8 100755 --- a/assets/in +++ b/assets/in @@ -72,7 +72,7 @@ git clone $depthflag $uri $branchflag $destination $tagflag cd $destination -git fetch origin refs/notes/*:refs/notes/* +git fetch origin refs/notes/*:refs/notes/* $tagflag # A shallow clone may not contain the Git commit $ref: # 1. The depth of the shallow clone is measured backwards from the latest @@ -103,13 +103,13 @@ if [ "$depth" -gt 0 ]; then if [ "$d" -gt $max_depth ]; then echo "Reached depth threshold ${max_depth}, falling back to deep clone..." - git fetch --unshallow origin $tagflag + git fetch --unshallow origin break fi echo "Deepening the shallow clone to depth ${d}..." - git fetch --depth $d origin $tagflag + git fetch --depth $d origin done fi From db3bdbc52dbc9d8df1d29260b6bec807c6a7c635 Mon Sep 17 00:00:00 2001 From: rgodden Date: Tue, 20 Nov 2018 20:22:06 +0000 Subject: [PATCH 5/5] fetching tags when depening clone --- assets/deepen_shallow_clone_until_ref_is_found | 5 +++-- assets/in | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/assets/deepen_shallow_clone_until_ref_is_found b/assets/deepen_shallow_clone_until_ref_is_found index 1b576d52..6c49ed38 100755 --- a/assets/deepen_shallow_clone_until_ref_is_found +++ b/assets/deepen_shallow_clone_until_ref_is_found @@ -6,6 +6,7 @@ set -e readonly max_depth=128 readonly ref="$1" +readonly tagflag="$2" # the concourse_git_resource__depth variable is exported by the 'in' script @@ -39,12 +40,12 @@ if [ "$concourse_git_resource__depth" -gt 0 ]; then if [ "$depth" -gt "$max_depth" ]; then echo "Reached depth threshold ${max_depth}, falling back to deep clone..." - git fetch --unshallow origin + git fetch --unshallow origin $tagflag break fi echo "Deepening the shallow clone to depth ${depth}..." - git fetch --depth "$depth" origin + git fetch --depth "$depth" origin $tagflag done fi diff --git a/assets/in b/assets/in index 445b3efb..39406711 100755 --- a/assets/in +++ b/assets/in @@ -80,7 +80,7 @@ cd $destination git fetch origin refs/notes/*:refs/notes/* $tagflag concourse_git_resource__depth="$depth" \ - "$bin_dir"/deepen_shallow_clone_until_ref_is_found "$ref" + "$bin_dir"/deepen_shallow_clone_until_ref_is_found "$ref" "$tagflag" git checkout -q $ref