/
98-source-repositories
executable file
·268 lines (233 loc) · 12 KB
/
98-source-repositories
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
#!/bin/bash
# dib-lint: disable=safe_sudo
if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then
set -x
fi
set -eu
set -o pipefail
# Don't provide any source repositories in environments where they are not allowed
if [ -n "${NO_SOURCE_REPOSITORIES:-}" ]; then
exit 0
fi
# If the old cache exists, move it to the new name
function make_new_cache(){
local OLD_CACHE_BASE=$1
local CACHE_BASE=$2
# If the old cache name exists, move it to the new cache name
if [ -e "$OLD_CACHE_BASE" ] ; then
if [ ! -e "$CACHE_BASE" ] ; then
mv -n $OLD_CACHE_BASE $CACHE_BASE
else
echo "Not replacing new cache location with old cache"
fi
fi
}
# Gets repositories or individual files listed in the a repository file
# and places them in the specified destination path.
# The format of the repository file is one or more lines matching
# <name> <type> <destination> <location> [<ref>]
function get_repos_for_element(){
local REPO_SOURCES=$1
local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url
local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$"
# this can be a rather long list (e.g. caching ~1000 openstack
# repos), so we make a rough count for progress display
local COUNT=1
local REPO_SOURCES_COUNT=$(grep -v '^$\|^\s*\#' $REPO_SOURCES | wc -l)
while read line; do
# temporarily turn off globbing '*' (since it may be used as
# the REPOREF for tarballs) and expand variables
set -f; line=$(eval echo $line); set +f
# ignore blank lines and lines beginning in '#'
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
# rough progress counter
printf "(%04d / %04d)\n" ${COUNT} ${REPO_SOURCES_COUNT}
COUNT=$(( COUNT + 1))
if [[ "$line" =~ $REGEX ]] ; then
local REPONAME=${BASH_REMATCH[1]}
local REPOTYPE=${BASH_REMATCH[2]}
local REPOPATH=${BASH_REMATCH[3]}
local REPOLOCATION=${BASH_REMATCH[4]}
local REPO_ORIG_LOCATION=$REPOLOCATION
if [ $REPONAME = "tar" -a -z "${BASH_REMATCH[5]:-}" ] ; then
echo "Warning: Default tarball REPOREF of '*' is deprecated; do not rely on it."
fi
# Default of '*' for tar repositories is deprecated; do not rely on it.
local REPOREF_DEFAULT_TAR=*
local REPOREF_DEFAULT_GIT=master
local REPOREF_LOOKUP_DEFAULT=REPOREF_DEFAULT_${REPOTYPE^^}
local REPOREF=${BASH_REMATCH[5]:-${!REPOREF_LOOKUP_DEFAULT:-}}
local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH
local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST)
# REPOTYPE can be overridden with DIB_REPOTYPE_{name}
local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//[^A-Za-z0-9]/_}
REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE}
# REPOLOCATION can be overridden with DIB_REPOLOCATION_{name}
local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//[^A-Za-z0-9]/_}
REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION}
# REPOREF can be overridden with DIB_REPOREF_{name}
local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//[^A-Za-z0-9]/_}
REPOREF=${!REPOREF_OVERRIDE:-$REPOREF}
# Determine a unique cache path for this repo
CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
OLD_CACHE_PATH=${CACHE_BASE}/${CACHE_NAME}
# Add the repo name to the sha1sum for readability
CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME}
CACHE_PATH=${CACHE_BASE}/$CACHE_NAME
make_new_cache $OLD_CACHE_PATH $CACHE_PATH
# Return if install type is not source
local INSTALL_TYPE_VAR=DIB_INSTALLTYPE_${REPONAME//[^A-Za-z0-9]/_}
local INSTALL_TYPE=${!INSTALL_TYPE_VAR:-$DIB_DEFAULT_INSTALLTYPE}
if [ ! $INSTALL_TYPE = "source" ]; then
echo "$REPONAME install type not set to source"
continue
fi
case $REPOTYPE in
git)
if [ -z "${!REPOLOCATION_OVERRIDE:-""}" -a -n "${DIB_GITREPOBASE:-""}" ] ; then
# Transform the current repo base to the new one
local NEW_REPOLOCATION=$(echo $REPOLOCATION |\
sed "s,^[^:]\+://[^/]\+/\(~[^/]\+\)\?\(.*\)$,${DIB_GITREPOBASE}\2,g")
echo "Transformed ${REPOLOCATION} to ${NEW_REPOLOCATION}"
REPOLOCATION=$NEW_REPOLOCATION
# Also update the cache location
CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME
fi
sudo mkdir -p $REPO_SUB_DIRECTORY
if [ ! -e "$CACHE_PATH" ] ; then
echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
git clone -q $REPOLOCATION $CACHE_PATH.tmp
mv ${CACHE_PATH}{.tmp,}
fi
if [ "$REPOREF" != "*" ] ; then
HAS_REF=$(git -C ${CACHE_PATH} name-rev $REPOREF 2>/dev/null || true)
fi
if [ -z "$DIB_OFFLINE" -o -z "${HAS_REF:-}" ] ; then
echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
# Copy named refs (which might be outside the usual heads
# pattern) - e.g. gerrit
if [ "$REPOREF" == "*" ] || ! git -C ${CACHE_PATH} fetch -q --prune --update-head-ok $REPOLOCATION \
+${REPOREF}:${REPOREF} ; then
# Copy all heads from the remote repository - this permits
# using a SHA1 object reference so long as the object
# reference is reachable from one of the heads. git does
# not permit arbitrary sha fetching from remote servers.
# This is a separate fetch to the prior one as the prior
# one will fail when REPOREF is a SHA1.
git -C ${CACHE_PATH} fetch -q --prune --update-head-ok $REPOLOCATION \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
fi
# Ensure that we have a reference to the revision.
if [ "$REPOREF" != "*" ] ; then
if ! git -C ${CACHE_PATH} rev-parse -q --verify $REPOREF^{commit} > /dev/null; then
echo "Failed to find reference to $REPOREF"
exit 1
fi
fi
fi
echo "Cloning from $REPONAME cache and applying ref $REPOREF"
# If the local dir is already used, see if the pertinent details differ
if [[ -d $REPO_DEST ]]; then
DESIRED="$(sudo git -C ${REPO_DEST} config remote.origin.url)"
if [[ "$CACHE_PATH" != "$DESIRED" ]]; then
echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2
exit 1
elif [[ "$REPOREF" != "*" ]]; then
# When we first clone we create a branch naming what we fetched
# that must match, or we are asking for two different references from the
# same repo, which is an error
if ! sudo git -C ${REPO_DEST} rev-parse fetch_$REPOREF; then
echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2
exit 1
fi
fi
else
sudo git clone -q $CACHE_PATH $REPO_DEST
if [[ "$REPOREF" == "*" ]]; then
sudo git -C ${REPO_DEST} fetch -q --prune --update-head-ok $CACHE_PATH \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
git_sha=$(sudo git -C ${REPO_DEST} rev-parse HEAD)
else
sudo git -C ${REPO_DEST} fetch -q $CACHE_PATH $REPOREF:fetch_$REPOREF
sudo git -C ${REPO_DEST} reset --hard FETCH_HEAD
# Get the sha in use
git_sha=$(sudo git -C ${REPO_DEST} rev-parse FETCH_HEAD)
fi
# Write the sha being used into the source-repositories manifest
echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST
fi
;;
tar)
# The top level directory of the tarball mightn't have a fixed name i.e.
# it could contain version numbers etc... so we write it to a tmpdir
# for inspection before transferring the contents into the target directory
local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d)
if [ -n "$CACHE_PATH" ] ; then
echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH"
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
$CACHE_URL $REPOLOCATION $CACHE_PATH
fi
tar -C $tmpdir -xzf $CACHE_PATH
else
echo "Fetching $REPONAME tarball from $REPOLOCATION"
curl -f $REPOLOCATION | tar -C $tmpdir -xzf -
fi
sudo mkdir -p $REPO_DEST
# A REPOREF of '.' will select the entire contents of the tarball,
# while '*' will select only the contents of its subdirectories.
sudo rsync -a --remove-source-files $tmpdir/$REPOREF/. $REPO_DEST
rm -rf $tmpdir
;;
file)
sudo mkdir -p $REPO_SUB_DIRECTORY
if [ -n "$CACHE_PATH" ] ; then
echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH"
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
$CACHE_URL $REPOLOCATION $CACHE_PATH
fi
sudo cp $CACHE_PATH $REPO_DEST
else
echo "Fetching $REPONAME file from $REPOLOCATION"
sudo curl -f $REPOLOCATION -o $REPO_DEST
fi
;;
*)
echo "Unsupported repository type: $REPOTYPE"
return 1
;;
esac
# Capture the in-instance repository path for later review / other
# elements (like a pypi dependency cache).
echo "$REPOPATH" | sudo tee -a $TMP_MOUNT_PATH/etc/dib-source-repositories > /dev/null
else
echo "Couldn't parse '$line' as a source repository"
return 1
fi
done < $REPO_SOURCES
}
CACHE_BASE=$DIB_IMAGE_CACHE/source-repositories
OLD_CACHE_BASE=$DIB_IMAGE_CACHE/repository-sources
make_new_cache $OLD_CACHE_BASE $CACHE_BASE
mkdir -p $CACHE_BASE
# Use the IMAGE_NAME from the calling script, and make it unique with the temporary path
GIT_MANIFEST_NAME=dib-manifest-git-$(basename ${IMAGE_NAME})
GIT_MANIFEST_CACHE_NAME=${GIT_MANIFEST_NAME}_$(dirname ${TMP_MOUNT_PATH##*.})
GIT_MANIFEST=$CACHE_BASE/${GIT_MANIFEST_CACHE_NAME}
rm -f $GIT_MANIFEST
# Get source repositories for the target
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do
repo_flock=$CACHE_BASE/repositories_flock
(
echo "Getting $repo_flock: $(date) for $_SOURCEREPO"
if ! flock -w 600 9 ; then
echo "Did not get $repo_flock: $(date)"
exit 1
fi
get_repos_for_element $_SOURCEREPO
) 9> $repo_flock
done
# Copy the manifest into the image if it exists (there may be no git repositories used)
if [ -e "$GIT_MANIFEST" ] ; then
sudo mv $GIT_MANIFEST $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR}/$GIT_MANIFEST_NAME
fi