Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 197 lines (168 sloc) 5.11 KB
#!/usr/bin/env nix-shell
#! nix-shell -i bash -p bash coreutils curl jq nix
# shellcheck shell=bash
# vim: ft=sh sw=2 et
#
# This script utilizes the ./allplugins.txt file to scan the nixos
# github nixops plugin repos for new releases. It then
# generates the corresponding nix code in ./data.nix as
# an attribute set.
set -euo pipefail
# the maximum number of attempts before giving up inside of GET and prefetch_github
readonly maxAttempts=30
# Globals for github rate limiting API availability and reset date
apiAvail=""
apiReset=""
GET() {
local url=$1
local retry=1
echo "fetching $url" >&2
rateLimit=">(grep 'X-RateLimit-Remaining:' | cut -f 2 -d ' ' | tr -cd '0-9' > $tmpfile)"
while ! eval curl -D "$rateLimit" -#fL -u "$GITHUB_AUTH" "$url"; do
# The curl -f option, which reduces a failure to a short succinct output message
# also prevent headers from being recorded to $tmpfile with -D.
# Hence the additional rate limit check on failure with get_rate_limit
get_rate_limit
echo "The curl command has failed. Attempt $retry/${maxAttempts}" >&2
if [[ "${retry}" -eq "${maxAttempts}" ]]; then
restore_and_exit
fi
retry=$(( retry + 1 ))
sleep 5
done
if [[ $(cat "$tmpfile") =~ ^[0-9]+$ ]]; then
echo "Github API calls remaining prior to rate limit: $(cat "$tmpfile")" >&2
fi;
}
get_org_repos() {
local org=$1
# local page=1
GET "https://api.github.com/orgs/$org/repos?per_page=100" | jq -r '.[].name'
}
get_repo_tags() {
local owner=$1
local repo=$2
GET "https://api.github.com/repos/$owner/$repo/git/refs/tags?per_page=100" | \
jq -r '.[].ref' | \
grep -v 'v\.' | \
cut -d '/' -f 3- | \
sort --version-sort
}
prefetch_github() {
local owner=$1
local repo=$2
local rev=$3
local retry=1
while ! nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"; do
echo "The nix-prefetch-url command has failed. Attempt $retry/${maxAttempts}" >&2
if [[ "${retry}" -eq "${maxAttempts}" ]]; then
restore_and_exit
fi
retry=$(( retry + 1 ))
sleep 5
done
}
echo_entry() {
local owner=$1
local repo=$2
local version=${3:1}
local sha256=$4
cat <<EOF
{
owner = "$owner";
repo = "$repo";
version = "$version";
sha256 = "$sha256";
};
EOF
}
indent() { sed 's/^/ /'; }
add_repo() {
org="${1}"
repo="${2}"
echo -e "\n*** $repo ***"
name=$(echo "$repo" | cut -d - -f 2-)
last_tag=$(get_repo_tags "$org" "$repo" | tail -1)
echo "Last tag = $last_tag"
last_tag_sha256=$(prefetch_github "$org" "$repo" "$last_tag")
{
echo " $name ="
echo_entry "$org" "$repo" "$last_tag" "$last_tag_sha256" | indent
} >> data.nix
}
get_rate_limit() {
rateLimitInfo=$(curl -sfL -u "$GITHUB_AUTH" https://api.github.com/rate_limit)
apiAvail=$(jq -r '.resources.core.remaining' <<< "$rateLimitInfo")
apiReset=$(jq -r '.resources.core.reset' <<< "$rateLimitInfo")
if [[ "$apiAvail" == "0" ]]; then rate_limit_fix; fi;
}
rate_limit_fix() {
cat <<RATELIMITED >&2
ERROR:
You currently have 0 Github API calls available.
They will be reset at $(date -d @"$apiReset").
Either:
A) Wait until the reset datetime and then rerun this script.
B) Configure a GITHUB_AUTH token and then rerun this script.
This is required to work around the github request per hour rate-limit.
To configure a GITHUB_AUTH token, go to the following URL:
https://github.com/settings/tokens
Then:
1) Create a new token with the "public_repo" scope.
2) \`export GITHUB_AUTH=<your user>:<your token>\`
3) Rerun this script.
RATELIMITED
restore_and_exit
}
restore_and_exit() {
echo -e "\nRestoring the original data.nix... Exiting.\n" >&2
cp -p "$backup" data.nix
exit 1
}
cleanup() {
rm "$tmpfile" "$backup"
}
trap cleanup EXIT
## Main ##
echo -e "\nUpdating ./data.nix with the latest nixops plugin releases defined in ./all-plugins.txt...\n"
tmpfile=$(mktemp -u -p /tmp tmpfile-XXXXXX)
touch "$tmpfile" && chmod 0600 "$tmpfile"
backup=$(mktemp -u -p /tmp tmpfile-XXXXXX)
cp -p data.nix "$backup"
cd "$(dirname "$0")"
GITHUB_AUTH="${GITHUB_AUTH:=":"}"
get_rate_limit
echo -e "\nInfo: You currently have $apiAvail Github API calls remaining prior to rate limiting."
echo -e " The Github API limit will be reset at $(date -d @"$apiReset").\n"
cat <<HEADER > data.nix
# Generated with ./update-all
{
HEADER
while read -r line; do
IFS=' ' read -r -a fields <<< "$line"
if [[ "${#fields[@]}" -eq 0 ]]; then
continue
fi
if [[ "${fields[0]}" = *"/"* ]]; then
org="$(echo "${fields[0]}" | cut -d/ -f1)"
repo="$(echo "${fields[0]}" | cut -d/ -f2)"
add_repo "${org}" "${repo}"
else
org="${fields[0]}"
repos=$(get_org_repos "$org")
if [[ "${#fields[@]}" -ge 2 ]] && [[ -n "${fields[1]}" ]]; then
repos="$( echo "${repos[@]}" | grep "${fields[1]}" )"
fi
if [[ "${#fields[@]}" -eq 3 ]] && [[ -n "${fields[2]}" ]]; then
repos="$( echo "${repos[@]}" | grep -v "${fields[2]}" )"
fi
repos="$( echo "${repos[@]}" | sort )"
for repo in $repos; do
add_repo "$org" "$repo"
done
fi
done < <(grep -v '^#\|^$' all-plugins.txt)
cat <<FOOTER >> data.nix
}
FOOTER
echo Done.
You can’t perform that action at this time.