Be less tightly coupled with SVN

config

@@ -1,14 +1,17 @@
 #!/hint/bash
 
 FTP_BASE="/srv/ftp"
-SVNREPO=''
-SVNUSER=''
 PKGREPOS=()
 PKGPOOL=''
 SRCPOOL=''
 TESTING_REPO=''
 STABLE_REPOS=()
 
+# VCS backend
+VCS=svn
+SVNREPO=''
+SVNUSER=''
+
 CLEANUP_DESTDIR="/var/tmp"
 CLEANUP_DRYRUN=false
 # Time in days to keep moved packages

cron-jobs/ftpdir-cleanup

@@ -44,7 +44,7 @@ for repo in "${PKGREPOS[@]}"; do
 			fi
 		done | sort > "${WORKDIR}/repo-${repo}-${arch}"
 		# get a list of package files defined in the repo db
-		bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
+		arch_expac "$repo" "$arch" '%f' | sort > "${WORKDIR}/db-${repo}-${arch}"
 
 		missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}"))
 		if (( ${#missing_pkgs[@]} >= 1 )); then

cron-jobs/sourceballs

@@ -19,24 +19,19 @@ renice +10 -p $$ > /dev/null
 # Create a readable file for each repo with the following format
 # <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>]
 for repo in "${PKGREPOS[@]}"; do
+	# The %n+awk+cut bit is because we can't trust %e, as makepkg
+	# <5.1 didn't set pkgbase in .PKGINFO for non-split packages
+	# when when pkgbase==pkgname; so until all packages have been
+	# rebuilt with makepkg 5.1, we need to look at both pkgbase
+	# (%e) and pkgname (%n) to reliably get pkgbase.
 	for arch in "${ARCHES[@]}"; do
 		# Repo does not exist; skip it
 		if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then
 			continue
 		fi
-		bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \
-			| awk '/^%NAME%/ { getline b };
-				/^%BASE%/ { getline b };
-				/^%VERSION%/ { getline v };
-				/^%LICENSE%/,/^$/ {
-					if ( !/^%LICENSE%/ ) { l=l" "$0 }
-					};
-				/^%ARCH%/ {
-					getline a;
-					printf "%s %s %s %s\n", b, v, a, l;
-					l="";
-				}'
-	done | sort -u > "${WORKDIR}/db-${repo}"
+		arch_expac "$repo" "$arch" '%n %e %v %a %L'
+	done | awk -F' ' '$2 == "(null)" { $2 = $1 } { print }' | cut -d' ' -f2- |
+		sort -u > "${WORKDIR}/db-${repo}"
 done
 
 for repo in "${PKGREPOS[@]}"; do
@@ -77,10 +72,10 @@ for repo in "${PKGREPOS[@]}"; do
 				continue
 			fi
 
-			# Get the sources from svn
+			# Get the sources from vcs
 			mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}"
-			arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \
-				"${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1
+			vcs_export "$repo" "$pkgarch" "$pkgbase" \
+				"${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}"
 			if (( $? >= 1 )); then
 				failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}")
 				continue

db-functions

@@ -6,9 +6,6 @@
 shopt -s extglob globstar nullglob
 
 
-# Some PKGBUILDs need CARCH to be set
-CARCH="x86_64"
-
 # Useful functions
 UMASK=""
 set_umask () {
@@ -39,11 +36,6 @@ mv_acl() {
 
 # set up general environment
 WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
-if [[ -n ${SVNUSER} ]]; then
-	setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}"
-	setfacl -m d:u:"${USER}":rwx "${WORKDIR}"
-	setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}"
-fi
 LOCKS=()
 REPO_MODIFIED=0
 
@@ -306,7 +298,7 @@ check_pkgfile() {
 	[[ ${pkgfile##*/} = "${pkgname}-${pkgver}-${pkgarch}"* ]]
 }
 
-check_pkgsvn() {
+check_pkgvcs() {
 	local pkgfile="${1}"
 	local _pkgbase="$(getpkgbase "${pkgfile}")" || return 1
 	local _pkgname="$(getpkgname "${pkgfile}")" || return 1
@@ -316,17 +308,17 @@ check_pkgsvn() {
 
 	in_array "${repo}" "${PKGREPOS[@]}" || return 1
 
-	if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then
+	if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD ]]; then
 		mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}"
-		arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \
+		vcs_export "$repo" "$_pkgarch" "$_pkgbase" \
 			"${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1
 	fi
 
-	local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version)"
-	[[ "${svnver}" = "${_pkgver}" ]] || return 1
+	local vcsver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; get_full_version)"
+	[[ "${vcsver}" = "${_pkgver}" ]] || return 1
 
-	local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}"))
-	in_array "${_pkgname}" "${svnnames[@]}" || return 1
+	local vcsnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}"))
+	in_array "${_pkgname}" "${vcsnames[@]}" || return 1
 
 	return 0
 }
@@ -337,7 +329,7 @@ check_splitpkgs() {
 	local pkgfiles=("${@}")
 	local pkgfile
 	local pkgdir
-	local svnname
+	local vcsname
 
 	mkdir -p "${WORKDIR}/check_splitpkgs/"
 	pushd "${WORKDIR}/check_splitpkgs" >/dev/null
@@ -350,22 +342,22 @@ check_splitpkgs() {
 		mkdir -p "${repo}/${_pkgarch}/${_pkgbase}"
 		echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging"
 
-		if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then
+		if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD ]]; then
 			mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}"
-			arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \
+			vcs_export "$repo" "$_pkgarch" "$_pkgbase" \
 				"${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1
 		fi
 
-		local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}"))
-		printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn"
+		local vcsnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}"))
+		printf '%s\n' "${vcsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/vcs"
 	done
 	popd >/dev/null
 
 	for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do
 		[[ ! -d ${pkgdir} ]] && continue
 		sort -u "${pkgdir}/staging" -o "${pkgdir}/staging"
-		sort -u "${pkgdir}/svn" -o "${pkgdir}/svn"
-		if [[ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]]; then
+		sort -u "${pkgdir}/vcs" -o "${pkgdir}/vcs"
+		if [[ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/vcs")" ]]; then
 			return 1
 		fi
 	done
@@ -436,6 +428,45 @@ set_repo_permission() {
 	fi
 }
 
+# usage: arch_expac repo arch expac_args...
+arch_expac() {
+	local repo=$1
+	local arch=$2
+	local args=("${@:3}")
+	local dir="${WORKDIR}/expac-${repo}-${arch}"
+
+	if ! [[ -d $dir ]]; then
+		mkdir -p -- "${dir}/root"
+		cat >"${dir}/pacman.conf" <<-EOT
+		[options]
+		RootDir = ${dir}/root
+		DBPath = ${dir}/root
+		Architecture = ${arch}
+
+		[$repo]
+		Server = file://${FTP_BASE}/\$repo/os/\$arch
+		EOT
+	fi
+
+	fakeroot pacman --config="${dir}/pacman.conf" -Sy >/dev/null
+	expac --config="${dir}/pacman.conf" --sync "${args[@]}"
+}
+
+# usage: arch_expac_pkgbase repo arch format <pkgname|pkgbase>
+# Like arch_expac, but 'target' may be a pkgbase, instead of the usual
+# pkgname.
+# Only accepts one target.
+arch_expac_pkgbase() {
+	local repo=$1
+	local arch=$2
+	local format=$3
+	local pkgbase=$4
+	arch_expac "$repo" "$arch" "%e %n $format" |
+		awk -F' ' -v pkgbase="$pkgbase" \
+			'$1 == pkgbase || $2 == pkgbase' |
+		cut -d' ' -f3-
+}
+
 arch_repo_modify() {
 	local action=$1
 	local repo=$2
@@ -458,10 +489,4 @@ arch_repo_modify() {
 	REPO_MODIFIED=1
 }
 
-arch_svn() {
-	if [[ -z ${SVNUSER} ]]; then
-		/usr/bin/svn "${@}"
-	else
-		sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}"
-	fi
-}
+. "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}"

db-functions-svn

@@ -0,0 +1,147 @@
+#!/hint/bash
+
+# Within this file (and in most of dbscripts, really):
+# - "pkgarch" refers to the package architecture; may be "any"
+# - "tarch" refers to the repo architecture; may NOT be "any"
+
+if [[ -n ${SVNUSER} ]]; then
+	setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}"
+	setfacl -m d:u:"${USER}":rwx "${WORKDIR}"
+	setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}"
+fi
+
+arch_svn() {
+	if [[ -z ${SVNUSER} ]]; then
+		/usr/bin/svn "${@}"
+	else
+		sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}"
+	fi
+}
+
+_svn_checkout() {
+	local pkgbase=$1
+	if ! [[ -d ${WORKDIR}/svn ]]; then
+		arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null
+	fi
+	if ! [[ -d ${WORKDIR}/svn/${pkgbase} ]]; then
+		arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null
+	fi
+}
+
+# usage: vcs_move_preflight_check repo_from pkgarch pkgbase
+#
+# Verify that $pkgbase exists in $repo_from in SVN, such that we can
+# move it to a different repo.
+vcs_move_preflight_check() {
+	local repo_from=$1
+	local tarch=$2
+	local pkgbase=$3
+
+	_svn_checkout "$pkgbase"
+	local reposdir="${WORKDIR}/svn/${pkgbase}/repos"
+	[[ -r ${reposdir}/${repo_from}-${tarch}/PKGBUILD || -r ${reposdir}/${repo_from}-any/PKGBUILD ]]
+}
+
+# usage: vcs_move_start repo_from repo_to pkgbase
+#
+# Begin a VCS transaction moving $pkgbase from $repo_from to $repo_to.
+# This should be followed by a call to vcs_move_arch for each
+# architecture we're movin it for, and finally by a call to
+# vcs_move_finish.
+vcs_move_start() {
+	svn_move_repo_from=$1
+	svn_move_repo_to=$2
+	svn_move_pkgbase=$3
+
+	svn_move_tag_list=""
+}
+
+# usage: vcs_move_arch tarch
+#
+# Add an architecture to a VCS transaction started by vcs_move_start.
+#
+# If the "from" PKGBUILD doesn't exist, this is a no-op (not an
+# error).  This is because we expect to be called exactly once for
+# each tarch (eg: x86_64, i686), but if arch=(any) then we only need
+# do the work once; on the subsequent calls the "from" PKGBUILD won't
+# exist anymore.  If the source PKGBUILD never existed, we expect that
+# to have already been caught by vcs_move_preflight_check.
+vcs_move_arch() {
+	local tarch=$1
+
+	local repo_from=$svn_move_repo_from
+	local repo_to=$svn_move_repo_to
+	local pkgbase=$svn_move_pkgbase
+
+	local pkgarch
+	if [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${tarch}/PKGBUILD" ]]; then
+		pkgarch=$tarch
+	elif [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-any/PKGBUILD" ]]; then
+		pkgarch=any
+	else
+		return 0
+	fi
+
+	local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}"
+	local svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}"
+	msg2 "%s (%s)" "$pkgbase" "$pkgarch"
+
+	if [[ -d ${svnrepo_to} ]]; then
+		for file in $(arch_svn ls "${svnrepo_to}"); do
+			arch_svn rm -q "${svnrepo_to}/$file@"
+		done
+	else
+		mkdir "${svnrepo_to}"
+		arch_svn add -q "${svnrepo_to}"
+	fi
+
+	for file in $(arch_svn ls "${svnrepo_from}"); do
+		arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/"
+	done
+	arch_svn rm --force -q "${svnrepo_from}"
+	svn_move_tag_list+=", $pkgarch"
+}
+
+# usage: vcs_move_finish
+#
+# Commit/finalize a VCS transaction started by vcs_move_start.
+vcs_move_finish() {
+	local repo_from=$svn_move_repo_from
+	local repo_to=$svn_move_repo_to
+	local pkgbase=$svn_move_pkgbase
+
+	local tag_list="${svn_move_tag_list#, }"
+	arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})"
+}
+
+# usage: vcs_remove repo pkgarch pkgbase
+#
+# Remove the given package in VCS.
+vcs_remove() {
+	local repo=$1
+	local arch=$2
+	local pkgbase=$3
+
+	local svnrepo="$repo-$arch"
+
+	_svn_checkout "$pkgbase"
+	if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then
+		arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo"
+		arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)"
+	else
+		warning "pkgbase '%s' not found in svn; unable to commit removal to svn" "$pkgbase"
+	fi
+}
+
+# usage: vcs_export repo pkgarch pkgbase dest
+#
+# Export the VCS files for a package to the given $dest directory.
+vcs_export() {
+	local repo=$1
+	local pkgarch=$2
+	local pkgbase=$3
+	local dest=$4
+
+	arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \
+		 "${dest}" >/dev/null 2>&1
+}