diff --git a/config b/config index fd79b6ee..1cfe11f4 100644 --- a/config +++ b/config @@ -1,14 +1,17 @@ #!/hint/bash FTP_BASE="/srv/ftp" -SVNREPO='' -SVNUSER='' PKGREPOS=() PKGPOOL='' SRCPOOL='' TESTING_REPO='' STABLE_REPOS=() +# VCS backend +VCS=svn +SVNREPO='' +SVNUSER='' + CLEANUP_DESTDIR="/var/tmp" CLEANUP_DRYRUN=false # Time in days to keep moved packages diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 9df5f99a..e64b7e4f 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -44,7 +44,7 @@ for repo in "${PKGREPOS[@]}"; do fi done | sort > "${WORKDIR}/repo-${repo}-${arch}" # get a list of package files defined in the repo db - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + arch_expac "$repo" "$arch" '%f' | sort > "${WORKDIR}/db-${repo}-${arch}" missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if (( ${#missing_pkgs[@]} >= 1 )); then diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6be28abc..4306b818 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -19,24 +19,19 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # - [ ] for repo in "${PKGREPOS[@]}"; do + # The %n+awk+cut bit is because we can't trust %e, as makepkg + # <5.1 didn't set pkgbase in .PKGINFO for non-split packages + # when when pkgbase==pkgname; so until all packages have been + # rebuilt with makepkg 5.1, we need to look at both pkgbase + # (%e) and pkgname (%n) to reliably get pkgbase. for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ - | awk '/^%NAME%/ { getline b }; - /^%BASE%/ { getline b }; - /^%VERSION%/ { getline v }; - /^%LICENSE%/,/^$/ { - if ( !/^%LICENSE%/ ) { l=l" "$0 } - }; - /^%ARCH%/ { - getline a; - printf "%s %s %s %s\n", b, v, a, l; - l=""; - }' - done | sort -u > "${WORKDIR}/db-${repo}" + arch_expac "$repo" "$arch" '%n %e %v %a %L' + done | awk -F' ' '$2 == "(null)" { $2 = $1 } { print }' | cut -d' ' -f2- | + sort -u > "${WORKDIR}/db-${repo}" done for repo in "${PKGREPOS[@]}"; do @@ -77,10 +72,10 @@ for repo in "${PKGREPOS[@]}"; do continue fi - # Get the sources from svn + # Get the sources from vcs mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + vcs_export "$repo" "$pkgarch" "$pkgbase" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" if (( $? >= 1 )); then failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue diff --git a/db-functions b/db-functions index 6d6084a9..c1576f44 100644 --- a/db-functions +++ b/db-functions @@ -6,9 +6,6 @@ shopt -s extglob globstar nullglob -# Some PKGBUILDs need CARCH to be set -CARCH="x86_64" - # Useful functions UMASK="" set_umask () { @@ -39,11 +36,6 @@ mv_acl() { # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -if [[ -n ${SVNUSER} ]]; then - setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" - setfacl -m d:u:"${USER}":rwx "${WORKDIR}" - setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" -fi LOCKS=() REPO_MODIFIED=0 @@ -306,7 +298,7 @@ check_pkgfile() { [[ ${pkgfile##*/} = "${pkgname}-${pkgver}-${pkgarch}"* ]] } -check_pkgsvn() { +check_pkgvcs() { local pkgfile="${1}" local _pkgbase="$(getpkgbase "${pkgfile}")" || return 1 local _pkgname="$(getpkgname "${pkgfile}")" || return 1 @@ -316,17 +308,17 @@ check_pkgsvn() { in_array "${repo}" "${PKGREPOS[@]}" || return 1 - if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then + if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD ]]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + vcs_export "$repo" "$_pkgarch" "$_pkgbase" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version)" - [[ "${svnver}" = "${_pkgver}" ]] || return 1 + local vcsver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; get_full_version)" + [[ "${vcsver}" = "${_pkgver}" ]] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) - in_array "${_pkgname}" "${svnnames[@]}" || return 1 + local vcsnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) + in_array "${_pkgname}" "${vcsnames[@]}" || return 1 return 0 } @@ -337,7 +329,7 @@ check_splitpkgs() { local pkgfiles=("${@}") local pkgfile local pkgdir - local svnname + local vcsname mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null @@ -350,22 +342,22 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then + if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD ]]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + vcs_export "$repo" "$_pkgarch" "$_pkgbase" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) - printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" + local vcsnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) + printf '%s\n' "${vcsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/vcs" done popd >/dev/null for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do [[ ! -d ${pkgdir} ]] && continue sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" - sort -u "${pkgdir}/svn" -o "${pkgdir}/svn" - if [[ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]]; then + sort -u "${pkgdir}/vcs" -o "${pkgdir}/vcs" + if [[ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/vcs")" ]]; then return 1 fi done @@ -436,6 +428,45 @@ set_repo_permission() { fi } +# usage: arch_expac repo arch expac_args... +arch_expac() { + local repo=$1 + local arch=$2 + local args=("${@:3}") + local dir="${WORKDIR}/expac-${repo}-${arch}" + + if ! [[ -d $dir ]]; then + mkdir -p -- "${dir}/root" + cat >"${dir}/pacman.conf" <<-EOT + [options] + RootDir = ${dir}/root + DBPath = ${dir}/root + Architecture = ${arch} + + [$repo] + Server = file://${FTP_BASE}/\$repo/os/\$arch + EOT + fi + + fakeroot pacman --config="${dir}/pacman.conf" -Sy >/dev/null + expac --config="${dir}/pacman.conf" --sync "${args[@]}" +} + +# usage: arch_expac_pkgbase repo arch format +# Like arch_expac, but 'target' may be a pkgbase, instead of the usual +# pkgname. +# Only accepts one target. +arch_expac_pkgbase() { + local repo=$1 + local arch=$2 + local format=$3 + local pkgbase=$4 + arch_expac "$repo" "$arch" "%e %n $format" | + awk -F' ' -v pkgbase="$pkgbase" \ + '$1 == pkgbase || $2 == pkgbase' | + cut -d' ' -f3- +} + arch_repo_modify() { local action=$1 local repo=$2 @@ -458,10 +489,4 @@ arch_repo_modify() { REPO_MODIFIED=1 } -arch_svn() { - if [[ -z ${SVNUSER} ]]; then - /usr/bin/svn "${@}" - else - sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" - fi -} +. "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}" diff --git a/db-functions-svn b/db-functions-svn new file mode 100644 index 00000000..49f2d07e --- /dev/null +++ b/db-functions-svn @@ -0,0 +1,147 @@ +#!/hint/bash + +# Within this file (and in most of dbscripts, really): +# - "pkgarch" refers to the package architecture; may be "any" +# - "tarch" refers to the repo architecture; may NOT be "any" + +if [[ -n ${SVNUSER} ]]; then + setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" +fi + +arch_svn() { + if [[ -z ${SVNUSER} ]]; then + /usr/bin/svn "${@}" + else + sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" + fi +} + +_svn_checkout() { + local pkgbase=$1 + if ! [[ -d ${WORKDIR}/svn ]]; then + arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null + fi + if ! [[ -d ${WORKDIR}/svn/${pkgbase} ]]; then + arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null + fi +} + +# usage: vcs_move_preflight_check repo_from pkgarch pkgbase +# +# Verify that $pkgbase exists in $repo_from in SVN, such that we can +# move it to a different repo. +vcs_move_preflight_check() { + local repo_from=$1 + local tarch=$2 + local pkgbase=$3 + + _svn_checkout "$pkgbase" + local reposdir="${WORKDIR}/svn/${pkgbase}/repos" + [[ -r ${reposdir}/${repo_from}-${tarch}/PKGBUILD || -r ${reposdir}/${repo_from}-any/PKGBUILD ]] +} + +# usage: vcs_move_start repo_from repo_to pkgbase +# +# Begin a VCS transaction moving $pkgbase from $repo_from to $repo_to. +# This should be followed by a call to vcs_move_arch for each +# architecture we're movin it for, and finally by a call to +# vcs_move_finish. +vcs_move_start() { + svn_move_repo_from=$1 + svn_move_repo_to=$2 + svn_move_pkgbase=$3 + + svn_move_tag_list="" +} + +# usage: vcs_move_arch tarch +# +# Add an architecture to a VCS transaction started by vcs_move_start. +# +# If the "from" PKGBUILD doesn't exist, this is a no-op (not an +# error). This is because we expect to be called exactly once for +# each tarch (eg: x86_64, i686), but if arch=(any) then we only need +# do the work once; on the subsequent calls the "from" PKGBUILD won't +# exist anymore. If the source PKGBUILD never existed, we expect that +# to have already been caught by vcs_move_preflight_check. +vcs_move_arch() { + local tarch=$1 + + local repo_from=$svn_move_repo_from + local repo_to=$svn_move_repo_to + local pkgbase=$svn_move_pkgbase + + local pkgarch + if [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${tarch}/PKGBUILD" ]]; then + pkgarch=$tarch + elif [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-any/PKGBUILD" ]]; then + pkgarch=any + else + return 0 + fi + + local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + local svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" + msg2 "%s (%s)" "$pkgbase" "$pkgarch" + + if [[ -d ${svnrepo_to} ]]; then + for file in $(arch_svn ls "${svnrepo_to}"); do + arch_svn rm -q "${svnrepo_to}/$file@" + done + else + mkdir "${svnrepo_to}" + arch_svn add -q "${svnrepo_to}" + fi + + for file in $(arch_svn ls "${svnrepo_from}"); do + arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" + done + arch_svn rm --force -q "${svnrepo_from}" + svn_move_tag_list+=", $pkgarch" +} + +# usage: vcs_move_finish +# +# Commit/finalize a VCS transaction started by vcs_move_start. +vcs_move_finish() { + local repo_from=$svn_move_repo_from + local repo_to=$svn_move_repo_to + local pkgbase=$svn_move_pkgbase + + local tag_list="${svn_move_tag_list#, }" + arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" +} + +# usage: vcs_remove repo pkgarch pkgbase +# +# Remove the given package in VCS. +vcs_remove() { + local repo=$1 + local arch=$2 + local pkgbase=$3 + + local svnrepo="$repo-$arch" + + _svn_checkout "$pkgbase" + if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then + arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" + arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" + else + warning "pkgbase '%s' not found in svn; unable to commit removal to svn" "$pkgbase" + fi +} + +# usage: vcs_export repo pkgarch pkgbase dest +# +# Export the VCS files for a package to the given $dest directory. +vcs_export() { + local repo=$1 + local pkgarch=$2 + local pkgbase=$3 + local dest=$4 + + arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + "${dest}" >/dev/null 2>&1 +} diff --git a/db-move b/db-move index b6448898..b02bc44b 100755 --- a/db-move +++ b/db-move @@ -24,38 +24,24 @@ for pkgarch in "${ARCHES[@]}"; do repo_lock "${repo_from}" "${pkgarch}" || exit 1 done -# check if packages to be moved exist in svn and ftp dir -arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null +# check if packages to be moved exist in vcs and ftp dir for pkgbase in "${args[@]:2}"; do - arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - if (( ${#pkgnames[@]} < 1 )); then - die "Could not read pkgname" - fi + found=false + for tarch in "${ARCHES[@]}"; do + while read -r pkgfile; do - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - if [[ -z ${pkgver} ]]; then - die "Could not read pkgver" + if ! vcs_move_preflight_check "$repo_from" "$tarch" "$pkgbase"; then + die "%s not found in %s-%s" "$pkgbase" "$repo_from" "$tarch" fi - if [[ "${pkgarch}" = any ]]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi + # getpkgfile will `exit` for us if it fails; + # no need to check its result + getpkgfile "${ftppath_from}/${tarch}/${pkgfile}" >/dev/null - for pkgname in "${pkgnames[@]}"; do - for tarch in "${tarches[@]}"; do - getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS} >/dev/null - done - done - continue 2 - fi + found=true + done < <(arch_expac_pkgbase "$repo_from" "$tarch" '%f' "$pkgbase") done - die "%s not found in %s" "$pkgbase" "$repo_from" + [[ $found = true ]] || die "%s not found in %s" "$pkgbase" "$repo_from" done msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" @@ -65,55 +51,22 @@ for arch in "${ARCHES[@]}"; do declare -a remove_pkgs_$arch done for pkgbase in "${args[@]:2}"; do - tag_list="" - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - - if [[ -f ${svnrepo_from}/PKGBUILD ]]; then - if [[ ${pkgarch} = any ]]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi - msg2 "%s (%s)" "$pkgbase" "${tarches[*]}" - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - - if [[ -d ${svnrepo_to} ]]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" - done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" + vcs_move_start "$repo_from" "$repo_to" "$pkgbase" + for tarch in "${ARCHES[@]}"; do + vcs_move_arch "$tarch" + while read -r pkgname pkgver pkgfile; do + declare -n add_pkgs="add_pkgs_${tarch}" + declare -n remove_pkgs="remove_pkgs_${tarch}" + + ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" + if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" fi - - for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" - done - arch_svn rm --force -q "${svnrepo_from}" - tag_list+=", $pkgarch" - - for tarch in "${tarches[@]}"; do - declare -n add_pkgs="add_pkgs_${tarch}" - declare -n remove_pkgs="remove_pkgs_${tarch}" - for pkgname in "${pkgnames[@]}"; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}) - pkgfile="${pkgpath##*/}" - - ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" - if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" - fi - add_pkgs+=("${FTP_BASE}/${PKGPOOL}/${pkgfile}") - remove_pkgs+=("${pkgname}") - done - done - fi + add_pkgs+=("${FTP_BASE}/${PKGPOOL}/${pkgfile}") + remove_pkgs+=("${pkgname}") + done < <(arch_expac_pkgbase "$repo_from" "$tarch" '%n %v %f' "$pkgbase") done - tag_list="${tag_list#, }" - arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" + vcs_move_finish done for tarch in "${ARCHES[@]}"; do diff --git a/db-remove b/db-remove index ac9a1688..60bd8d9b 100755 --- a/db-remove +++ b/db-remove @@ -13,7 +13,6 @@ arch="$2" pkgbases=("${@:3}") ftppath="$FTP_BASE/$repo/os" -svnrepo="$repo-$arch" if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from %s" "$repo" @@ -32,18 +31,11 @@ done remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null - - if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then - remove_pkgs+=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) - arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" - else - warning "%s not found in %s" "$pkgbase" "$svnrepo" - warning "Removing only %s from the repo" "$pkgbase" - warning "If it was a split package you have to remove the others yourself!" - remove_pkgs+=("$pkgbase") - fi + + mapfile -t pkgnames < <(arch_expac_pkgbase "$repo" "${tarches[0]}" '%n' "$pkgbase") + remove_pkgs+=("${pkgnames[@]}") + + vcs_remove "$repo" "$arch" "$pkgbase" done for tarch in "${tarches[@]}"; do diff --git a/db-update b/db-update index 8eb27dfe..0f7ed1fd 100755 --- a/db-update +++ b/db-update @@ -46,8 +46,8 @@ for repo in "${repos[@]}"; do if ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then die "Package %s does not have a valid signature" "$repo/${pkg##*/}" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package %s is not consistent with svn repository" "$repo/${pkg##*/}" + if ! check_pkgvcs "${pkg}" "${repo}"; then + die "Package %s is not consistent with %s repository" "$repo/${pkg##*/}" "$VCS" fi if ! check_pkgrepos "${pkg}"; then die "Package %s already exists in another repository" "$repo/${pkg##*/}" diff --git a/test/Dockerfile b/test/Dockerfile index 83c84499..cb40570f 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM archlinux/base -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep expac RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /build -m tester diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index fe373bca..0597b4dc 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -44,6 +44,25 @@ load ../lib/common done } +@test "remove partial split package" { + local arches=('i686' 'x86_64') + local arch db + + releasePackage extra pkg-split-a + db-update + + for arch in ${arches[@]}; do + db-remove extra "${arch}" pkg-split-a1 + + for db in db files; do + if bsdtar -xf "$FTP_BASE/extra/os/${arch}/extra.${db}" -O | grep pkg-split-a1; then + return 1 + fi + bsdtar -xf "$FTP_BASE/extra/os/${arch}/extra.${db}" -O | grep pkg-split-a2 + done + done +} + @test "remove any packages" { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 9ee06321..0be47d72 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -207,7 +207,7 @@ load ../lib/common @test "add package with inconsistent pkgbuild fails" { releasePackage extra 'pkg-any-a' - updateRepoPKGBUILD 'pkg-any-a' extra any + updateRepoPKGBUILD 'pkg-any-a' extra run db-update [ "$status" -ne 0 ] diff --git a/test/lib/common-svn.bash b/test/lib/common-svn.bash new file mode 100644 index 00000000..00004f7a --- /dev/null +++ b/test/lib/common-svn.bash @@ -0,0 +1,125 @@ +#!/hint/bash + +# usage: vcsSetup +# Do any test-suite setup. +# +# Count on $TMP and $DBSCRIPTS_CONFIG being set, but not much else +# being done. Notably $DBSCRIPTS_CONFIG hasn't been loaded yet (so +# that we have a chance to edit it). +vcsSetup() { + cat <> "${DBSCRIPTS_CONFIG}" + SVNREPO="file://${TMP}/svn-packages-repo" +eot + + mkdir -p "${TMP}"/svn-packages-{copy,repo} + + svnadmin create "${TMP}/svn-packages-repo" + svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" +} + +# usage: vcsDirOfPKGBUILD $pkgbase +# +# Print the path to the directory storing the "trunk"/"master" working +# copy of the PKGBUILD for the given $pkgbase. +vcsDirOfPKGBUILD() { + local pkgbase=$1 + + echo "${TMP}/svn-packages-copy/${pkgbase}/trunk/" +} + +# usage: vcsInitFixture $pkgbase +# +# Initialize from the test fixtures the "trunk"/"master" working copy +# of the PKGBUILD for the given $pkgbase. +vcsInitFixture() { + local pkgbase=$1 + + mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} + cp -r "fixtures/${pkgbase}"/* "${TMP}/svn-packages-copy/${pkgbase}/trunk/" + svn add -q "${TMP}/svn-packages-copy/${pkgbase}" + svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" +} + +# usage: vcsCommit $msg +# Commit changes to the PKGBUILD in the current directory +vcsCommit() { + local msg=$1 + svn commit -q -m"$msg" +} + +# usage: vcsRelease $repo +# Run from the "trunk"/"master" PKGBUILD directory. +# +# This is a cheap imitation of the `archrelease` program that is part +# of the `devtools` package. +vcsRelease() { + local repo=$1 + local pkgarches + local tarch + local tag + + pkgarches=($(. PKGBUILD; echo ${arch[@]})) + pushd .. + for tarch in ${pkgarches[@]}; do + tag=${repo}-${tarch} + + if [[ -d repos/$tag ]]; then + svn rm repos/$tag/PKGBUILD + else + mkdir -p repos/$tag + svn add repos/$tag + fi + + svn copy -r HEAD trunk/PKGBUILD repos/$tag/ + done + svn commit -m "__archrelease" + popd +} + +# usage: vcsCheckPackage $repo $pkgbase $pkgver +# +# Verify that the PKGBUILD for the given $pkgbase is tagged as +# existing in $repo, and has the correct $pkgver. +vcsCheckPackage() { + local repo=$1 + local pkgbase=$2 + local pkgver=$3 + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + + local dirarches=() pkgbuildarches=() + local pkgbuild dirarch pkgbuildver + for pkgbuild in "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-"+([^-])"/PKGBUILD"; do + [[ -e $pkgbuild ]] || continue + dirarch=${pkgbuild%/PKGBUILD} + dirarch=${dirarch##*-} + + dirarches+=("$dirarch") + pkgbuildarches+=($(. "$pkgbuild"; echo ${arch[@]})) + pkgbuildver=$(. "$pkgbuild"; get_full_version) + [[ $pkgver = "$pkgbuildver" ]] + done + # Verify that the arches-from-dirnames and + # arches-from-PKGBUILDs agree (that a PKGBUILD existed for + # every arch). + (( ${#dirarches[@]} > 0 )) + mapfile -d '' dirarches < <(printf '%s\0' "${dirarches[@]}" | sort -uz) + mapfile -d '' pkgbuildarches < <(printf '%s\0' "${pkgbuildarches[@]}" | sort -uz) + declare -p dirarches pkgbuildarches + [[ "${dirarches[*]}" = "${pkgbuildarches[*]}" ]] +} + +# usage: vcsCheckRemovedPackage $repo $pkgbase +# +# Verify that no PKGBUILD for the given $pkgbase is tagged as existing +# in $repo. +vcsCheckRemovedPackage() { + local repo=$1 + local pkgbase=$2 + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + + if __isGlobfile "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-"+([^-])"/PKGBUILD"; then + return 1 + fi +} diff --git a/test/lib/common.bash b/test/lib/common.bash index 5ba31e42..8fc1bdb7 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -1,11 +1,12 @@ . /usr/share/makepkg/util.sh +shopt -s extglob __updatePKGBUILD() { local pkgrel pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=${pkgrel}" + vcsCommit "update pkg to pkgrel=${pkgrel}" } __getCheckSum() { @@ -62,30 +63,6 @@ __buildPackage() { done } -__archrelease() { - local repo=$1 - local pkgarches - local tarch - local tag - - pkgarches=($(. PKGBUILD; echo ${arch[@]})) - pushd .. - for tarch in ${pkgarches[@]}; do - tag=${repo}-${tarch} - - if [[ -d repos/$tag ]]; then - svn rm repos/$tag/PKGBUILD - else - mkdir -p repos/$tag - svn add repos/$tag - fi - - svn copy -r HEAD trunk/PKGBUILD repos/$tag/ - done - svn commit -m "__archrelease" - popd -} - setup() { local p local pkg @@ -98,7 +75,6 @@ setup() { export DBSCRIPTS_CONFIG=${TMP}/config.local cat < "${DBSCRIPTS_CONFIG}" FTP_BASE="${TMP}/ftp" - SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=('core' 'extra' 'testing') PKGPOOL='pool/packages' SRCPOOL='sources/packages' @@ -112,9 +88,11 @@ setup() { CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false eot + . "lib/common-$(. config; echo "$VCS").bash" + vcsSetup . config - mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup} for r in ${PKGREPOS[@]}; do mkdir -p "${TMP}/staging/${r}" @@ -124,9 +102,6 @@ eot done mkdir -p "${TMP}/ftp/${PKGPOOL}" mkdir -p "${TMP}/ftp/${SRCPOOL}" - - svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" } teardown() { @@ -137,23 +112,22 @@ releasePackage() { local repo=$1 local pkgbase=$2 - if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then - mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} - cp -r "fixtures/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkgbase} - svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" + local dir=$(vcsDirOfPKGBUILD "$pkgbase") + + if [[ ! -d $dir ]]; then + vcsInitFixture "$pkgbase" fi - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + pushd "$dir" __buildPackage "${STAGING}"/${repo} - __archrelease ${repo} + vcsRelease ${repo} popd } updatePackage() { local pkgbase=$1 - pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" + pushd "$(vcsDirOfPKGBUILD "$pkgbase")" __updatePKGBUILD __buildPackage popd @@ -162,10 +136,12 @@ updatePackage() { updateRepoPKGBUILD() { local pkgbase=$1 local repo=$2 - local arch=$3 - pushd "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}/" + # Update the PKGBUILD and release the PKGBUILD, but *without* + # ever building or releasing the binary package. + pushd "$(vcsDirOfPKGBUILD "$pkgbase")" __updatePKGBUILD + vcsRelease ${repo} popd } @@ -226,9 +202,7 @@ checkPackage() { local pkgbase=$2 local pkgver=$3 - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - # TODO: Does not fail if one arch is missing - compgen -G "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-*" >/dev/null + vcsCheckPackage "$repo" "$pkgbase" "$pkgver" checkPackageDB "$repo" "$pkgbase" "$pkgver" } @@ -237,10 +211,7 @@ checkRemovedPackage() { local repo=$1 local pkgbase=$2 - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - if compgen -G "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-*" >/dev/null; then - return 1 - fi + vcsCheckRemovedPackage "$repo" "$pkgbase" checkRemovedPackageDB $repo $pkgbase } diff --git a/testing2x b/testing2x index d3b096b3..99280b59 100755 --- a/testing2x +++ b/testing2x @@ -19,31 +19,25 @@ done declare -A pkgs for pkgbase in "$@"; do - if [[ ! -d ${WORKDIR}/${pkgbase} ]]; then - arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null - - found_source=false - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/${pkgbase}/${TESTING_REPO}-${pkgarch}" - if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - found_source=true - break + found_source=false + for tarch in "${ARCHES[@]}"; do + if [[ -n $(arch_expac_pkgbase "$TESTING_REPO" "$tarch" '%n' "$pkgbase") ]]; then + found_source=true + break + fi + done + [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" + found_target=false + for tarch in "${ARCHES[@]}"; do + for repo in "${STABLE_REPOS[@]}"; do + if [[ -n $(arch_expac_pkgbase "$repo" "$tarch" '%n' "$pkgbase") ]]; then + found_target=true + pkgs[${repo}]+="${pkgbase} " + break 2 fi done - [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" - found_target=false - for pkgarch in "${ARCHES[@]}" 'any'; do - for repo in "${STABLE_REPOS[@]}"; do - svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" - if [[ -r ${svnrepo_to}/PKGBUILD ]]; then - found_target=true - pkgs[${repo}]+="${pkgbase} " - break 2 - fi - done - done - [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" - fi + done + [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" done for pkgarch in "${ARCHES[@]}"; do