From 6fae43d643a9e6829a3f2acadc0b1e0bb1c9e9fe Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Sun, 14 Jun 2020 15:59:52 -0400 Subject: [PATCH 01/24] wip: add support for debug packages --- db-functions | 11 ++++++++++- db-update | 31 ++++++++++++++++++++++++------- test/lib/common.bash | 4 ++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/db-functions b/db-functions index 22455319..b6fd867d 100644 --- a/db-functions +++ b/db-functions @@ -234,6 +234,14 @@ getpkgarch() { echo "$_ver" } +is_debug_package() { + local pkgfile=${1} + local pkgbase="$(getpkgbase "${pkgfile}")" + local pkgname="$(getpkgname "${pkgfile}")" + + [[ ${pkgbase}-debug = ${pkgname} ]] +} + check_packager() { local _packager @@ -310,7 +318,7 @@ check_pkgvcs() { read -ra vcsnames <<<"${vcsnames}" [[ "${vcsver}" = "${_pkgver}" ]] || return 1 - in_array "${_pkgname}" "${vcsnames[@]}" || return 1 + in_array "${_pkgname}" "${vcsnames[@]}" "${_pkgbase}-debug" || return 1 return 0 } @@ -334,6 +342,7 @@ check_splitpkgs() { # not a split package (( ${#vcsnames[@]} > 1 )) || continue + [[ ${_pkgbase}-debug = ${_pkgname} ]] && continue mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" diff --git a/db-update b/db-update index b85295f3..586c17a4 100755 --- a/db-update +++ b/db-update @@ -82,28 +82,45 @@ for repo in "${repos[@]}"; do any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() + debug_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*"-${pkgarch}"${PKGEXTS} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" + + if is_debug_package "${pkg}"; then + debug_pkgs+=("${pkgfile}") + currentpool=${PKGPOOL}-debug + currentrepo=${repo}-debug + else + add_pkgs+=("${pkgfile}") + currentpool=${PKGPOOL} + currentrepo=${repo} + fi + msg2 '%s (%s)' "$pkgfile" "$pkgarch" # any packages might have been moved by the previous run if [[ -f ${pkg} ]]; then - mv "${pkg}" "$FTP_BASE/${PKGPOOL}" + mv "${pkg}" "$FTP_BASE/${currentpool}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + ln -s "../../../${currentpool}/${pkgfile}" "$FTP_BASE/${currentrepo}/os/${pkgarch}" # also move signatures if [[ -f ${pkg}.sig ]]; then - mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" + mv "${pkg}.sig" "$FTP_BASE/${currentpool}" fi - "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-archive" "${FTP_BASE}/${PKGPOOL}/${pkg##*/}" - if [[ -f $FTP_BASE/${PKGPOOL}/${pkgfile}.sig ]]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" + if [[ ${PKGPOOL} = ${currentpool} ]]; then + # do not archive debug info, this is not of historic interest + "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-archive" "${FTP_BASE}/${PKGPOOL}/${pkg##*/}" + fi + if [[ -f $FTP_BASE/${currentpool}/${pkgfile}.sig ]]; then + ln -s "../../../${currentpool}/${pkgfile}.sig" "$FTP_BASE/${currentrepo}/os/${pkgarch}" fi - add_pkgs+=("${pkgfile}") done if (( ${#add_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}" "${pkgarch}" ${add_pkgs[@]} fi + if (( ${#debug_pkgs[@]} >= 1 )); then + arch_repo_modify add "${repo}-debug" "${pkgarch}" ${debug_pkgs[@]} + fi done done diff --git a/test/lib/common.bash b/test/lib/common.bash index 02693809..604f27b2 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -121,9 +121,9 @@ eot mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} for r in ${PKGREPOS[@]}; do - mkdir -p "${TMP}/staging/${r}" + mkdir -p "${TMP}"/staging/${r}{,-debug} for a in ${ARCHES[@]}; do - mkdir -p "${TMP}/ftp/${r}/os/${a}" + mkdir -p "${TMP}"/ftp/${r}{,-debug}/os/${a} done done mkdir -p "${TMP}/ftp/${PKGPOOL}" From 3c8016a810bc6343155349d7446dc2ad66523a59 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Sep 2020 15:43:39 -0400 Subject: [PATCH 02/24] debug prints --- db-functions | 4 ++++ db-move | 4 ++++ db-update | 2 ++ test/Dockerfile | 2 +- test/cases/db-move.bats | 6 ++++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index b6fd867d..36348574 100644 --- a/db-functions +++ b/db-functions @@ -435,6 +435,7 @@ check_repo_permission() { } set_repo_permission() { + local -; set -x local repo=$1 local arch=$2 local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" @@ -452,11 +453,13 @@ set_repo_permission() { } arch_repo_modify() { + echo "call: arch_repo_modify ${@@Q}" local action=$1 local repo=$2 local arch=$3 local pkgs=("${@:4}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + tree "${FTP_BASE}" if [[ ${action} = remove && ! -f ${dbfile} ]]; then error "No database found at '%s'" "$dbfile" @@ -465,6 +468,7 @@ arch_repo_modify() { # package files for repo-add might be relative to repo dir pushd "${dbfile%/*}" >/dev/null + ls /usr/bin/"repo-${action}" -q "${dbfile}" "${pkgs[@]}" \ || error '%s' "repo-${action} ${dbfile@Q} ${pkgs[*]@Q}" set_repo_permission "${repo}" "${arch}" diff --git a/db-move b/db-move index 6f20397c..1839c61c 100755 --- a/db-move +++ b/db-move @@ -65,7 +65,9 @@ msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" for arch in "${ARCHES[@]}"; do declare -a add_pkgs_$arch + declare -a add_debug_pkgs_$arch declare -a remove_pkgs_$arch + declare -a remove_debug_pkgs_$arch done for pkgbase in "${args[@]:2}"; do tag_list="" @@ -88,7 +90,9 @@ for pkgbase in "${args[@]:2}"; do for tarch in "${tarches[@]}"; do declare -n add_pkgs="add_pkgs_${tarch}" + declare -n add_debug_pkgs="add_debug_pkgs_${tarch}" declare -n remove_pkgs="remove_pkgs_${tarch}" + declare -n remove_debug_pkgs="remove_debug_pkgs_${tarch}" for pkgname in "${pkgnames[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}) pkgfile="${pkgpath##*/}" diff --git a/db-update b/db-update index 586c17a4..069c564d 100755 --- a/db-update +++ b/db-update @@ -79,6 +79,7 @@ done for repo in "${repos[@]}"; do msg "Updating [%s]..." "$repo" + tree "${STAGING}/${repo}" any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() @@ -115,6 +116,7 @@ for repo in "${repos[@]}"; do ln -s "../../../${currentpool}/${pkgfile}.sig" "$FTP_BASE/${currentrepo}/os/${pkgarch}" fi done + declare -p add_pkgs debug_pkgs if (( ${#add_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}" "${pkgarch}" ${add_pkgs[@]} fi diff --git a/test/Dockerfile b/test/Dockerfile index 83c84499..5109a7c0 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM archlinux/base -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /build -m tester diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index f4407168..05bd3776 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -7,15 +7,21 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do + echo "releasing to testing: $pkgbase" releasePackage testing ${pkgbase} done + echo "db-updating..." db-update + echo "db-move testing -> extra pkg-simple-a" db-move testing extra pkg-simple-a + echo checkRemovedPackage testing pkg-simple-a checkRemovedPackage testing pkg-simple-a + echo checkPackage extra pkg-simple-a 1-1 checkPackage extra pkg-simple-a 1-1 + echo checkPackage testing pkg-simple-b 1-1 checkPackage testing pkg-simple-b 1-1 } From b54cfdc988ef0053e969fff64d0f9fd49816d541 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Sep 2020 15:09:15 -0400 Subject: [PATCH 03/24] test: sync dummy packages impl with makepkg Or else it doesn't work if you have packages installed but not from a repo --- test/lib/common.bash | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 604f27b2..44feb4ba 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -130,12 +130,16 @@ eot mkdir -p "${TMP}/ftp/${SRCPOOL}" # make dummy packages for "reproducibility" - pacman -Qq | pacman -Sddp - | while read -r line; do - line=${line##*/} - pkgname=${line%-*-*-*} - mkdir -p "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}" - touch "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}/${line}"{,.sig} - done + pacman -Qi | awk -F': ' '\ + /^Name .*/ {printf "%s", $2} \ + /^Version .*/ {printf "-%s", $2} \ + /^Architecture .*/ {print "-"$2} \ + ' | while read -r line; do + line=$line.pkg.tar.xz + pkgname=${line%-*-*-*} + mkdir -p "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}" + touch "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}/${line}"{,.sig} + done svnadmin create "${TMP}/svn-packages-repo" svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" From 838a0dcfdea4b4f8d9f27666225414c4dc45bc99 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Sep 2020 15:16:14 -0400 Subject: [PATCH 04/24] test: let the Makefile drive the needed work BUILDDIR is a testsuite requirement, cases might be set manually --- test/Dockerfile | 1 - test/Makefile | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/Dockerfile b/test/Dockerfile index 5109a7c0..48e6cf30 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -16,5 +16,4 @@ Expire-Date: 0\n\ | gpg --quiet --batch --no-tty --no-permission-warning --gen-key RUN gpg --export | sudo pacman-key -a - RUN sudo pacman-key --lsign-key tester@localhost -ENV BUILDDIR=/build ENV PACKAGER="Bob Tester " diff --git a/test/Makefile b/test/Makefile index d9437bfd..d7ac08a6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,11 +1,13 @@ +CASES ?= cases + test: - PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats cases + BUILDDIR=/build PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats $(CASES) test-coverage: - PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) kcov \ + BUILDDIR=/build PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) kcov \ --include-path=$(CURDIR)/../ \ --exclude-path=$(CURDIR)/../test,$(CURDIR)/../cron-jobs/makepkg.conf,$(CURDIR)/../config \ $(COVERAGE_DIR) \ - bats cases + bats $(CASES) .PHONY: test test-coverage From c9c030bae37605ef5dea0009ac4d5547633edf5c Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Sep 2020 15:37:28 -0400 Subject: [PATCH 05/24] wip: debug: lock both repos at once --- db-functions | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index 36348574..d42b2058 100644 --- a/db-functions +++ b/db-functions @@ -109,7 +109,14 @@ trap cleanup EXIT #repo_lock [timeout] -repo_lock () { +repo_lock() { + local repo base=${1}; shift + for repo in ${base} ${base}-debug; do + _repo_lock ${repo} "${@}" + done +} + +_repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" local _count From 190f762f171922e822180b0be838cc10bab65e16 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Sep 2020 16:04:01 -0400 Subject: [PATCH 06/24] test: add fixture for splitdebug package --- test/fixtures/pkg-debuginfo/PKGBUILD | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 test/fixtures/pkg-debuginfo/PKGBUILD diff --git a/test/fixtures/pkg-debuginfo/PKGBUILD b/test/fixtures/pkg-debuginfo/PKGBUILD new file mode 100644 index 00000000..0da276a5 --- /dev/null +++ b/test/fixtures/pkg-debuginfo/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-debuginfo +pkgver=1 +pkgrel=1 +pkgdesc="A package named ${pkgname}" +arch=('i686' 'x86_64') + +# guarantee that split debug packages are turned on +options=('strip' 'debug') + +package() { + install -D /dev/null "${pkgdir}"/usr/bin/hello + install -D /dev/null "${pkgdir}"/../${pkgname}-debug/usr/lib/debug/usr/bin/hello.debug +} From b99091ec5b0bd8e2750b4e6aa0f739dbddb2c2ce Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Wed, 24 Feb 2021 22:44:41 +0100 Subject: [PATCH 07/24] tests: Added CASES to top-level Makefile Also added cache for built packages Signed-off-by: Morten Linderud --- Makefile | 10 ++++++---- test/Dockerfile | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 988d3354..9b939826 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,17 @@ IMAGE:=dbscripts/test -RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test +RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro -v dbscripts:/build --tmpfs=/tmp:exec -w /dbscripts/test +CASES ?= cases +CRUN := docker test-image: - docker build --pull -t $(IMAGE) test + $(CRUN) build --pull -t $(IMAGE) test test: test-image - docker run $(RUN_OPTIONS) $(IMAGE) make test + $(CRUN) run $(RUN_OPTIONS) $(IMAGE) make CASES=$(CASES) test test-coverage: test-image rm -rf ${PWD}/coverage mkdir -m 777 ${PWD}/coverage - docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) make test-coverage + $(CRUN) run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) make test-coverage .PHONY: test-image test test-coverage diff --git a/test/Dockerfile b/test/Dockerfile index 48e6cf30..a3594858 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,8 +1,10 @@ -FROM archlinux/base +FROM docker.io/archlinux/base RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel -RUN useradd -N -g users -G wheel -d /build -m tester +RUN useradd -N -g users -G wheel -d /tester -m tester +RUN mkdir -p /build; chown -R tester: /build +VOLUME ["/build"] USER tester RUN echo -e "\ Key-Type: RSA\n\ From bf6f0df46b05a4f59801b296db43b30c95f60ef2 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 2 Mar 2021 21:36:17 +0100 Subject: [PATCH 08/24] wip: debug: unlock both repos at once Signed-off-by: Morten Linderud --- db-functions | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index d42b2058..3a3269e3 100644 --- a/db-functions +++ b/db-functions @@ -159,7 +159,14 @@ _repo_lock () { return 1 } -repo_unlock () { #repo_unlock +repo_unlock(){ + local repo base=${1}; shift + for repo in ${base} ${base}-debug; do + _repo_unlock ${repo} "${@}" + done +} + +_repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [[ ! -d $LOCKDIR ]]; then warning "Repo lock [%s] (%s) was not locked!" "$1" "$2" From 482d91fdc4796be71a86b1a2522b91d4fae4cd46 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 2 Mar 2021 22:27:02 +0100 Subject: [PATCH 09/24] wip: debug: support debug packages with db-remove Signed-off-by: Morten Linderud --- db-remove | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/db-remove b/db-remove index a27a9f29..4ba7ead7 100755 --- a/db-remove +++ b/db-remove @@ -45,6 +45,16 @@ for pkgbase in "${pkgbases[@]}"; do done for tarch in "${tarches[@]}"; do - arch_repo_modify remove "${repo}" "${tarch}" "${remove_pkgs[@]}" + if (( ${#remove_pkgs[@]} >= 1 )); then + arch_repo_modify remove "${repo}" "${tarch}" "${remove_pkgs[@]}" + for pkg in "${remove_pkgs[@]}"; do + if is_globfile "${FTP_BASE}/${repo}-debug/os/${tarch}/${pkg}-debug"*; then + msg "Found debug package. Removing %s from [%s]..." "$pkg" "$repo" + remove_debug_pkgs+=("${pkg}-debug") + arch_repo_modify remove "${repo}-debug" "${tarch}" "${pkg}-debug" + fi + done + fi + repo_unlock "$repo" "$tarch" done From 64b2993d7d6940c8e6066f206a1be160d3a2a3f6 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Wed, 3 Mar 2021 19:25:47 +0100 Subject: [PATCH 10/24] wip: debug: add strip binary Signed-off-by: Morten Linderud --- test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Dockerfile b/test/Dockerfile index a3594858..ae91cfbd 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM docker.io/archlinux/base -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /tester -m tester From a4f759e452b9eb4ecc9c755262fa008a3f2ac680 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Wed, 3 Mar 2021 23:15:39 +0100 Subject: [PATCH 11/24] wip: debug: Added package pool creation for debug Signed-off-by: Morten Linderud --- test/lib/common.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/lib/common.bash b/test/lib/common.bash index 44feb4ba..af4bf6a9 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -128,6 +128,8 @@ eot done mkdir -p "${TMP}/ftp/${PKGPOOL}" mkdir -p "${TMP}/ftp/${SRCPOOL}" + mkdir -p "${TMP}/ftp/${PKGPOOL}-debug" + mkdir -p "${TMP}/ftp/${SRCPOOL}-debug" # make dummy packages for "reproducibility" pacman -Qi | awk -F': ' '\ From 4978995d06e65cdf81f2dce9567977c696792fe7 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 00:25:45 +0100 Subject: [PATCH 12/24] wip: debug: added first debug package test It's passing! Signed-off-by: Morten Linderud --- test/cases/db-update.bats | 7 +++++++ test/lib/common.bash | 23 +++++++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index ddeb4060..2aec4ee5 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -23,6 +23,13 @@ load ../lib/common checkPackage extra 'pkg-single-arch' 1-1 } +@test "add debug package" { + releasePackage extra 'pkg-debuginfo' + db-update + checkPackage extra 'pkg-debuginfo' 1-1 + checkPackage extra-debug 'pkg-debuginfo' 1-1 +} + @test "add single epoch package" { releasePackage extra 'pkg-single-epoch' db-update diff --git a/test/lib/common.bash b/test/lib/common.bash index af4bf6a9..5cc29f3c 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -199,7 +199,13 @@ checkPackageDB() { local pkgname local pkgarches=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${arch[@]})) - local pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) + # TODO: We need a better way to figure out when we are dealing with + # debug packages + if [[ "${repo}" = *-debug ]]; then + local pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo "${pkgname[@]}-debug")) + else + local pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) + fi if [[ ${pkgarches[@]} == any ]]; then repoarches=(${ARCHES[@]}) @@ -210,9 +216,8 @@ checkPackageDB() { for pkgarch in ${pkgarches[@]}; do for pkgname in ${pkgnames[@]}; do pkgfile="${pkgname}-${pkgver}-${pkgarch}${PKGEXT}" - - [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] - [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] + [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] || [ -r ${FTP_BASE}/${PKGPOOL}-debug/${pkgfile} ] + [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] || [ -r ${FTP_BASE}/${PKGPOOL}-debug/${pkgfile}.sig ] [ ! -r ${STAGING}/${repo}/${pkgfile} ] [ ! -r ${STAGING}/${repo}/${pkgfile}.sig ] @@ -223,12 +228,14 @@ checkPackageDB() { continue fi fi - + debug echo "${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}" [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile} ] - [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] || \ + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${PKGPOOL}-debug/${pkgfile} ] [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig ] - [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] || \ + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${PKGPOOL}-debug/${pkgfile}.sig ] for db in ${DBEXT} ${FILESEXT}; do [ -r "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" ] @@ -248,7 +255,7 @@ checkPackage() { local dirarches=() pkgbuildarches=() local pkgbuild dirarch pkgbuildver - for pkgbuild in "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-"+([^-])"/PKGBUILD"; do + for pkgbuild in "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; do [[ -e $pkgbuild ]] || continue dirarch=${pkgbuild%/PKGBUILD} dirarch=${dirarch##*-} From 8de07415caa38c8bcb32bd9b4823089c8d3644a8 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 00:31:12 +0100 Subject: [PATCH 13/24] wip: debug: Added debug shortcut Signed-off-by: Morten Linderud --- test/lib/common.bash | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/lib/common.bash b/test/lib/common.bash index 5cc29f3c..21ad5233 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -147,6 +147,12 @@ eot svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" } +debug(){ + local -; set -x + $@ + set +x +} + teardown() { rm -rf "${TMP}" } From 39e8367e16247a7594df1d81390f7678b5518293 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 23:34:17 +0100 Subject: [PATCH 14/24] wip: debug: checkRemovedPackage finds debug packages Signed-off-by: Morten Linderud --- test/lib/common.bash | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 21ad5233..15d36b73 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -288,8 +288,7 @@ checkRemovedPackage() { local pkgbase=$2 svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - - if __isGlobfile "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-"+([^-])"/PKGBUILD"; then + if __isGlobfile "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; then return 1 fi @@ -308,7 +307,14 @@ checkRemovedPackageDB() { local pkgname pkgarches=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${arch[@]})) - pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) + + # TODO: We need a better way to figure out when we are dealing with + # debug packages + if [[ "${repo}" = *-debug ]]; then + pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo "${pkgname[@]}-debug")) + else + pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) + fi if [[ ${pkgarches[@]} == any ]]; then tarches=(${ARCHES[@]}) From e8c8d4b9c37d0ef48156095e62b7f5bf5b1879c8 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 23:35:24 +0100 Subject: [PATCH 15/24] wip: debug: added debug db-move test case Signed-off-by: Morten Linderud --- test/cases/db-move.bats | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index 05bd3776..8199dcbd 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -25,6 +25,33 @@ load ../lib/common checkPackage testing pkg-simple-b 1-1 } +@test "move debug package" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + echo "releasing to testing: $pkgbase" + releasePackage testing ${pkgbase} + done + + echo "db-updating..." + db-update + + echo "db-move testing -> extra pkg-debuginfo" + db-move testing extra pkg-debuginfo + + echo checkRemovedPackage testing-debug pkg-debuginfo-debug + checkRemovedPackage testing-debug pkg-debuginfo + + echo checkPackage extra-debug pkg-debuginfo 1-1 + checkPackage extra-debug pkg-debuginfo 1-1 + + echo checkPackage testing pkg-simple-b 1-1 + checkPackage testing pkg-simple-b 1-1 +} + @test "move multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') From 1d63a2495f3d9ca568e6a2a701ae03fc937c1094 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 23:36:18 +0100 Subject: [PATCH 16/24] wip: debug: db-move debug package support Signed-off-by: Morten Linderud --- db-move | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/db-move b/db-move index 1839c61c..4e577b53 100755 --- a/db-move +++ b/db-move @@ -12,7 +12,9 @@ args=("${@}") repo_from="${args[0]}" repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" +ftppath_debug_from="${FTP_BASE}/${repo_from}-debug/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" +ftppath_debug_to="${FTP_BASE}/${repo_to}-debug/os/" check_leapfrog=false if in_array "${repo_from}" "${STAGING_REPOS[@]}" && in_array "${repo_to}" "${STABLE_REPOS[@]}"; then @@ -103,6 +105,18 @@ for pkgbase in "${args[@]:2}"; do fi add_pkgs+=("${FTP_BASE}/${PKGPOOL}/${pkgfile}") remove_pkgs+=("${pkgname}") + + debug_pkgpath=$(getpkgfile "${ftppath_debug_from}/${tarch}/${pkgname}-debug-${pkgver}-${pkgarch}"${PKGEXTS}) + debug_pkgfile="${debug_pkgpath##*/}" + if [[ -f ${debug_pkgpath} ]]; then + msg2 "Found debug package %s" "${pkgname}-debug" + ln -s "../../../${PKGPOOL}-debug/${debug_pkgfile}" "${ftppath_debug_to}/${tarch}/" + if [[ -f ${FTP_BASE}/${PKGPOOL}-debug/${debug_pkgfile}.sig ]]; then + ln -s "../../../${PKGPOOL}-debug/${debug_pkgfile}.sig" "${ftppath_debug_to}/${tarch}/" + fi + add_debug_pkgs+=("${FTP_BASE}/${PKGPOOL}-debug/${debug_pkgfile}") + remove_debug_pkgs+=("${pkgname}-debug") + fi done done fi @@ -113,11 +127,17 @@ done for tarch in "${ARCHES[@]}"; do declare -n add_pkgs="add_pkgs_${tarch}" + declare -n add_debug_pkgs="add_debug_pkgs_${tarch}" declare -n remove_pkgs="remove_pkgs_${tarch}" + declare -n remove_debug_pkgs="remove_debug_pkgs_${tarch}" if [[ -n ${add_pkgs[@]} ]]; then arch_repo_modify add "${repo_to}" "${tarch}" "${add_pkgs[@]}" arch_repo_modify remove "${repo_from}" "${tarch}" "${remove_pkgs[@]}" fi + if [[ -n ${add_debug_pkgs[@]} ]]; then + arch_repo_modify add "${repo_to}-debug" "${tarch}" "${add_debug_pkgs[@]}" + arch_repo_modify remove "${repo_from}-debug" "${tarch}" "${remove_debug_pkgs[@]}" + fi done for pkgarch in "${ARCHES[@]}"; do From e6546ad38a657302bf152b38c25ad2703eb85fb8 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 23:44:39 +0100 Subject: [PATCH 17/24] wip: debug: Ensure the actual package is moved Signed-off-by: Morten Linderud --- test/cases/db-move.bats | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index 8199dcbd..243e956a 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -42,9 +42,15 @@ load ../lib/common echo "db-move testing -> extra pkg-debuginfo" db-move testing extra pkg-debuginfo + echo checkRemovedPackage testing pkg-debuginfo + checkRemovedPackage testing pkg-debuginfo + echo checkRemovedPackage testing-debug pkg-debuginfo-debug checkRemovedPackage testing-debug pkg-debuginfo + echo checkPackage extra pkg-debuginfo 1-1 + checkPackage extra pkg-debuginfo 1-1 + echo checkPackage extra-debug pkg-debuginfo 1-1 checkPackage extra-debug pkg-debuginfo 1-1 From 3da9a2310a2a56a7cfb7b23cdf67af6f1779df52 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 23:55:00 +0100 Subject: [PATCH 18/24] wip: debug: added db-remove test for debug packages Signed-off-by: Morten Linderud --- test/cases/db-remove.bats | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index 0597b4dc..6f3bb42e 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -23,6 +23,28 @@ load ../lib/common done } +@test "remove debug package" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgbase} + done + done + + checkRemovedPackage extra pkg-debuginfo + checkRemovedPackage extra-debug pkg-debuginfo +} + @test "remove multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') From 683f13fd2e9600d297f636eb1cc173a9e65ccf3b Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Thu, 4 Mar 2021 23:55:13 +0100 Subject: [PATCH 19/24] db-remove: removed unused variable Signed-off-by: Morten Linderud --- db-remove | 1 - 1 file changed, 1 deletion(-) diff --git a/db-remove b/db-remove index 4ba7ead7..7daed118 100755 --- a/db-remove +++ b/db-remove @@ -12,7 +12,6 @@ repo="$1" arch="$2" pkgbases=("${@:3}") -ftppath="$FTP_BASE/$repo/os" vcsrepo="$repo-$arch" if ! check_repo_permission "$repo"; then From e504fa54e63f93eebe5a634c6edce001556753b0 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Fri, 5 Mar 2021 00:13:38 +0100 Subject: [PATCH 20/24] wip: tests: extend db-remove debug pkg test Signed-off-by: Morten Linderud --- test/cases/db-remove.bats | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index 6f3bb42e..5f2674a4 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -25,7 +25,8 @@ load ../lib/common @test "remove debug package" { local arches=('i686' 'x86_64') - local pkgs=('pkg-debuginfo') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch' 'pkg-debuginfo') + local debug_pkgs=('pkg-debuginfo') local pkgbase local arch @@ -42,7 +43,9 @@ load ../lib/common done checkRemovedPackage extra pkg-debuginfo - checkRemovedPackage extra-debug pkg-debuginfo + for pkgbase in ${debug_pkgs[@]}; do + checkRemovedPackage extra-debug ${pkgbase} + done } @test "remove multiple packages" { From 6f544f39cd0f4ca317c5bba3ec8fc1336f45a736 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sun, 7 Mar 2021 17:30:25 +0100 Subject: [PATCH 21/24] wip: debug: Added db-repo-add test case Signed-off-by: Morten Linderud --- db-functions | 2 +- test/cases/db-repo-add.bats | 32 ++++++++++++++++++++++++++++++++ test/lib/common.bash | 2 ++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index 3a3269e3..632153b6 100644 --- a/db-functions +++ b/db-functions @@ -433,7 +433,7 @@ check_repo_permission() { (( ${#PKGREPOS[@]} == 0 )) && return 1 [[ -z "${PKGPOOL}" ]] && return 1 - in_array "${repo}" "${PKGREPOS[@]}" || return 1 + in_array "${repo}" "${PKGREPOS[@]}" "${DEBUGREPOS[@]}" || return 1 [[ -w $FTP_BASE/${PKGPOOL} ]] || return 1 diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats index e49c2fad..1217588f 100644 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -1,11 +1,16 @@ load ../lib/common +is_globfile() { + [[ -f $1 ]] +} + __movePackageToRepo() { local repo=$1 local pkgbase=$2 local arch=$3 local tarch local tarches + local is_debug=0 if [[ $arch == any ]]; then tarches=(${ARCHES[@]}) @@ -14,10 +19,18 @@ __movePackageToRepo() { fi # FIXME: pkgbase might not be part of the package filename + if is_globfile "${STAGING}"/${repo}/${pkgbase}-debug-*-*-${arch}${PKGEXT}; then + mv -v "${STAGING}"/${repo}/${pkgbase}-debug-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${PKGPOOL}-debug/" + is_debug=1 + fi mv -v "${STAGING}"/${repo}/${pkgbase}-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${PKGPOOL}/" for tarch in ${tarches[@]}; do ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT} "${FTP_BASE}/${repo}/os/${tarch}/" ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT}.sig "${FTP_BASE}/${repo}/os/${tarch}/" + if ((is_debug)); then + ln -sv ${FTP_BASE}/${PKGPOOL}-debug/${pkgbase}-*-*-${arch}${PKGEXT} "${FTP_BASE}/${repo}-debug/os/${tarch}/" + ln -sv ${FTP_BASE}/${PKGPOOL}-debug/${pkgbase}-*-*-${arch}${PKGEXT}.sig "${FTP_BASE}/${repo}-debug/os/${tarch}/" + fi done } @@ -40,6 +53,25 @@ __movePackageToRepo() { done } +@test "add debug packages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + for arch in ${arches[@]}; do + __movePackageToRepo extra ${pkgbase} ${arch} + db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}${PKGEXT} + db-repo-add extra-debug ${arch} ${pkgbase}-debug-1-1-${arch}${PKGEXT} + done + done + + checkPackageDB extra ${pkgbase} 1-1 + checkPackageDB extra-debug ${pkgbase} 1-1 +} + @test "add multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') diff --git a/test/lib/common.bash b/test/lib/common.bash index 15d36b73..fdf427b6 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -103,7 +103,9 @@ setup() { ARCHIVEUSER="" SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=('core' 'extra' 'testing' 'staging') + DEBUGREPOS=('core-debug' 'extra-debug' 'testing-debug' 'staging-debug') PKGPOOL='pool/packages' + DEBUGPKGPOOL='pool/packages-debug' SRCPOOL='sources/packages' STAGING_REPOS=('staging') TESTING_REPOS=('testing') From 7a24942485bc5d0a20bd5d5611954823764892eb Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sun, 7 Mar 2021 17:45:41 +0100 Subject: [PATCH 22/24] wip: debug: add db-repo-remove test Signed-off-by: Morten Linderud --- test/cases/db-repo-remove.bats | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/cases/db-repo-remove.bats b/test/cases/db-repo-remove.bats index 77bcc241..262c4f15 100644 --- a/test/cases/db-repo-remove.bats +++ b/test/cases/db-repo-remove.bats @@ -23,6 +23,30 @@ load ../lib/common done } + +@test "remove debug packages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-repo-remove extra ${arch} ${pkgbase} + db-repo-remove extra-debug ${arch} ${pkgbase}-debug + done + done + + checkRemovedPackageDB extra ${pkgbase} + checkRemovedPackageDB extra-debug ${pkgbase}-debug +} + @test "remove multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') From 79cee7faeab3ef2da72eb9ab7ab817bc6f7b160f Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sun, 7 Mar 2021 17:47:10 +0100 Subject: [PATCH 23/24] wip: debug: Added testing2x test suite Signed-off-by: Morten Linderud --- test/cases/testing2x.bats | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/cases/testing2x.bats b/test/cases/testing2x.bats index 6127cf96..c0db45c3 100644 --- a/test/cases/testing2x.bats +++ b/test/cases/testing2x.bats @@ -14,3 +14,21 @@ load ../lib/common checkPackage core pkg-any-a 1-2 checkRemovedPackage testing pkg-any-a } + + +@test "move debug package" { + releasePackage core pkg-debuginfo + db-update + + updatePackage pkg-debuginfo + + releasePackage testing pkg-debuginfo + db-update + + testing2x pkg-debuginfo + + checkPackage core pkg-debuginfo 1-2 + checkRemovedPackage testing pkg-debuginfo + checkPackage core-debug pkg-debuginfo 1-2 + checkRemovedPackage testing-debug pkg-debuginfo +} From 51b3373fe88653d23e2c508b1defd3877e776946 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sun, 7 Mar 2021 17:53:11 +0100 Subject: [PATCH 24/24] wip: debug: found internal is_globfile Signed-off-by: Morten Linderud --- test/cases/db-repo-add.bats | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats index 1217588f..33834337 100644 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -1,9 +1,5 @@ load ../lib/common -is_globfile() { - [[ -f $1 ]] -} - __movePackageToRepo() { local repo=$1 local pkgbase=$2 @@ -19,7 +15,7 @@ __movePackageToRepo() { fi # FIXME: pkgbase might not be part of the package filename - if is_globfile "${STAGING}"/${repo}/${pkgbase}-debug-*-*-${arch}${PKGEXT}; then + if __isGlobfile "${STAGING}"/${repo}/${pkgbase}-debug-*-*-${arch}${PKGEXT}; then mv -v "${STAGING}"/${repo}/${pkgbase}-debug-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${PKGPOOL}-debug/" is_debug=1 fi