Skip to content

Commit

Permalink
Switch to building stage repository by combining production and diff …
Browse files Browse the repository at this point in the history
…of Copr

When the version is nightly, this will not pull from production but instead
treat what is in Copr as the source of truth. At the end, the output for
a versioned repository is a list of unsigned packages.
  • Loading branch information
ehelms committed Oct 2, 2023
1 parent 791bc36 commit 706ec5f
Show file tree
Hide file tree
Showing 4 changed files with 234 additions and 52 deletions.
203 changes: 151 additions & 52 deletions build_stage_repository
Original file line number Diff line number Diff line change
Expand Up @@ -11,33 +11,19 @@ import hashlib
import sys
import tempfile
import dnf.comps
import argparse


def move_rpms_from_copr_to_stage(collection, version, src_folder, dest_folder, dist, arch, source=False):
if source:
print(f"Moving {collection} Source RPMs from Copr directory to stage repository")
else:
print(f"Moving {collection} RPMs from Copr directory to stage repository")

if not os.path.exists(dest_folder):
os.mkdir(dest_folder)

repo_folder = f"{src_folder}/{dist}-{collection}-{version}-{arch}"
packages_to_include = packages_from_comps(comps(collection, version, dist))

if source:
files = glob.glob(repo_folder + f"/**/*.src.rpm")
else:
files = glob.glob(repo_folder + f"/**/*.rpm")
def filter_packages(target_dir, packages):
files = glob.glob(f"{target_dir}/*.rpm")

for file in files:
file_name = os.path.basename(file)
rpm_name = get_rpm_name(file)

if rpm_name in packages_to_include:
shutil.move(file, os.path.join(dest_folder, file_name))

shutil.rmtree(src_folder)
if rpm_name not in packages:
print(f"Removed {rpm_name}. Package not in comps.")
os.remove(os.path.join(target_dir, file_name))


def get_rpm_name(rpm):
Expand All @@ -62,10 +48,12 @@ def modulemd_yaml(collection, version):
def comps(collection, version, dist):
branch = 'develop' if version == 'nightly' else version

if collection == 'foreman-katello':
name = 'katello'
elif collection == 'foreman-candlepin':
if collection == 'candlepin':
name = 'katello-candlepin'
elif collection == 'client':
name = 'foreman-client'
elif collection == 'plugins':
name = 'foreman-plugins'
else:
name = collection

Expand Down Expand Up @@ -121,28 +109,30 @@ def create_modulemd(collection, version, stage_dir):


def create_repository(repo_dir):
check_output(['createrepo', repo_dir])
check_output(['createrepo_c', repo_dir], stderr=STDOUT)


def sync_copr_repository(collection, version, target_dir, dist, arch, source=False):
if source:
print(f"Syncing {collection} {version} Source RPM repository from Copr")
def sync_prod_repository(collection, version, target_dir, dist, arch):
if arch == 'source':
print(f"Syncing {collection} {version} Source RPM repository from yum.theforeman.org")
else:
print(f"Syncing {collection} {version} RPM repository from Copr")
print(f"Syncing {collection} {version} RPM repository from yum.theforeman.org")

cmd = [
'dnf',
'reposync',
'--newest-only',
'--refresh',
'--download-metadata',
'--norepopath',
'--repo',
f"{dist}-{collection}-{version}-{arch}",
'--repofrompath',
f"{dist}-{collection}-{version}-{arch},https://download.copr.fedorainfracloud.org/results/@theforeman/{collection}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}/",
f"{dist}-{collection}-{version}-{arch},{prod_repository(collection, version, dist, arch)}",
'--download-path',
target_dir
]

if source:
if arch == 'source':
cmd.extend([
'--source',
])
Expand Down Expand Up @@ -172,40 +162,149 @@ def packages_from_comps(comps):
return packages


def copr_repository(collection, version, dist, arch):
if collection == 'candlepin':
name = 'katello-candlepin'
elif collection == 'client':
name = 'foreman-client'
elif collection == 'plugins':
name = 'foreman-plugins'
else:
name = collection

return f"https://download.copr.fedorainfracloud.org/results/@theforeman/{name}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}"


def prod_repository(collection, version, dist, arch):
if collection == "foreman" and version == "nightly":
return f"https://yum.theforeman.org/{version}/{dist}/{arch}"
else:
return f"https://yum.theforeman.org/{collection.replace('foreman-', '')}/{version}/{dist}/{arch}"


def copr_repository_urls(repository):
cmd = [
'dnf',
'reposync',
'--urls',
'--refresh',
'--repofrompath',
f"copr,{repository}",
'--repo',
'copr'
]

urls = check_output(cmd, universal_newlines=True, stderr=STDOUT)
return urls.split("\n")


def compare_repositories(new_repository, old_repository, source=False):
cmd = [
'dnf',
'repodiff',
'--simple',
'--refresh',
'--compare-arch',
'--repofrompath',
f"new,{new_repository}",
'--repofrompath',
f"old,{old_repository}",
'--repo-old',
'old',
'--repo-new',
'new'
]

if not source:
cmd.extend(['--arch', 'noarch,x86_64'])

print(' '.join(cmd))
return check_output(cmd, universal_newlines=True, stderr=STDOUT)


def parse_repodiff(diff):
split_diff = diff.split("\n")
packages = []

for line in split_diff:
if line.startswith('Added package :'):
packages.append(line.replace('Added package : ', ''))

if ' -> ' in line:
packages.append(line.split(' -> ')[1])

return packages


def download_copr_packages(packages, urls, repository, downloads_dir, included_packages):
for package in packages:
name = f"{package}.rpm".replace("-1:", "-")

for url in urls:
if name in url:
download_url = url

if not os.path.exists(f"{downloads_dir}/{name}"):
print(f"Downloading {repository}/{name} to {downloads_dir}")
urlretrieve(download_url, f"{downloads_dir}/{name}")
else:
print(f"Skipping {repository}/{name}. Already downloaded.")


def handle_args():
parser = argparse.ArgumentParser(description='Generate a stage repository')
parser.add_argument(
'collection',
help='Repository to generate for (e.g. foreman)'
)
parser.add_argument(
'version',
help='Version to generate the repository for (e.g. nightly)'
)
parser.add_argument(
'dist',
help='Dist to generate repository for (e.g. el8)'
)

return parser.parse_args()


def main():
try:
collection = sys.argv[1]
version = sys.argv[2]
dist = sys.argv[3]
arch = 'x86_64'
except IndexError:
raise SystemExit(f"Usage: {sys.argv[0]} collection version os")
args = handle_args()

base_dir = 'tmp'
rpm_sync_dir = f"{base_dir}/rpms"
srpm_sync_dir = f"{base_dir}/srpms"
collection = args.collection
version = args.version
dist = args.dist
arch = 'x86_64'

stage_dir = f"{base_dir}/{collection}/{version}/{dist}/"
base_dir = 'tmp'
stage_dir = f"{base_dir}/{collection}/{version}/{dist}"
rpm_dir = f"{stage_dir}/{arch}"
srpm_dir = f"{stage_dir}/source"

if not os.path.exists(rpm_sync_dir):
os.makedirs(rpm_sync_dir)

if not os.path.exists(srpm_sync_dir):
os.makedirs(srpm_sync_dir)

if not os.path.exists(rpm_dir):
os.makedirs(rpm_dir)

if not os.path.exists(srpm_dir):
os.makedirs(srpm_dir)

sync_copr_repository(collection, version, rpm_sync_dir, dist, arch)
sync_copr_repository(collection, version, srpm_sync_dir, dist, arch, source=True)
if version != 'nightly':
sync_prod_repository(collection, version, rpm_dir, dist, arch)
sync_prod_repository(collection, version, srpm_dir, dist, 'source')

move_rpms_from_copr_to_stage(collection, version, srpm_sync_dir, srpm_dir, dist, arch, source=True)
move_rpms_from_copr_to_stage(collection, version, rpm_sync_dir, rpm_dir, dist, arch)
packages_to_include = packages_from_comps(comps(collection, version, dist))
copr_repo = copr_repository(collection, version, dist, arch)
copr_package_urls = copr_repository_urls(copr_repo)

rpm_diff = compare_repositories(copr_repo, rpm_dir)
packages = parse_repodiff(rpm_diff)
download_copr_packages(packages, copr_package_urls, copr_repo, rpm_dir, packages_to_include)
filter_packages(rpm_dir, packages_to_include)

srpm_diff = compare_repositories(copr_repo, srpm_dir, True)
srpm_packages = parse_repodiff(srpm_diff)
download_copr_packages(srpm_packages, copr_package_urls, copr_repo, srpm_dir, packages_to_include)
filter_packages(srpm_dir, packages_to_include)

create_repository(rpm_dir)
create_repository(srpm_dir)
Expand Down
63 changes: 63 additions & 0 deletions list_unsigned_rpms
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env python3

from subprocess import check_output, STDOUT, CalledProcessError
import argparse
import glob


def find_unsigned_packages(target_dir, gpgkey):
unsigned = []
packages = glob.glob(f"{target_dir}/*.rpm")

for package in packages:
cmd = [
'rpm',
'-qpi',
package
]

output = check_output(cmd, universal_newlines=True, stderr=STDOUT)

if gpgkey.lower() not in output:
unsigned.append(package)

return unsigned


def handle_args():
parser = argparse.ArgumentParser(description='Sign unsigned RPMs in local stage repository')
parser.add_argument(
'repository_path',
help='Path to repository to sign unsigned RPMs'
)
parser.add_argument(
'gpgkey',
help='Short form gpgkey for the version being generated, does not apply to nightly'
)

args = parser.parse_args()

if args.gpgkey:
if len(args.gpgkey) != 8:
raise SystemExit("GPG key must be in short form and 8 characters long")

return args


def main():
args = handle_args()

repository_path = args.repository_path
gpgkey = args.gpgkey

unsigned_rpms = find_unsigned_packages(f"{repository_path}/x86_64", gpgkey)
unsigned_srpms = find_unsigned_packages(f"{repository_path}/source", gpgkey)

for rpm in unsigned_rpms:
print(rpm)
for srpm in unsigned_srpms:
print(srpm)


if __name__ == '__main__':
main()
12 changes: 12 additions & 0 deletions sign_stage_rpms
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash -e

. settings

echo "./list_unsigned_rpms $1 $GPGKEY"
UNSIGNED_RPMS=($(./list_unsigned_rpms "$1" "$GPGKEY"))

Check warning

Code scanning / shellcheck

Prefer mapfile or read -a to split command output (or quote to avoid splitting). Warning

Prefer mapfile or read -a to split command output (or quote to avoid splitting).

for rpm_path in "${UNSIGNED_RPMS[@]}"
do
echo "Signing $rpm_path"
./sign_rpms "$rpm_path"
done
8 changes: 8 additions & 0 deletions upload_stage_rpms
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash -e

. settings

USER='yumrepostage'
HOST='web01.osuosl.theforeman.org'

rsync --archive --verbose --partial --one-file-system --delete-after "$PROJECT/$VERSION" "$USER@$HOST:rsync_cache/$PROJECT"

0 comments on commit 706ec5f

Please sign in to comment.