Switch to building stage repository by combining production and diff …

…of Copr

When the version is nightly, this will not pull from production but instead
treat what is in Copr as the source of truth. At the end, the output for
a versioned repository is a list of unsigned packages.

build_stage_repository

@@ -11,33 +11,19 @@ import hashlib
 import sys
 import tempfile
 import dnf.comps
+import argparse
 
 
-def move_rpms_from_copr_to_stage(collection, version, src_folder, dest_folder, dist, arch, source=False):
-    if source:
-        print(f"Moving {collection} Source RPMs from Copr directory to stage repository")
-    else:
-        print(f"Moving {collection} RPMs from Copr directory to stage repository")
-
-    if not os.path.exists(dest_folder):
-        os.mkdir(dest_folder)
-
-    repo_folder = f"{src_folder}/{dist}-{collection}-{version}-{arch}"
-    packages_to_include = packages_from_comps(comps(collection, version, dist))
-
-    if source:
-        files = glob.glob(repo_folder + f"/**/*.src.rpm")
-    else:
-        files = glob.glob(repo_folder + f"/**/*.rpm")
+def filter_packages(target_dir, packages):
+    files = glob.glob(f"{target_dir}/*.rpm")
 
     for file in files:
         file_name = os.path.basename(file)
         rpm_name = get_rpm_name(file)
 
-        if rpm_name in packages_to_include:
-            shutil.move(file, os.path.join(dest_folder, file_name))
-
-    shutil.rmtree(src_folder)
+        if rpm_name not in packages:
+            print(f"Removed {rpm_name}. Package not in comps.")
+            os.remove(os.path.join(target_dir, file_name))
 
 
 def get_rpm_name(rpm):
@@ -61,18 +47,24 @@ def modulemd_yaml(collection, version):
 
 def comps(collection, version, dist):
     branch = 'develop' if version == 'nightly' else version
-
-    if collection == 'foreman-katello':
-        name = 'katello'
-    elif collection == 'foreman-candlepin':
-        name = 'katello-candlepin'
-    else:
-        name = collection
-
-    if collection == 'foreman-client' and dist == 'el7':
+    repo = 'foreman-packaging'
+    name = collection
+
+    if collection == 'client':
+        name = 'foreman-client'
+    elif collection == 'plugins':
+        name = 'foreman-plugins'
+    elif collection == 'candlepin':
+        repo = 'candlepin-packaging'
+    elif collection == 'pulpcore':
+        repo = 'pulpcore-packaging'
+
+    if collection == 'client' and dist == 'el7':
         dist = 'rhel7'
 
-    path, headers = urlretrieve(f"https://raw.githubusercontent.com/theforeman/foreman-packaging/rpm/{branch}/comps/comps-{name}-{dist}.xml")
+    url = f"https://raw.githubusercontent.com/theforeman/{repo}/rpm/{branch}/comps/comps-{name}-{dist}.xml"
+    print(f"Fetching comps: {url}")
+    path, headers = urlretrieve(url)
     return path
 
 
@@ -121,28 +113,30 @@ def create_modulemd(collection, version, stage_dir):
 
 
 def create_repository(repo_dir):
-    check_output(['createrepo', repo_dir])
+    check_output(['createrepo_c', repo_dir], stderr=STDOUT)
 
 
-def sync_copr_repository(collection, version, target_dir, dist, arch, source=False):
-    if source:
-        print(f"Syncing {collection} {version} Source RPM repository from Copr")
+def sync_prod_repository(collection, version, target_dir, dist, arch):
+    if arch == 'source':
+        print(f"Syncing {collection} {version} Source RPM repository from yum.theforeman.org")
     else:
-        print(f"Syncing {collection} {version} RPM repository from Copr")
+        print(f"Syncing {collection} {version} RPM repository from yum.theforeman.org")
 
     cmd = [
         'dnf',
         'reposync',
-        '--newest-only',
+        '--refresh',
+        '--download-metadata',
+        '--norepopath',
         '--repo',
         f"{dist}-{collection}-{version}-{arch}",
         '--repofrompath',
-        f"{dist}-{collection}-{version}-{arch},https://download.copr.fedorainfracloud.org/results/@theforeman/{collection}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}/",
+        f"{dist}-{collection}-{version}-{arch},{prod_repository(collection, version, dist, arch)}",
         '--download-path',
         target_dir
     ]
 
-    if source:
+    if arch == 'source':
         cmd.extend([
             '--source',
         ])
@@ -172,40 +166,143 @@ def packages_from_comps(comps):
     return packages
 
 
+def copr_repository(collection, version, dist, arch):
+    return f"https://download.copr.fedorainfracloud.org/results/@theforeman/{collection}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}"
+
+
+def prod_repository(collection, version, dist, arch):
+    if collection == "foreman" and version == "nightly":
+        return f"https://yum.theforeman.org/{version}/{dist}/{arch}"
+    else:
+        return f"https://yum.theforeman.org/{collection.replace('foreman-', '')}/{version}/{dist}/{arch}"
+
+
+def copr_repository_urls(repository):
+    cmd = [
+        'dnf',
+        'reposync',
+        '--urls',
+        '--refresh',
+        '--repofrompath',
+        f"copr,{repository}",
+        '--repo',
+        'copr'
+    ]
+
+    urls = check_output(cmd, universal_newlines=True, stderr=STDOUT)
+    return urls.split("\n")
+
+
+def compare_repositories(new_repository, old_repository, source=False):
+    cmd = [
+        'dnf',
+        'repodiff',
+        '--simple',
+        '--refresh',
+        '--compare-arch',
+        '--repofrompath',
+        f"new,{new_repository}",
+        '--repofrompath',
+        f"old,{old_repository}",
+        '--repo-old',
+        'old',
+        '--repo-new',
+        'new'
+    ]
+
+    if not source:
+        cmd.extend(['--arch', 'noarch,x86_64'])
+
+    print(' '.join(cmd))
+    return check_output(cmd, universal_newlines=True)
+
+
+def parse_repodiff(diff):
+    split_diff = diff.split("\n")
+    packages = []
+
+    for line in split_diff:
+        if line.startswith('Added package  :'):
+            packages.append(line.replace('Added package  : ', ''))
+
+        if ' -> ' in line:
+            packages.append(line.split(' -> ')[1])
+
+    return packages
+
+
+def download_copr_packages(packages, urls, repository, downloads_dir, included_packages):
+    for package in packages:
+        name = f"{package}.rpm".replace("-1:", "-")
+
+        for url in urls:
+            if name in url:
+                download_url = url
+
+        if not os.path.exists(f"{downloads_dir}/{name}"):
+            print(f"Downloading {repository}/{name} to {downloads_dir}")
+            urlretrieve(download_url, f"{downloads_dir}/{name}")
+        else:
+            print(f"Skipping {repository}/{name}. Already downloaded.")
+
+
+def handle_args():
+    parser = argparse.ArgumentParser(description='Generate a stage repository')
+    parser.add_argument(
+        'collection',
+        help='Repository to generate for (e.g. foreman)'
+    )
+    parser.add_argument(
+        'version',
+        help='Version to generate the repository for (e.g. nightly)'
+    )
+    parser.add_argument(
+        'dist',
+        help='Dist to generate repository for (e.g. el8)'
+    )
+
+    return parser.parse_args()
+
+
 def main():
-    try:
-        collection = sys.argv[1]
-        version = sys.argv[2]
-        dist = sys.argv[3]
-        arch = 'x86_64'
-    except IndexError:
-        raise SystemExit(f"Usage: {sys.argv[0]} collection version os")
+    args = handle_args()
 
-    base_dir = 'tmp'
-    rpm_sync_dir = f"{base_dir}/rpms"
-    srpm_sync_dir = f"{base_dir}/srpms"
+    collection = args.collection
+    version = args.version
+    dist = args.dist
+    arch = 'x86_64'
 
-    stage_dir = f"{base_dir}/{collection}/{version}/{dist}/"
+    base_dir = 'tmp'
+    stage_dir = f"{base_dir}/{collection}/{version}/{dist}"
     rpm_dir = f"{stage_dir}/{arch}"
     srpm_dir = f"{stage_dir}/source"
 
-    if not os.path.exists(rpm_sync_dir):
-        os.makedirs(rpm_sync_dir)
-
-    if not os.path.exists(srpm_sync_dir):
-        os.makedirs(srpm_sync_dir)
-
     if not os.path.exists(rpm_dir):
         os.makedirs(rpm_dir)
 
     if not os.path.exists(srpm_dir):
         os.makedirs(srpm_dir)
 
-    sync_copr_repository(collection, version, rpm_sync_dir, dist, arch)
-    sync_copr_repository(collection, version, srpm_sync_dir, dist, arch, source=True)
+    if version != 'nightly':
+        sync_prod_repository(collection, version, rpm_dir, dist, arch)
+        sync_prod_repository(collection, version, srpm_dir, dist, 'source')
+    else:
+        create_repository(rpm_dir)
+        create_repository(srpm_dir)
 
-    move_rpms_from_copr_to_stage(collection, version, srpm_sync_dir, srpm_dir, dist, arch, source=True)
-    move_rpms_from_copr_to_stage(collection, version, rpm_sync_dir, rpm_dir, dist, arch)
+    packages_to_include = packages_from_comps(comps(collection, version, dist))
+    copr_repo = copr_repository(collection, version, dist, arch)
+    copr_package_urls = copr_repository_urls(copr_repo)
+
+    rpm_diff = compare_repositories(copr_repo, rpm_dir)
+    packages = parse_repodiff(rpm_diff)
+    download_copr_packages(packages, copr_package_urls, copr_repo, rpm_dir, packages_to_include)
+    filter_packages(rpm_dir, packages_to_include)
+
+    srpm_diff = compare_repositories(copr_repo, srpm_dir, True)
+    srpm_packages = parse_repodiff(srpm_diff)
+    download_copr_packages(srpm_packages, copr_package_urls, copr_repo, srpm_dir, packages_to_include)
+    filter_packages(srpm_dir, packages_to_include)
 
     create_repository(rpm_dir)
     create_repository(srpm_dir)

list_unsigned_rpms

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+from subprocess import check_output, STDOUT, CalledProcessError
+import argparse
+import glob
+
+
+def find_unsigned_packages(target_dir, gpgkey):
+    packages = glob.glob(f"{target_dir}/*.rpm")
+
+    for package in packages:
+        cmd = [
+            'rpm',
+            '-qpi',
+            package
+        ]
+
+        output = check_output(cmd, universal_newlines=True, stderr=STDOUT)
+
+        if gpgkey.lower() not in output:
+            yield package
+
+
+def handle_args():
+    parser = argparse.ArgumentParser(description='Sign unsigned RPMs in local stage repository')
+    parser.add_argument(
+        'repository_path',
+        help='Path to repository to sign unsigned RPMs'
+    )
+    parser.add_argument(
+        'gpgkey',
+        help='Short form gpgkey for the version being generated'
+    )
+
+    args = parser.parse_args()
+
+    if len(args.gpgkey) != 8:
+        raise SystemExit("GPG key must be in short form and 8 characters long")
+
+    return args
+
+
+def main():
+    args = handle_args()
+
+    repository_path = args.repository_path
+    gpgkey = args.gpgkey
+
+
+    for rpm in find_unsigned_packages(f"{repository_path}/x86_64", gpgkey):
+        print(rpm)
+    for srpm in find_unsigned_packages(f"{repository_path}/source", gpgkey):
+        print(srpm)
+
+
+if __name__ == '__main__':
+    main()

sign_stage_rpms

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+. settings
+
+echo "./list_unsigned_rpms $1 $GPGKEY"
+UNSIGNED_RPMS=$(./list_unsigned_rpms "$1" "$GPGKEY")
+
+for rpm_path in "${UNSIGNED_RPMS[@]}"
+do
+  echo "Signing $rpm_path"
+  ./sign_rpms "$rpm_path"
+done

upload_stage_rpms

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+. settings
+
+USER='yumrepostage'
+HOST='web01.osuosl.theforeman.org'
+
+rsync --archive --verbose --partial --one-file-system --delete-after "$PROJECT/$VERSION" "$USER@$HOST:rsync_cache/$PROJECT"