From 8a8048e204018e35aed122bd0a5630af852f557d Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Wed, 28 Aug 2019 11:26:26 -0300 Subject: [PATCH] publishing kickstarts closes #5206 https://pulp.plan.io/issues/5206 Required PR: https://github.com/pulp/pulp_rpm/pull/1418 Required PR: https://github.com/pulp/pulp_rpm/pull/1427 Required PR: https://github.com/pulp/pulp_rpm/pull/1440 --- CHANGES/5206.feature | 1 + pulp_rpm/app/tasks/publishing.py | 309 +++++++++++++++++++++---------- pulp_rpm/app/tasks/utils.py | 131 +++++++++++++ 3 files changed, 341 insertions(+), 100 deletions(-) create mode 100644 CHANGES/5206.feature diff --git a/CHANGES/5206.feature b/CHANGES/5206.feature new file mode 100644 index 000000000..3dea046c9 --- /dev/null +++ b/CHANGES/5206.feature @@ -0,0 +1 @@ +Sync and Publish kickstart trees. diff --git a/pulp_rpm/app/tasks/publishing.py b/pulp_rpm/app/tasks/publishing.py index 3def5918d..1b93b0b62 100644 --- a/pulp_rpm/app/tasks/publishing.py +++ b/pulp_rpm/app/tasks/publishing.py @@ -16,7 +16,14 @@ from pulpcore.plugin.tasking import WorkingDirectory -from pulp_rpm.app.models import Package, RepoMetadataFile, RpmPublication, UpdateRecord +from pulp_rpm.app.models import ( + DistributionTree, + Package, + RepoMetadataFile, + RpmPublication, + UpdateRecord, +) +from pulp_rpm.app.tasks.utils import create_treeinfo log = logging.getLogger(__name__) @@ -30,6 +37,8 @@ class PublicationData: Attributes: publication (pulpcore.plugin.models.Publication): A Publication to populate. packages (pulp_rpm.models.Package): A list of published packages. + published_artifacts (pulpcore.plugin.models.PublishedArtifact): A published artifacts list. + sub_repos (list): A list of tuples with sub_repos data. repomdrecords (list): A list of tuples with repomdrecords data. """ @@ -44,16 +53,27 @@ def __init__(self, publication): """ self.publication = publication self.packages = [] + self.published_artifacts = [] + self.sub_repos = [] self.repomdrecords = [] - def prepare_metadata_files(self): + def prepare_metadata_files(self, contents, folder=None): """ Copies metadata files from the Artifact storage. + Args: + contents (pulpcore.plugin.models.Content): A list of contents. + + Keyword Args: + folder(str): name of the directory. + + Returns: + repomdrecords (list): A list of tuples with repomdrecords data. + """ - publication = self.publication + repomdrecords = [] repo_metadata_files = RepoMetadataFile.objects.filter( - pk__in=publication.repository_version.content).prefetch_related('contentartifact_set') + pk__in=contents).prefetch_related('contentartifact_set') for repo_metadata_file in repo_metadata_files: content_artifact = repo_metadata_file.contentartifact_set.get() @@ -61,9 +81,29 @@ def prepare_metadata_files(self): path = content_artifact.relative_path.split("/")[-1] if repo_metadata_file.checksum in path: path = path.split("-")[-1] + if folder: + path = os.path.join(folder, path) with open(path, "wb") as new_file: shutil.copyfileobj(current_file, new_file) - self.repomdrecords.append((repo_metadata_file.data_type, new_file.name, None)) + repomdrecords.append((repo_metadata_file.data_type, new_file.name, None)) + + return repomdrecords + + def get_packages(self, contents): + """ + Get packages from content. + + Args: + contents (pulpcore.plugin.models.Content): A list of contents. + + Returns: + packages (pulp_rpm.models.Package): A list of packages. + + """ + packages = Package.objects.filter(pk__in=contents).\ + prefetch_related('contentartifact_set') + + return packages def populate(self): """ @@ -73,20 +113,62 @@ def populate(self): """ publication = self.publication + main_content = publication.repository_version.content + + distribution_trees = DistributionTree.objects.filter( + pk__in=publication.repository_version.content + ).prefetch_related( + "addons", + "variants", + "addons__repository", + "variants__repository", + "contentartifact_set" + ) + + for distribution_tree in distribution_trees: + for content_artifact in distribution_tree.contentartifact_set.all(): + self.published_artifacts.append(PublishedArtifact( + relative_path=content_artifact.relative_path, + publication=publication, + content_artifact=content_artifact) + ) + for addon in distribution_tree.addons.all(): + repository_version = RepositoryVersion.latest(addon.repository) + if repository_version and repository_version.content != main_content: + self.sub_repos.append((addon.addon_id, repository_version.content)) + for variant in distribution_tree.variants.all(): + repository_version = RepositoryVersion.latest(variant.repository) + if repository_version and repository_version.content != main_content: + self.sub_repos.append((variant.variant_id, repository_version.content)) + + treeinfo_file = create_treeinfo(distribution_tree) + metadata = PublishedMetadata( + relative_path=treeinfo_file.name, + publication=publication, + file=File(open(treeinfo_file.name, 'rb')) + ) + metadata.save() - self.packages = Package.objects.filter(pk__in=publication.repository_version.content).\ - prefetch_related('contentartifact_set') - published_artifacts = [] + self.packages = self.get_packages(main_content) + self.repomdrecords = self.prepare_metadata_files(main_content) - for package in self.packages: + all_packages = self.packages + for name, content in self.sub_repos: + os.mkdir(name) + sub_repo_packages = self.get_packages(content) + all_packages = all_packages | sub_repo_packages + setattr(self, f"{name}_packages", sub_repo_packages) + setattr(self, f"{name}_repomdrecords", self.prepare_metadata_files(content, name)) + + for package in all_packages.distinct(): for content_artifact in package.contentartifact_set.all(): - published_artifacts.append(PublishedArtifact( + self.published_artifacts.append(PublishedArtifact( relative_path=content_artifact.relative_path, - publication=publication, + publication=self.publication, content_artifact=content_artifact) ) - PublishedArtifact.objects.bulk_create(published_artifacts) + PublishedArtifact.objects.bulk_create(self.published_artifacts) def update_record_xml(update_record): @@ -165,95 +247,122 @@ def publish(repository_version_pk): with RpmPublication.create(repository_version) as publication: publication_data = PublicationData(publication) publication_data.populate() + packages = publication_data.packages - # Prepare metadata files - repomd_path = os.path.join(os.getcwd(), "repomd.xml") - pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz") - fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz") - oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz") - pri_db_path = os.path.join(os.getcwd(), "primary.sqlite") - fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite") - oth_db_path = os.path.join(os.getcwd(), "other.sqlite") - upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz") - - pri_xml = cr.PrimaryXmlFile(pri_xml_path) - fil_xml = cr.FilelistsXmlFile(fil_xml_path) - oth_xml = cr.OtherXmlFile(oth_xml_path) - pri_db = cr.PrimarySqlite(pri_db_path) - fil_db = cr.FilelistsSqlite(fil_db_path) - oth_db = cr.OtherSqlite(oth_db_path) - upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) - - pri_xml.set_num_of_pkgs(len(packages)) - fil_xml.set_num_of_pkgs(len(packages)) - oth_xml.set_num_of_pkgs(len(packages)) - - # Process all packages - for package in packages: - pkg = package.to_createrepo_c() - pkg.location_href = package.contentartifact_set.first().relative_path - pri_xml.add_pkg(pkg) - fil_xml.add_pkg(pkg) - oth_xml.add_pkg(pkg) - pri_db.add_pkg(pkg) - fil_db.add_pkg(pkg) - oth_db.add_pkg(pkg) - - # Process update records - for update_record in UpdateRecord.objects.filter( - pk__in=publication.repository_version.content): - upd_xml.add_chunk(update_record_xml(update_record)) - - pri_xml.close() - fil_xml.close() - oth_xml.close() - upd_xml.close() - - repomd = cr.Repomd() - - publication_data.prepare_metadata_files() - - repomdrecords = [("primary", pri_xml_path, pri_db), - ("filelists", fil_xml_path, fil_db), - ("other", oth_xml_path, oth_db), - ("primary_db", pri_db_path, None), - ("filelists_db", fil_db_path, None), - ("other_db", oth_db_path, None), - ("updateinfo", upd_xml_path, None)] - - repomdrecords.extend(publication_data.repomdrecords) - - sqlite_files = ("primary_db", "filelists_db", "other_db") - for name, path, db_to_update in repomdrecords: - record = cr.RepomdRecord(name, path) - if name in sqlite_files: - record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2) - record_bz.type = name - record_bz.rename_file() - path = record_bz.location_href.split('/')[-1] - repomd.set_record(record_bz) - else: - record.fill(cr.SHA256) - if (db_to_update): - db_to_update.dbinfo_update(record.checksum) - db_to_update.close() - record.rename_file() - path = record.location_href.split('/')[-1] - repomd.set_record(record) - metadata = PublishedMetadata( - relative_path=os.path.join(REPODATA_PATH, os.path.basename(path)), - publication=publication, - file=File(open(os.path.basename(path), 'rb')) - ) - metadata.save() + # Main repo + create_rempomd_xml(packages, publication, publication_data.repomdrecords) - with open(repomd_path, "w") as repomd_f: - repomd_f.write(repomd.xml_dump()) + for sub_repo in publication_data.sub_repos: + name = sub_repo[0] + packages = getattr(publication_data, f"{name}_packages") + extra_repomdrecords = getattr(publication_data, f"{name}_repomdrecords") + create_rempomd_xml(packages, publication, extra_repomdrecords, name) - metadata = PublishedMetadata( - relative_path=os.path.join(REPODATA_PATH, os.path.basename(repomd_path)), - publication=publication, - file=File(open(os.path.basename(repomd_path), 'rb')) - ) - metadata.save() + +def create_rempomd_xml(packages, publication, extra_repomdrecords, sub_folder=None): + """ + Creates a repomd.xml file. + + Args: + packages(app.models.Package): set of packages + publication(pulpcore.plugin.models.Publication): the publication + extra_repomdrecords(list): list with data relative to repo metadata files + sub_folder(str): name of the folder for sub repos + + """ + cwd = os.getcwd() + repodata_path = REPODATA_PATH + + if sub_folder: + cwd = os.path.join(cwd, sub_folder) + repodata_path = os.path.join(repodata_path, sub_folder) + + # Prepare metadata files + repomd_path = os.path.join(cwd, "repomd.xml") + pri_xml_path = os.path.join(cwd, "primary.xml.gz") + fil_xml_path = os.path.join(cwd, "filelists.xml.gz") + oth_xml_path = os.path.join(cwd, "other.xml.gz") + pri_db_path = os.path.join(cwd, "primary.sqlite") + fil_db_path = os.path.join(cwd, "filelists.sqlite") + oth_db_path = os.path.join(cwd, "other.sqlite") + upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") + + pri_xml = cr.PrimaryXmlFile(pri_xml_path) + fil_xml = cr.FilelistsXmlFile(fil_xml_path) + oth_xml = cr.OtherXmlFile(oth_xml_path) + pri_db = cr.PrimarySqlite(pri_db_path) + fil_db = cr.FilelistsSqlite(fil_db_path) + oth_db = cr.OtherSqlite(oth_db_path) + upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) + + pri_xml.set_num_of_pkgs(len(packages)) + fil_xml.set_num_of_pkgs(len(packages)) + oth_xml.set_num_of_pkgs(len(packages)) + + # Process all packages + for package in packages: + pkg = package.to_createrepo_c() + pkg.location_href = package.contentartifact_set.first().relative_path + pri_xml.add_pkg(pkg) + fil_xml.add_pkg(pkg) + oth_xml.add_pkg(pkg) + pri_db.add_pkg(pkg) + fil_db.add_pkg(pkg) + oth_db.add_pkg(pkg) + + # Process update records + for update_record in UpdateRecord.objects.filter( + pk__in=publication.repository_version.content): + upd_xml.add_chunk(update_record_xml(update_record)) + + pri_xml.close() + fil_xml.close() + oth_xml.close() + upd_xml.close() + + repomd = cr.Repomd() + + repomdrecords = [("primary", pri_xml_path, pri_db), + ("filelists", fil_xml_path, fil_db), + ("other", oth_xml_path, oth_db), + ("primary_db", pri_db_path, None), + ("filelists_db", fil_db_path, None), + ("other_db", oth_db_path, None), + ("updateinfo", upd_xml_path, None)] + + repomdrecords.extend(extra_repomdrecords) + + sqlite_files = ("primary_db", "filelists_db", "other_db") + for name, path, db_to_update in repomdrecords: + record = cr.RepomdRecord(name, path) + if name in sqlite_files: + record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2) + record_bz.type = name + record_bz.rename_file() + path = record_bz.location_href.split('/')[-1] + repomd.set_record(record_bz) + else: + record.fill(cr.SHA256) + if (db_to_update): + db_to_update.dbinfo_update(record.checksum) + db_to_update.close() + record.rename_file() + path = record.location_href.split('/')[-1] + repomd.set_record(record) + metadata = PublishedMetadata( + relative_path=os.path.join(repodata_path, os.path.basename(path)), + publication=publication, + file=File(open(os.path.basename(path), 'rb')) + ) + metadata.save() + + with open(repomd_path, "w") as repomd_f: + repomd_f.write(repomd.xml_dump()) + + metadata = PublishedMetadata( + relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)), + publication=publication, + file=File(open(os.path.basename(repomd_path), 'rb')) + ) + metadata.save() diff --git a/pulp_rpm/app/tasks/utils.py b/pulp_rpm/app/tasks/utils.py index 59ce782b9..e71eb9110 100644 --- a/pulp_rpm/app/tasks/utils.py +++ b/pulp_rpm/app/tasks/utils.py @@ -52,6 +52,137 @@ def repodata_exists(remote, url): return True +def create_treeinfo(distribution_tree): + """ + Create treeinfo file. + + Args: + distirbution_tree(app.models.DistributionTree): a distirbution_tree object + + Returns: + f(File): treeinfo file. + + """ + parser = SortedConfigParser() + treeinfo = {} + + treeinfo["header"] = { + "type": "productmd.treeinfo", + "version": distribution_tree.header_version, + } + + treeinfo["release"] = { + "name": distribution_tree.release_name, + "short": distribution_tree.release_short, + "version": distribution_tree.release_version, + "is_layered": distribution_tree.release_is_layered, + } + + if distribution_tree.base_product_name: + treeinfo["base_product"] = { + "name": distribution_tree.base_product_name, + "short": distribution_tree.base_product_short, + "version": distribution_tree.base_product_version, + } + + treeinfo["tree"] = { + "arch": distribution_tree.arch, + "build_timestamp": distribution_tree.build_timestamp, + } + + stage2 = {} + media = {} + + if distribution_tree.instimage: + stage2.update({"instimage": distribution_tree.instimage}) + + if distribution_tree.mainimage: + stage2.update({"mainimage": distribution_tree.mainimage}) + + if stage2: + treeinfo["stage2"] = stage2 + + if distribution_tree.discnum: + media.update({"discnum": distribution_tree.discnum}) + + if distribution_tree.totaldiscs: + media.update({"totaldiscs": distribution_tree.totaldiscs}) + + if media: + treeinfo["media"] = media + + checksums = {} + for checksum in distribution_tree.checksums.all(): + checksums.update({checksum.path: checksum.checksum}) + + if checksums: + treeinfo["checksums"] = checksums + + tree_platforms = set() + for image in distribution_tree.images.all(): + platforms = image.platforms.split(",") + tree_platforms.update(platforms) + + for platform in platforms: + current = treeinfo.get(f"images-{platform}", {}) + current.update({image.name: image.path}) + treeinfo[f"images-{platform}"] = current + + treeinfo["tree"].update({"platforms": ", ".join(tree_platforms)}) + + for addon in distribution_tree.addons.all(): + treeinfo[f"addon-{addon.uid}"] = { + "id": addon.addon_id, + "uid": addon.uid, + "name": addon.name, + "type": addon.type, + "packages": addon.packages, + "repository": addon.addon_id, + } + variants = [] + nullables = [ + "source_packages", "source_repository", "debug_packages", "debug_repository", "identity" + ] + for variant in distribution_tree.variants.all(): + variants.append(variant.uid) + treeinfo[f"variant-{variant.uid}"] = { + "id": variant.variant_id, + "uid": variant.uid, + "name": variant.name, + "type": variant.type, + "packages": variant.packages, + "repository": variant.variant_id, + } + + for nullable in nullables: + if getattr(variant, nullable): + treeinfo[f"variant-{variant.uid}"].update({nullable: getattr(variant, nullable)}) + + treeinfo["tree"].update({"variants": ", ".join(variants)}) + + variants = sorted(variants) + first_variant = treeinfo[f"variant-{variants[0]}"] + first_variant["repository"] = "." + treeinfo["general"] = { + "family": treeinfo["release"]["name"], + "version": treeinfo["release"]["version"], + "name": f"{treeinfo['release']['name']} {treeinfo['release']['version']}", + "arch": treeinfo["tree"]["arch"], + "platforms": treeinfo["tree"]["platforms"], + "packagedir": first_variant["packages"], + "repository": first_variant["repository"], + "timestamp": treeinfo["tree"]["build_timestamp"], + "variant": first_variant["uid"], + } + + parser.read_dict(treeinfo) + + with open(".treeinfo", "w") as f: + parser.write(f) + + return f + + class KickstartData: """ Treat parsed kickstart data.