From ec0ed466399308df9927c31a385926f0ababdd19 Mon Sep 17 00:00:00 2001 From: Dana Walker Date: Mon, 16 Sep 2019 11:26:04 -0400 Subject: [PATCH] Add sync support for content types from comps.xml Updates model relations, adds serializers, constants, viewsets, libcomps parsing, and sync support for comps.xml content types closes #5423 https://pulp.plan.io/issues/5423 --- CHANGES/5423.misc | 1 + pulp_rpm/app/comps.py | 36 +++ pulp_rpm/app/constants.py | 74 +++++ .../app/migrations/0006_auto_20191025_1850.py | 122 ++++++++ .../app/migrations/0007_auto_20191101_1648.py | 23 ++ pulp_rpm/app/models.py | 261 ++++++++++++++++-- pulp_rpm/app/serializers.py | 212 ++++++++++++++ pulp_rpm/app/tasks/synchronizing.py | 190 +++++++++++++ pulp_rpm/app/viewsets.py | 52 ++++ pulp_rpm/tests/functional/constants.py | 30 +- 10 files changed, 973 insertions(+), 28 deletions(-) create mode 100644 CHANGES/5423.misc create mode 100644 pulp_rpm/app/comps.py create mode 100644 pulp_rpm/app/migrations/0006_auto_20191025_1850.py create mode 100644 pulp_rpm/app/migrations/0007_auto_20191101_1648.py diff --git a/CHANGES/5423.misc b/CHANGES/5423.misc new file mode 100644 index 0000000000..0e71c970a9 --- /dev/null +++ b/CHANGES/5423.misc @@ -0,0 +1 @@ +Comps.xml sync diff --git a/pulp_rpm/app/comps.py b/pulp_rpm/app/comps.py new file mode 100644 index 0000000000..25e39fd5c6 --- /dev/null +++ b/pulp_rpm/app/comps.py @@ -0,0 +1,36 @@ +import hashlib + + +def strdict_to_dict(value): + """ + Convert libcomps StrDict type object to standard dict. + + Args: + value: a libcomps StrDict + + Returns: + lang_dict: a dict + + """ + lang_dict = {} + if len(value): + for i, j in value.items(): + lang_dict[i] = j + return lang_dict + + +def dict_digest(dict): + """ + Calculate a hexdigest for a given dictionary. + + Args: + dict: a dictionary + + Returns: + A digest + + """ + prep_hash = list(dict.values()) + str_prep_hash = [str(i) for i in prep_hash] + str_prep_hash.sort() + return hashlib.sha256(''.join(str_prep_hash).encode('utf-8')).hexdigest() diff --git a/pulp_rpm/app/constants.py b/pulp_rpm/app/constants.py index 47ee37d3b7..f5acc45785 100644 --- a/pulp_rpm/app/constants.py +++ b/pulp_rpm/app/constants.py @@ -76,6 +76,8 @@ PACKAGE_DB_REPODATA = ['primary_db', 'filelists_db', 'other_db'] UPDATE_REPODATA = ['updateinfo'] MODULAR_REPODATA = ['modules'] +COMPS_REPODATA = ['group'] +SKIP_REPODATA = ['group_gz'] CR_UPDATE_RECORD_ATTRS = SimpleNamespace( ID='id', @@ -167,3 +169,75 @@ ) PULP_MODULE_ATTR = MODULEMD_MODULE_ATTR + +LIBCOMPS_GROUP_ATTRS = SimpleNamespace( + ID='id', + DEFAULT='default', + USER_VISIBLE='uservisible', + DISPLAY_ORDER='display_order', + NAME='name', + DESCRIPTION='desc', + PACKAGES='packages', + BIARCH_ONLY='biarchonly', + DESC_BY_LANG='desc_by_lang', + NAME_BY_LANG='name_by_lang' +) + +LIBCOMPS_CATEGORY_ATTRS = SimpleNamespace( + ID='id', + NAME='name', + DESCRIPTION='desc', + DISPLAY_ORDER='display_order', + GROUP_IDS='group_ids', + DESC_BY_LANG='desc_by_lang', + NAME_BY_LANG='name_by_lang' +) + +LIBCOMPS_ENVIRONMENT_ATTRS = SimpleNamespace( + ID='id', + NAME='name', + DESCRIPTION='desc', + DISPLAY_ORDER='display_order', + GROUP_IDS='group_ids', + OPTION_IDS='option_ids', + DESC_BY_LANG='desc_by_lang', + NAME_BY_LANG='name_by_lang' +) + +PULP_LANGPACKS_ATTRS = SimpleNamespace( + MATCHES='matches' +) + +PULP_GROUP_ATTRS = SimpleNamespace( + ID='id', + DEFAULT='default', + USER_VISIBLE='user_visible', + DISPLAY_ORDER='display_order', + NAME='name', + DESCRIPTION='description', + PACKAGES='packages', + BIARCH_ONLY='biarch_only', + DESC_BY_LANG='desc_by_lang', + NAME_BY_LANG='name_by_lang' +) + +PULP_CATEGORY_ATTRS = SimpleNamespace( + ID='id', + NAME='name', + DESCRIPTION='description', + DISPLAY_ORDER='display_order', + GROUP_IDS='group_ids', + DESC_BY_LANG='desc_by_lang', + NAME_BY_LANG='name_by_lang' +) + +PULP_ENVIRONMENT_ATTRS = SimpleNamespace( + ID='id', + NAME='name', + DESCRIPTION='description', + DISPLAY_ORDER='display_order', + GROUP_IDS='group_ids', + OPTION_IDS='option_ids', + DESC_BY_LANG='desc_by_lang', + NAME_BY_LANG='name_by_lang' +) diff --git a/pulp_rpm/app/migrations/0006_auto_20191025_1850.py b/pulp_rpm/app/migrations/0006_auto_20191025_1850.py new file mode 100644 index 0000000000..90c6ef4985 --- /dev/null +++ b/pulp_rpm/app/migrations/0006_auto_20191025_1850.py @@ -0,0 +1,122 @@ +# Generated by Django 2.2.6 on 2019-10-25 18:50 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0011_relative_path'), + ('rpm', '0005_pulp_fields'), + ] + + operations = [ + migrations.CreateModel( + name='PackageCategory', + fields=[ + ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_packagecategory', serialize=False, to='core.Content')), + ('id', models.CharField(max_length=255)), + ('name', models.CharField(max_length=255)), + ('description', models.TextField(default='')), + ('display_order', models.IntegerField()), + ('group_ids', django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ('desc_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('name_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('digest', models.CharField(max_length=64, unique=True)), + ], + options={ + 'default_related_name': '%(app_label)s_%(model_name)s', + }, + bases=('core.content',), + ), + migrations.CreateModel( + name='PackageEnvironment', + fields=[ + ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_packageenvironment', serialize=False, to='core.Content')), + ('id', models.CharField(max_length=255)), + ('name', models.CharField(max_length=255)), + ('description', models.TextField(default='')), + ('display_order', models.IntegerField(null=True)), + ('group_ids', django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ('option_ids', django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ('desc_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('name_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('digest', models.CharField(max_length=64, unique=True)), + ], + options={ + 'default_related_name': '%(app_label)s_%(model_name)s', + }, + bases=('core.content',), + ), + migrations.CreateModel( + name='PackageLangpacks', + fields=[ + ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_packagelangpacks', serialize=False, to='core.Content')), + ('matches', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('digest', models.CharField(max_length=64, unique=True)), + ], + options={ + 'default_related_name': '%(app_label)s_%(model_name)s', + }, + bases=('core.content',), + ), + migrations.RemoveField( + model_name='environment', + name='content_ptr', + ), + migrations.RemoveField( + model_name='langpacks', + name='content_ptr', + ), + migrations.AddField( + model_name='packagegroup', + name='related_packages', + field=models.ManyToManyField(related_name='rpm_packagegroup', to='rpm.Package'), + ), + migrations.AlterField( + model_name='packagegroup', + name='desc_by_lang', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='packagegroup', + name='description', + field=models.TextField(default=''), + ), + migrations.AlterField( + model_name='packagegroup', + name='name_by_lang', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='packagegroup', + name='packages', + field=django.contrib.postgres.fields.jsonb.JSONField(default=list), + ), + migrations.DeleteModel( + name='Category', + ), + migrations.DeleteModel( + name='Environment', + ), + migrations.DeleteModel( + name='Langpacks', + ), + migrations.AddField( + model_name='packageenvironment', + name='optionalgroups', + field=models.ManyToManyField(related_name='optionalgroups_to_env', to='rpm.PackageGroup'), + ), + migrations.AddField( + model_name='packageenvironment', + name='packagegroups', + field=models.ManyToManyField(related_name='packagegroups_to_env', to='rpm.PackageGroup'), + ), + migrations.AddField( + model_name='packagecategory', + name='packagegroups', + field=models.ManyToManyField(related_name='rpm_packagecategory', to='rpm.PackageGroup'), + ), + ] diff --git a/pulp_rpm/app/migrations/0007_auto_20191101_1648.py b/pulp_rpm/app/migrations/0007_auto_20191101_1648.py new file mode 100644 index 0000000000..e615195970 --- /dev/null +++ b/pulp_rpm/app/migrations/0007_auto_20191101_1648.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2019-11-01 16:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rpm', '0006_auto_20191025_1850'), + ] + + operations = [ + migrations.AlterField( + model_name='packagecategory', + name='display_order', + field=models.IntegerField(null=True), + ), + migrations.AlterField( + model_name='packagegroup', + name='display_order', + field=models.IntegerField(null=True), + ), + ] diff --git a/pulp_rpm/app/models.py b/pulp_rpm/app/models.py index fd0eeed9cf..da8a8e0bfb 100644 --- a/pulp_rpm/app/models.py +++ b/pulp_rpm/app/models.py @@ -21,6 +21,13 @@ CR_UPDATE_RECORD_ATTRS, CR_UPDATE_COLLECTION_ATTRS_MODULE, CR_UPDATE_REFERENCE_ATTRS, + LIBCOMPS_CATEGORY_ATTRS, + LIBCOMPS_ENVIRONMENT_ATTRS, + LIBCOMPS_GROUP_ATTRS, + PULP_CATEGORY_ATTRS, + PULP_ENVIRONMENT_ATTRS, + PULP_GROUP_ATTRS, + PULP_LANGPACKS_ATTRS, PULP_PACKAGE_ATTRS, PULP_UPDATE_COLLECTION_ATTRS, PULP_UPDATE_COLLECTION_ATTRS_MODULE, @@ -29,6 +36,8 @@ PULP_UPDATE_REFERENCE_ATTRS ) +from pulp_rpm.app.comps import strdict_to_dict + log = getLogger(__name__) @@ -724,6 +733,8 @@ class PackageGroup(Content): A dictionary of names by language digest (Text): A checksum for the group + related_packages (ManyToMany): + Packages related to this PackageGroup """ TYPE = 'packagegroup' @@ -734,23 +745,83 @@ class PackageGroup(Content): default = models.BooleanField(default=False) user_visible = models.BooleanField(default=False) - display_order = models.IntegerField() + display_order = models.IntegerField(null=True) name = models.CharField(max_length=255) - description = models.TextField() - packages = models.TextField() + description = models.TextField(default='') + packages = JSONField(default=list) biarch_only = models.BooleanField(default=False) - desc_by_lang = models.TextField() - name_by_lang = models.TextField() + desc_by_lang = JSONField(default=dict) + name_by_lang = JSONField(default=dict) digest = models.CharField(unique=True, max_length=64) + related_packages = models.ManyToManyField(Package) + class Meta: default_related_name = "%(app_label)s_%(model_name)s" + @classmethod + def natural_key_fields(cls): + """ + Digest is used as a natural key for PackageGroups. + """ + return ('digest',) + + @classmethod + def pkglist_to_lst(cls, value): + """ + Convert libcomps PkgList to list. + + Args: + value: a libcomps PkgList + + Returns: + A list + + """ + package_list = [] + if len(value): + for i in value: + package_list.append({'name': i.name, + 'type': i.type, + 'basearchonly': i.basearchonly, + 'requires': i.requires}) + return package_list + + @classmethod + def libcomps_to_dict(cls, group): + """ + Convert libcomps group object to dict for instantiating PackageGroup object. -class Category(Content): + Args: + group(libcomps.group): a RPM/SRPM group to convert + + Returns: + dict: all data for RPM/SRPM group content creation + + """ + return { + PULP_GROUP_ATTRS.ID: getattr(group, LIBCOMPS_GROUP_ATTRS.ID), + PULP_GROUP_ATTRS.DEFAULT: getattr(group, LIBCOMPS_GROUP_ATTRS.DEFAULT), + PULP_GROUP_ATTRS.USER_VISIBLE: getattr(group, LIBCOMPS_GROUP_ATTRS.USER_VISIBLE), + PULP_GROUP_ATTRS.DISPLAY_ORDER: getattr(group, LIBCOMPS_GROUP_ATTRS.DISPLAY_ORDER), + PULP_GROUP_ATTRS.NAME: getattr(group, LIBCOMPS_GROUP_ATTRS.NAME), + PULP_GROUP_ATTRS.DESCRIPTION: getattr(group, LIBCOMPS_GROUP_ATTRS.DESCRIPTION) or '', + PULP_GROUP_ATTRS.PACKAGES: cls.pkglist_to_lst(getattr(group, + LIBCOMPS_GROUP_ATTRS.PACKAGES)), + PULP_GROUP_ATTRS.BIARCH_ONLY: getattr(group, LIBCOMPS_GROUP_ATTRS.BIARCH_ONLY), + PULP_GROUP_ATTRS.DESC_BY_LANG: strdict_to_dict( + getattr(group, LIBCOMPS_GROUP_ATTRS.DESC_BY_LANG) + ), + PULP_GROUP_ATTRS.NAME_BY_LANG: strdict_to_dict( + getattr(group, LIBCOMPS_GROUP_ATTRS.NAME_BY_LANG) + ), + } + + +class PackageCategory(Content): """ The "Category" content type. Formerly "PackageCategory" in Pulp 2. @@ -775,29 +846,89 @@ class Category(Content): A dictionary of names by language digest (Text): A checksum for the category + packagegroups (ManyToMany): + PackageGroups related to this category """ - TYPE = 'category' + TYPE = 'packagecategory' # Required metadata id = models.CharField(max_length=255) name = models.CharField(max_length=255) - description = models.TextField() - display_order = models.IntegerField() + description = models.TextField(default='') + display_order = models.IntegerField(null=True) group_ids = JSONField(default=list) - desc_by_lang = models.TextField() - name_by_lang = models.TextField() + desc_by_lang = JSONField(default=dict) + name_by_lang = JSONField(default=dict) digest = models.CharField(unique=True, max_length=64) + packagegroups = models.ManyToManyField(PackageGroup) + class Meta: default_related_name = "%(app_label)s_%(model_name)s" + @classmethod + def natural_key_fields(cls): + """ + Digest is used as a natural key for PackageCategory. + """ + return ('digest',) + + @classmethod + def grplist_to_lst(cls, value): + """ + Convert libcomps GrpList to list. + + Args: + value: a libcomps GrpList + + Returns: + A list + + """ + grp_list = [] + if len(value): + for i in value: + grp_list.append({'name': i.name, + 'default': i.default}) + return grp_list + + @classmethod + def libcomps_to_dict(cls, category): + """ + Convert libcomps category object to dict for instantiating PackageCategory object. + + Args: + category(libcomps.category): a RPM/SRPM category to convert + + Returns: + dict: all data for RPM/SRPM category content creation -class Environment(Content): + """ + return { + PULP_CATEGORY_ATTRS.ID: getattr(category, LIBCOMPS_CATEGORY_ATTRS.ID), + PULP_CATEGORY_ATTRS.NAME: getattr(category, LIBCOMPS_CATEGORY_ATTRS.NAME), + PULP_CATEGORY_ATTRS.DESCRIPTION: getattr(category, + LIBCOMPS_CATEGORY_ATTRS.DESCRIPTION) or '', + PULP_CATEGORY_ATTRS.DISPLAY_ORDER: getattr(category, + LIBCOMPS_CATEGORY_ATTRS.DISPLAY_ORDER), + PULP_CATEGORY_ATTRS.GROUP_IDS: cls.grplist_to_lst( + getattr(category, LIBCOMPS_CATEGORY_ATTRS.GROUP_IDS) + ), + PULP_CATEGORY_ATTRS.DESC_BY_LANG: strdict_to_dict( + getattr(category, LIBCOMPS_CATEGORY_ATTRS.DESC_BY_LANG) + ), + PULP_CATEGORY_ATTRS.NAME_BY_LANG: strdict_to_dict( + getattr(category, LIBCOMPS_CATEGORY_ATTRS.NAME_BY_LANG) + ), + } + + +class PackageEnvironment(Content): """ The "Environment" content type. Formerly "PackageEnvironment" in Pulp 2. @@ -824,30 +955,97 @@ class Environment(Content): A dictionary of names by language digest (Text): A checksum for the environment + packagegroups (ManyToMany): + PackageGroups related to this environment + optionalgroups (ManyToMany): + PackageGroups optionally related to this environment """ - TYPE = 'environment' + TYPE = 'packageenvironment' # Required metadata id = models.CharField(max_length=255) name = models.CharField(max_length=255) - description = models.TextField() - display_order = models.IntegerField() + description = models.TextField(default='') + display_order = models.IntegerField(null=True) group_ids = JSONField(default=list) option_ids = JSONField(default=list) - desc_by_lang = models.TextField() - name_by_lang = models.TextField() + desc_by_lang = JSONField(default=dict) + name_by_lang = JSONField(default=dict) digest = models.CharField(unique=True, max_length=64) + packagegroups = models.ManyToManyField(PackageGroup, related_name='packagegroups_to_env') + optionalgroups = models.ManyToManyField(PackageGroup, related_name='optionalgroups_to_env') + class Meta: default_related_name = "%(app_label)s_%(model_name)s" + @classmethod + def natural_key_fields(cls): + """ + Digest is used as a natural key for PackageEnvironment. + """ + return ('digest',) + + @classmethod + def grplist_to_lst(cls, value): + """ + Convert libcomps GrpList to list. + + Args: + value: a libcomps GrpList + + Returns: + A list -class Langpacks(Content): + """ + grp_list = [] + if len(value): + for i in value: + grp_list.append({'name': i.name, + 'default': i.default}) + return grp_list + + @classmethod + def libcomps_to_dict(cls, environment): + """ + Convert libcomps environment object to dict for instantiating PackageEnvironment object. + + Args: + environment(libcomps.environment): a RPM/SRPM environment to convert + + Returns: + dict: all data for RPM/SRPM environment content creation + + """ + return { + PULP_ENVIRONMENT_ATTRS.ID: getattr(environment, LIBCOMPS_ENVIRONMENT_ATTRS.ID), + PULP_ENVIRONMENT_ATTRS.NAME: getattr(environment, LIBCOMPS_ENVIRONMENT_ATTRS.NAME), + PULP_ENVIRONMENT_ATTRS.DESCRIPTION: getattr( + environment, LIBCOMPS_ENVIRONMENT_ATTRS.DESCRIPTION + ) or '', + PULP_ENVIRONMENT_ATTRS.DISPLAY_ORDER: getattr(environment, + LIBCOMPS_ENVIRONMENT_ATTRS.DISPLAY_ORDER), + PULP_ENVIRONMENT_ATTRS.GROUP_IDS: cls.grplist_to_lst( + getattr(environment, LIBCOMPS_ENVIRONMENT_ATTRS.GROUP_IDS) + ), + PULP_ENVIRONMENT_ATTRS.OPTION_IDS: cls.grplist_to_lst( + getattr(environment, LIBCOMPS_ENVIRONMENT_ATTRS.OPTION_IDS) + ), + PULP_ENVIRONMENT_ATTRS.DESC_BY_LANG: strdict_to_dict( + getattr(environment, LIBCOMPS_ENVIRONMENT_ATTRS.DESC_BY_LANG) + ), + PULP_ENVIRONMENT_ATTRS.NAME_BY_LANG: strdict_to_dict( + getattr(environment, LIBCOMPS_ENVIRONMENT_ATTRS.NAME_BY_LANG) + ), + } + + +class PackageLangpacks(Content): """ The "Langpacks" content type. Formerly "PackageLangpacks" in Pulp 2. @@ -860,15 +1058,38 @@ class Langpacks(Content): The langpacks dictionary """ - TYPE = 'langpacks' + TYPE = 'packagelangpacks' - matches = models.TextField() + matches = JSONField(default=dict) digest = models.CharField(unique=True, max_length=64) class Meta: default_related_name = "%(app_label)s_%(model_name)s" + @classmethod + def natural_key_fields(cls): + """ + Digest is used as a natural key for PackageLangpacks. + """ + return ('digest',) + + @classmethod + def libcomps_to_dict(cls, langpacks): + """ + Convert libcomps langpacks object to dict for instantiating PackageLangpacks object. + + Args: + langpacks(libcomps.langpacks): a RPM/SRPM langpacks to convert + + Returns: + dict: all data for RPM/SRPM langpacks content creation + + """ + return { + PULP_LANGPACKS_ATTRS.MATCHES: strdict_to_dict(langpacks) + } + class RpmRemote(Remote): """ diff --git a/pulp_rpm/app/serializers.py b/pulp_rpm/app/serializers.py index 75a7af472f..899787f86c 100644 --- a/pulp_rpm/app/serializers.py +++ b/pulp_rpm/app/serializers.py @@ -18,6 +18,7 @@ PublicationSerializer, PublicationDistributionSerializer, NestedRelatedField, + RelatedField, validate_unknown_fields, ) @@ -27,6 +28,10 @@ Image, Variant, DistributionTree, + PackageGroup, + PackageCategory, + PackageEnvironment, + PackageLangpacks, Modulemd, ModulemdDefaults, Package, @@ -505,6 +510,213 @@ def validate(self, data): return new_data +class PackageGroupSerializer(NoArtifactContentSerializer): + """ + PackageGroup serializer. + """ + + id = serializers.CharField( + help_text=_("PackageGroup id."), + ) + default = serializers.BooleanField( + help_text=_("PackageGroup default."), + required=False + ) + user_visible = serializers.BooleanField( + help_text=_("PackageGroup user visibility."), + required=False + ) + display_order = serializers.IntegerField( + help_text=_("PackageGroup display order."), + allow_null=True + ) + name = serializers.CharField( + help_text=_("PackageGroup name."), + allow_null=True + ) + description = serializers.CharField( + help_text=_("PackageGroup description."), + allow_null=True + ) + packages = serializers.JSONField( + help_text=_("PackageGroup package list."), + allow_null=True + ) + biarch_only = serializers.BooleanField( + help_text=_("PackageGroup biarch only."), + required=False + ) + desc_by_lang = serializers.JSONField( + help_text=_("PackageGroup description by language."), + allow_null=True + ) + name_by_lang = serializers.JSONField( + help_text=_("PackageGroup name by language."), + allow_null=True + ) + digest = serializers.CharField( + help_text=_("PackageGroup digest."), + allow_null=True + ) + related_packages = RelatedField( + help_text=_("Packages related to this PackageGroup."), + allow_null=True, + required=False, + queryset=Package.objects.all(), + many=True, + view_name='content-rpm/packages-detail' + ) + + class Meta: + fields = ( + 'id', 'default', 'user_visible', 'display_order', + 'name', 'description', 'packages', 'biarch_only', + 'desc_by_lang', 'name_by_lang', 'digest', 'related_packages' + ) + model = PackageGroup + + +class PackageCategorySerializer(NoArtifactContentSerializer): + """ + PackageCategory serializer. + """ + + id = serializers.CharField( + help_text=_("Category id."), + ) + name = serializers.CharField( + help_text=_("Category name."), + allow_null=True + ) + description = serializers.CharField( + help_text=_("Category description."), + allow_null=True + ) + display_order = serializers.IntegerField( + help_text=_("Category display order."), + allow_null=True + ) + group_ids = serializers.JSONField( + help_text=_("Category group list."), + allow_null=True + ) + desc_by_lang = serializers.JSONField( + help_text=_("Category description by language."), + allow_null=True + ) + name_by_lang = serializers.JSONField( + help_text=_("Category name by language."), + allow_null=True + ) + digest = serializers.CharField( + help_text=_("Category digest."), + allow_null=True + ) + packagegroups = RelatedField( + help_text=_("PackageGroups related to this category."), + allow_null=True, + required=False, + queryset=PackageGroup.objects.all(), + many=True, + view_name='content-rpm/packagegroup-detail' + ) + + class Meta: + fields = ( + 'id', 'name', 'description', 'display_order', + 'group_ids', 'desc_by_lang', 'name_by_lang', 'digest', + 'packagegroups' + ) + model = PackageCategory + + +class PackageEnvironmentSerializer(NoArtifactContentSerializer): + """ + PackageEnvironment serializer. + """ + + id = serializers.CharField( + help_text=_("Environment id."), + ) + name = serializers.CharField( + help_text=_("Environment name."), + allow_null=True + ) + description = serializers.CharField( + help_text=_("Environment description."), + allow_null=True + ) + display_order = serializers.IntegerField( + help_text=_("Environment display order."), + allow_null=True + ) + group_ids = serializers.JSONField( + help_text=_("Environment group list."), + allow_null=True + ) + option_ids = serializers.JSONField( + help_text=_("Environment option ids"), + allow_null=True + ) + desc_by_lang = serializers.JSONField( + help_text=_("Environment description by language."), + allow_null=True + ) + name_by_lang = serializers.JSONField( + help_text=_("Environment name by language."), + allow_null=True + ) + digest = serializers.CharField( + help_text=_("Environment digest."), + allow_null=True + ) + packagegroups = RelatedField( + help_text=_("Groups related to this Environment."), + allow_null=True, + required=False, + queryset=PackageGroup.objects.all(), + many=True, + view_name='content-rpm/packagegroup-detail' + ) + optionalgroups = RelatedField( + help_text=_("Groups optionally related to this Environment."), + allow_null=True, + required=False, + queryset=PackageGroup.objects.all(), + many=True, + view_name='content-rpm/packagegroup-detail' + ) + + class Meta: + fields = ( + 'id', 'name', 'description', 'display_order', + 'group_ids', 'option_ids', 'desc_by_lang', 'name_by_lang', + 'digest', 'packagegroups', 'optionalgroups' + ) + model = PackageEnvironment + + +class PackageLangpacksSerializer(NoArtifactContentSerializer): + """ + PackageLangpacks serializer. + """ + + matches = serializers.JSONField( + help_text=_("Langpacks matches."), + allow_null=True + ) + digest = serializers.CharField( + help_text=_("Langpacks digest."), + allow_null=True + ) + + class Meta: + fields = ( + 'matches', 'digest' + ) + model = PackageLangpacks + + class ModulemdSerializer(SingleArtifactContentUploadSerializer): """ Modulemd serializer. diff --git a/pulp_rpm/app/tasks/synchronizing.py b/pulp_rpm/app/tasks/synchronizing.py index f4cbec4d61..3e131c489f 100644 --- a/pulp_rpm/app/tasks/synchronizing.py +++ b/pulp_rpm/app/tasks/synchronizing.py @@ -10,6 +10,7 @@ from urllib.parse import urljoin import createrepo_c as cr +import libcomps from django.db import transaction @@ -32,11 +33,13 @@ from pulp_rpm.app.constants import ( CHECKSUM_TYPES, + COMPS_REPODATA, MODULAR_REPODATA, PACKAGE_DB_REPODATA, PACKAGE_REPODATA, PULP_MODULE_ATTR, PULP_MODULEDEFAULTS_ATTR, + SKIP_REPODATA, UPDATE_REPODATA ) from pulp_rpm.app.models import ( @@ -49,6 +52,10 @@ ModulemdDefaults, Package, RepoMetadataFile, + PackageGroup, + PackageCategory, + PackageEnvironment, + PackageLangpacks, RpmRemote, UpdateCollection, UpdateCollectionPackage, @@ -59,6 +66,8 @@ from pulp_rpm.app.modulemd import parse_defaults, parse_modulemd +from pulp_rpm.app.comps import strdict_to_dict, dict_digest + import gi gi.require_version('Modulemd', '2.0') from gi.repository import Modulemd as mmdlib # noqa: E402 @@ -286,9 +295,11 @@ async def run(self): modulemd_pb = ProgressReport(message='Parse Modulemd', code='parsing.modulemds') modulemd_defaults_pb = ProgressReport(message='Parse Modulemd-defaults', code='parsing.modulemddefaults') + comps_pb = ProgressReport(message='Parsed Comps', code='parsing.comps') packages_pb.save() errata_pb.save() + comps_pb.save() remote_url = self.new_url or self.remote.url remote_url = remote_url if remote_url[-1] == "/" else f"{remote_url}/" @@ -327,8 +338,16 @@ async def run(self): package_repodata_urls = {} downloaders = [] modulemd_list = list() + dc_groups = [] + dc_categories = [] + dc_environments = [] nevra_to_module = defaultdict(dict) + pkgname_to_groups = defaultdict(list) + group_to_categories = defaultdict(list) + group_to_environments = defaultdict(list) + optionalgroup_to_environments = defaultdict(list) modulemd_results = None + comps_downloader = None for record in repomd.records: if record.type in PACKAGE_REPODATA: @@ -338,10 +357,19 @@ async def run(self): updateinfo_url = urljoin(remote_url, record.location_href) downloader = self.remote.get_downloader(url=updateinfo_url) downloaders.append([downloader.run()]) + + elif record.type in COMPS_REPODATA: + comps_url = urljoin(remote_url, record.location_href) + comps_downloader = self.remote.get_downloader(url=comps_url) + + elif record.type in SKIP_REPODATA: + continue + elif record.type in MODULAR_REPODATA: modules_url = urljoin(remote_url, record.location_href) modulemd_downloader = self.remote.get_downloader(url=modules_url) modulemd_results = await modulemd_downloader.run() + elif record.type not in PACKAGE_DB_REPODATA: file_data = {record.checksum_type: record.checksum, "size": record.size} da = DeclarativeArtifact( @@ -418,6 +446,96 @@ async def run(self): dc = DeclarativeContent(content=default_content, d_artifacts=[da]) await self.put(dc) + if comps_downloader: + comps_result = await comps_downloader.run() + + comps = libcomps.Comps() + comps.fromxml_f(comps_result.path) + + comps_pb.total = ( + len(comps.groups) + len(comps.categories) + len(comps.environments) + ) + comps_pb.state = 'running' + comps_pb.save() + + if comps.langpacks: + langpack_dict = PackageLangpacks.libcomps_to_dict(comps.langpacks) + packagelangpack = PackageLangpacks( + matches=strdict_to_dict(comps.langpacks), + digest=dict_digest(langpack_dict) + ) + dc = DeclarativeContent(content=packagelangpack) + dc.extra_data = defaultdict(list) + await self.put(dc) + + if comps.categories: + for category in comps.categories: + category_dict = PackageCategory.libcomps_to_dict(category) + category_dict['digest'] = dict_digest(category_dict) + packagecategory = PackageCategory(**category_dict) + dc = DeclarativeContent(content=packagecategory) + dc.extra_data = defaultdict(list) + + if packagecategory.group_ids: + for group_id in packagecategory.group_ids: + group_to_categories[group_id['name']].append(dc) + dc_categories.append(dc) + + if comps.environments: + for environment in comps.environments: + environment_dict = PackageEnvironment.libcomps_to_dict(environment) + environment_dict['digest'] = dict_digest(environment_dict) + packageenvironment = PackageEnvironment(**environment_dict) + dc = DeclarativeContent(content=packageenvironment) + dc.extra_data = defaultdict(list) + + if packageenvironment.option_ids: + for option_id in packageenvironment.option_ids: + optionalgroup_to_environments[option_id['name']].append(dc) + + if packageenvironment.group_ids: + for group_id in packageenvironment.group_ids: + group_to_environments[group_id['name']].append(dc) + + dc_environments.append(dc) + + if comps.groups: + for group in comps.groups: + group_dict = PackageGroup.libcomps_to_dict(group) + group_dict['digest'] = dict_digest(group_dict) + packagegroup = PackageGroup(**group_dict) + dc = DeclarativeContent(content=packagegroup) + dc.extra_data = defaultdict(list) + + if packagegroup.packages: + for package in packagegroup.packages: + pkgname_to_groups[package['name']].append(dc) + + if dc.content.id in group_to_categories.keys(): + for dc_category in group_to_categories[dc.content.id]: + dc.extra_data['category_relations'].append(dc_category) + dc_category.extra_data['packagegroups'].append(dc) + + if dc.content.id in group_to_environments.keys(): + for dc_environment in group_to_environments[dc.content.id]: + dc.extra_data['environment_relations'].append(dc_environment) + dc_environment.extra_data['packagegroups'].append(dc) + + if dc.content.id in optionalgroup_to_environments.keys(): + for dc_environment in optionalgroup_to_environments[dc.content.id]: + dc.extra_data['env_relations_optional'].append(dc_environment) + dc_environment.extra_data['optionalgroups'].append(dc) + + dc_groups.append(dc) + + for dc_category in dc_categories: + comps_pb.increment() + await self.put(dc_category) + + for dc_environment in dc_environments: + comps_pb.increment() + await self.put(dc_environment) + # to preserve order, downloaders are created after all repodata urls are identified package_repodata_downloaders = [] for repodata_type in PACKAGE_REPODATA: @@ -471,6 +589,11 @@ async def run(self): dc.extra_data['modulemd_relation'].append(dc_modulemd) dc_modulemd.extra_data['package_relation'].append(dc) + if dc.content.name in pkgname_to_groups.keys(): + for dc_group in pkgname_to_groups[dc.content.name]: + dc.extra_data['group_relations'].append(dc_group) + dc_group.extra_data['related_packages'].append(dc) + packages_pb.increment() await self.put(dc) @@ -513,14 +636,20 @@ async def run(self): modulemd_pb.increment() await self.put(modulemd) + for dc_group in dc_groups: + comps_pb.increment() + await self.put(dc_group) + packages_pb.state = 'completed' errata_pb.state = 'completed' modulemd_pb.state = 'completed' modulemd_defaults_pb.state = 'completed' + comps_pb.state = 'completed' packages_pb.save() errata_pb.save() modulemd_pb.save() modulemd_defaults_pb.save() + comps_pb.save() class RpmContentSaver(ContentSaver): @@ -633,6 +762,60 @@ def _handle_distribution_tree(declarative_content): except ValueError: pass + elif isinstance(declarative_content.content, PackageCategory): + for grp in declarative_content.extra_data['packagegroups']: + try: + with transaction.atomic(): + declarative_content.content.packagegroups.add(grp.content) + except ValueError: + pass + + elif isinstance(declarative_content.content, PackageEnvironment): + for grp in declarative_content.extra_data['packagegroups']: + try: + with transaction.atomic(): + declarative_content.content.packagegroups.add(grp.content) + except ValueError: + pass + + for opt in declarative_content.extra_data['optionalgroups']: + try: + with transaction.atomic(): + declarative_content.content.optionalgroups.add(opt.content) + except ValueError: + pass + + elif isinstance(declarative_content.content, PackageGroup): + for pkg in declarative_content.extra_data['related_packages']: + try: + with transaction.atomic(): + declarative_content.content.related_packages.add(pkg.content) + except ValueError: + pass + + for packagecategory in declarative_content.extra_data['category_relations']: + try: + with transaction.atomic(): + packagecategory.content.packagegroups.add(declarative_content.content) + except ValueError: + pass + + for packageenvironment in declarative_content.extra_data['environment_relations']: + try: + with transaction.atomic(): + packageenvironment.content.packagegroups.add( + declarative_content.content) + except ValueError: + pass + + for packageenvironment in declarative_content.extra_data['env_relations_optional']: + try: + with transaction.atomic(): + packageenvironment.content.optionalgroups.add( + declarative_content.content) + except ValueError: + pass + elif isinstance(declarative_content.content, Package): for modulemd in declarative_content.extra_data['modulemd_relation']: try: @@ -641,6 +824,13 @@ def _handle_distribution_tree(declarative_content): except ValueError: pass + for packagegroup in declarative_content.extra_data['group_relations']: + try: + with transaction.atomic(): + packagegroup.content.related_packages.add(declarative_content.content) + except ValueError: + pass + if update_collections_to_save: UpdateCollection.objects.bulk_create(update_collections_to_save) diff --git a/pulp_rpm/app/viewsets.py b/pulp_rpm/app/viewsets.py index ac75a4deaf..3691520fc1 100644 --- a/pulp_rpm/app/viewsets.py +++ b/pulp_rpm/app/viewsets.py @@ -22,6 +22,10 @@ from pulp_rpm.app.models import ( DistributionTree, Package, + PackageCategory, + PackageEnvironment, + PackageGroup, + PackageLangpacks, RepoMetadataFile, RpmDistribution, RpmRemote, @@ -38,6 +42,10 @@ ModulemdDefaultsSerializer, ModulemdSerializer, PackageSerializer, + PackageCategorySerializer, + PackageEnvironmentSerializer, + PackageGroupSerializer, + PackageLangpacksSerializer, RepoMetadataFileSerializer, RpmDistributionSerializer, RpmRemoteSerializer, @@ -230,6 +238,50 @@ def create(self, request): return OperationPostponedResponse(async_result, request) +class PackageGroupViewSet(ReadOnlyContentViewSet, + mixins.DestroyModelMixin): + """ + PackageGroup ViewSet. + """ + + endpoint_name = 'packagegroup' + queryset = PackageGroup.objects.all() + serializer_class = PackageGroupSerializer + + +class PackageCategoryViewSet(ReadOnlyContentViewSet, + mixins.DestroyModelMixin): + """ + PackageCategory ViewSet. + """ + + endpoint_name = 'packagecategory' + queryset = PackageCategory.objects.all() + serializer_class = PackageCategorySerializer + + +class PackageEnvironmentViewSet(ReadOnlyContentViewSet, + mixins.DestroyModelMixin): + """ + PackageEnvironment ViewSet. + """ + + endpoint_name = 'packageenvironment' + queryset = PackageEnvironment.objects.all() + serializer_class = PackageEnvironmentSerializer + + +class PackageLangpacksViewSet(ReadOnlyContentViewSet, + mixins.DestroyModelMixin): + """ + PackageLangpacks ViewSet. + """ + + endpoint_name = 'packagelangpacks' + queryset = PackageLangpacks.objects.all() + serializer_class = PackageLangpacksSerializer + + class DistributionTreeViewSet(ReadOnlyContentViewSet, mixins.DestroyModelMixin): """ diff --git a/pulp_rpm/tests/functional/constants.py b/pulp_rpm/tests/functional/constants.py index b0baacdd95..06a866da7c 100644 --- a/pulp_rpm/tests/functional/constants.py +++ b/pulp_rpm/tests/functional/constants.py @@ -16,9 +16,13 @@ RPM_PACKAGE_CONTENT_NAME = 'rpm.package' -RPM_ADVISORY_CONTENT_NAME = 'rpm.advisory' +RPM_PACKAGECATEGORY_CONTENT_NAME = 'rpm.packagecategory' + +RPM_PACKAGEGROUP_CONTENT_NAME = 'rpm.packagegroup' -RPM_METADATA_CONTENT_NAME = 'rpm.repo_metadata_file' +RPM_PACKAGELANGPACKS_CONTENT_NAME = 'rpm.packagelangpacks' + +RPM_ADVISORY_CONTENT_NAME = 'rpm.advisory' RPM_ALT_LAYOUT_FIXTURE_URL = urljoin(PULP_FIXTURES_BASE_URL, 'rpm-alt-layout/') """The URL to a signed RPM repository. See :data:`RPM_SIGNED_FIXTURE_URL`.""" @@ -76,16 +80,24 @@ :data:`RPM_SIGNED_FIXTURE_URL` and :data:`RPM_UNSIGNED_FIXTURE_URL` """ +RPM_PACKAGECATEGORY_COUNT = 1 +"""The number of packagecategories.""" + +RPM_PACKAGEGROUP_COUNT = 2 +"""The number of packagegroups.""" + +RPM_PACKAGELANGPACKS_COUNT = 1 +"""The number of packagelangpacks.""" + RPM_ADVISORY_COUNT = 4 """The number of updated record units.""" -RPM_METADATA_COUNT = 2 -"""The number of metadata file units.""" - RPM_FIXTURE_SUMMARY = { RPM_PACKAGE_CONTENT_NAME: RPM_PACKAGE_COUNT, RPM_ADVISORY_CONTENT_NAME: RPM_ADVISORY_COUNT, - RPM_METADATA_CONTENT_NAME: RPM_METADATA_COUNT, + RPM_PACKAGECATEGORY_CONTENT_NAME: RPM_PACKAGECATEGORY_COUNT, + RPM_PACKAGEGROUP_CONTENT_NAME: RPM_PACKAGEGROUP_COUNT, + RPM_PACKAGELANGPACKS_CONTENT_NAME: RPM_PACKAGELANGPACKS_COUNT, } """The breakdown of how many of each type of content unit are present in the standard repositories, i.e. :data:`RPM_SIGNED_FIXTURE_URL` and @@ -122,9 +134,11 @@ RPM_MODULAR_FIXTURE_SUMMARY = { RPM_PACKAGE_CONTENT_NAME: RPM_PACKAGE_COUNT, RPM_ADVISORY_CONTENT_NAME: RPM_ADVISORY_MODULAR_COUNT, - RPM_METADATA_CONTENT_NAME: RPM_METADATA_COUNT, RPM_MODULES_CONTENT_NAME: RPM_MODULES_COUNT, - RPM_MODULES_DEFAULTS_CONTENT_NAME: RPM_MODULES_DEFAULTS_COUNT + RPM_MODULES_DEFAULTS_CONTENT_NAME: RPM_MODULES_DEFAULTS_COUNT, + RPM_PACKAGECATEGORY_CONTENT_NAME: RPM_PACKAGECATEGORY_COUNT, + RPM_PACKAGEGROUP_CONTENT_NAME: RPM_PACKAGEGROUP_COUNT, + RPM_PACKAGELANGPACKS_CONTENT_NAME: RPM_PACKAGELANGPACKS_COUNT, } """The breakdown of how many of each type of content unit are present in the i.e. :data:`RPM_MODULAR_FIXTURE_URL`."""