Add sync support for content types from comps.xml

Updates model relations, adds serializers, constants, viewsets,
libcomps parsing, and sync support for comps.xml content types

closes #5423
https://pulp.plan.io/issues/5423

CHANGES/5423.misc

@@ -0,0 +1 @@
+Comps.xml sync

pulp_rpm/app/comps.py

@@ -0,0 +1,36 @@
+import hashlib
+
+
+def strdict_to_dict(value):
+    """
+    Convert libcomps StrDict type object to standard dict.
+
+    Args:
+        value: a libcomps StrDict
+
+    Returns:
+        lang_dict: a dict
+
+    """
+    lang_dict = {}
+    if len(value):
+        for i, j in value.items():
+            lang_dict[i] = j
+    return lang_dict
+
+
+def dict_digest(dict):
+    """
+    Calculate a hexdigest for a given dictionary.
+
+    Args:
+        dict: a dictionary
+
+    Returns:
+        A digest
+
+    """
+    prep_hash = list(dict.values())
+    str_prep_hash = [str(i) for i in prep_hash]
+    str_prep_hash.sort()
+    return hashlib.sha256(''.join(str_prep_hash).encode('utf-8')).hexdigest()

pulp_rpm/app/constants.py

@@ -76,6 +76,8 @@
 PACKAGE_DB_REPODATA = ['primary_db', 'filelists_db', 'other_db']
 UPDATE_REPODATA = ['updateinfo']
 MODULAR_REPODATA = ['modules']
+COMPS_REPODATA = ['group']
+SKIP_REPODATA = ['group_gz']
 
 CR_UPDATE_RECORD_ATTRS = SimpleNamespace(
     ID='id',
@@ -167,3 +169,75 @@
 )
 
 PULP_MODULE_ATTR = MODULEMD_MODULE_ATTR
+
+LIBCOMPS_GROUP_ATTRS = SimpleNamespace(
+    ID='id',
+    DEFAULT='default',
+    USER_VISIBLE='uservisible',
+    DISPLAY_ORDER='display_order',
+    NAME='name',
+    DESCRIPTION='desc',
+    PACKAGES='packages',
+    BIARCH_ONLY='biarchonly',
+    DESC_BY_LANG='desc_by_lang',
+    NAME_BY_LANG='name_by_lang'
+)
+
+LIBCOMPS_CATEGORY_ATTRS = SimpleNamespace(
+    ID='id',
+    NAME='name',
+    DESCRIPTION='desc',
+    DISPLAY_ORDER='display_order',
+    GROUP_IDS='group_ids',
+    DESC_BY_LANG='desc_by_lang',
+    NAME_BY_LANG='name_by_lang'
+)
+
+LIBCOMPS_ENVIRONMENT_ATTRS = SimpleNamespace(
+    ID='id',
+    NAME='name',
+    DESCRIPTION='desc',
+    DISPLAY_ORDER='display_order',
+    GROUP_IDS='group_ids',
+    OPTION_IDS='option_ids',
+    DESC_BY_LANG='desc_by_lang',
+    NAME_BY_LANG='name_by_lang'
+)
+
+PULP_LANGPACKS_ATTRS = SimpleNamespace(
+    MATCHES='matches'
+)
+
+PULP_GROUP_ATTRS = SimpleNamespace(
+    ID='id',
+    DEFAULT='default',
+    USER_VISIBLE='user_visible',
+    DISPLAY_ORDER='display_order',
+    NAME='name',
+    DESCRIPTION='description',
+    PACKAGES='packages',
+    BIARCH_ONLY='biarch_only',
+    DESC_BY_LANG='desc_by_lang',
+    NAME_BY_LANG='name_by_lang'
+)
+
+PULP_CATEGORY_ATTRS = SimpleNamespace(
+    ID='id',
+    NAME='name',
+    DESCRIPTION='description',
+    DISPLAY_ORDER='display_order',
+    GROUP_IDS='group_ids',
+    DESC_BY_LANG='desc_by_lang',
+    NAME_BY_LANG='name_by_lang'
+)
+
+PULP_ENVIRONMENT_ATTRS = SimpleNamespace(
+    ID='id',
+    NAME='name',
+    DESCRIPTION='description',
+    DISPLAY_ORDER='display_order',
+    GROUP_IDS='group_ids',
+    OPTION_IDS='option_ids',
+    DESC_BY_LANG='desc_by_lang',
+    NAME_BY_LANG='name_by_lang'
+)

pulp_rpm/app/migrations/0006_auto_20191025_1850.py

@@ -0,0 +1,122 @@
+# Generated by Django 2.2.6 on 2019-10-25 18:50
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0011_relative_path'),
+        ('rpm', '0005_pulp_fields'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PackageCategory',
+            fields=[
+                ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_packagecategory', serialize=False, to='core.Content')),
+                ('id', models.CharField(max_length=255)),
+                ('name', models.CharField(max_length=255)),
+                ('description', models.TextField(default='')),
+                ('display_order', models.IntegerField()),
+                ('group_ids', django.contrib.postgres.fields.jsonb.JSONField(default=list)),
+                ('desc_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
+                ('name_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
+                ('digest', models.CharField(max_length=64, unique=True)),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+            },
+            bases=('core.content',),
+        ),
+        migrations.CreateModel(
+            name='PackageEnvironment',
+            fields=[
+                ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_packageenvironment', serialize=False, to='core.Content')),
+                ('id', models.CharField(max_length=255)),
+                ('name', models.CharField(max_length=255)),
+                ('description', models.TextField(default='')),
+                ('display_order', models.IntegerField(null=True)),
+                ('group_ids', django.contrib.postgres.fields.jsonb.JSONField(default=list)),
+                ('option_ids', django.contrib.postgres.fields.jsonb.JSONField(default=list)),
+                ('desc_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
+                ('name_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
+                ('digest', models.CharField(max_length=64, unique=True)),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+            },
+            bases=('core.content',),
+        ),
+        migrations.CreateModel(
+            name='PackageLangpacks',
+            fields=[
+                ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_packagelangpacks', serialize=False, to='core.Content')),
+                ('matches', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
+                ('digest', models.CharField(max_length=64, unique=True)),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+            },
+            bases=('core.content',),
+        ),
+        migrations.RemoveField(
+            model_name='environment',
+            name='content_ptr',
+        ),
+        migrations.RemoveField(
+            model_name='langpacks',
+            name='content_ptr',
+        ),
+        migrations.AddField(
+            model_name='packagegroup',
+            name='related_packages',
+            field=models.ManyToManyField(related_name='rpm_packagegroup', to='rpm.Package'),
+        ),
+        migrations.AlterField(
+            model_name='packagegroup',
+            name='desc_by_lang',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name='packagegroup',
+            name='description',
+            field=models.TextField(default=''),
+        ),
+        migrations.AlterField(
+            model_name='packagegroup',
+            name='name_by_lang',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name='packagegroup',
+            name='packages',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=list),
+        ),
+        migrations.DeleteModel(
+            name='Category',
+        ),
+        migrations.DeleteModel(
+            name='Environment',
+        ),
+        migrations.DeleteModel(
+            name='Langpacks',
+        ),
+        migrations.AddField(
+            model_name='packageenvironment',
+            name='optionalgroups',
+            field=models.ManyToManyField(related_name='optionalgroups_to_env', to='rpm.PackageGroup'),
+        ),
+        migrations.AddField(
+            model_name='packageenvironment',
+            name='packagegroups',
+            field=models.ManyToManyField(related_name='packagegroups_to_env', to='rpm.PackageGroup'),
+        ),
+        migrations.AddField(
+            model_name='packagecategory',
+            name='packagegroups',
+            field=models.ManyToManyField(related_name='rpm_packagecategory', to='rpm.PackageGroup'),
+        ),
+    ]

pulp_rpm/app/migrations/0007_auto_20191101_1648.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.6 on 2019-11-01 16:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rpm', '0006_auto_20191025_1850'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='packagecategory',
+            name='display_order',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='packagegroup',
+            name='display_order',
+            field=models.IntegerField(null=True),
+        ),
+    ]