Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Added support for importing Pulp Exports
fixes #6329
fixes #6137
  • Loading branch information
David Davis committed Apr 14, 2020
1 parent bab111c commit be696b2
Show file tree
Hide file tree
Showing 20 changed files with 614 additions and 13 deletions.
1 change: 1 addition & 0 deletions CHANGES/6137.feature
@@ -0,0 +1 @@
Added API for importing Pulp Exports at ``POST /importers/core/pulp/<uuid>/imports/``.
2 changes: 2 additions & 0 deletions CHANGES/6329.feature
@@ -0,0 +1,2 @@
Added PulpImporter API at ``/pulp/api/v3/importers/core/pulp/``. PulpImporters are used for
importing exports from Pulp.
2 changes: 2 additions & 0 deletions CHANGES/plugin_api/6329.feature
@@ -0,0 +1,2 @@
Added models Import and Importer (as well as serializers and viewsets) that can be used for
importing data into Pulp.
6 changes: 1 addition & 5 deletions pulpcore/app/apps.py
@@ -1,4 +1,3 @@
import inspect
from collections import defaultdict
from importlib import import_module

Expand Down Expand Up @@ -146,10 +145,7 @@ def import_modelresources(self):
modelrsrc_module_name = '{name}.{module}'.format(
name=self.name, module=MODELRESOURCE_MODULE_NAME)
self.modelresource_module = import_module(modelrsrc_module_name)
self.exportable_classes = []
for (classname, cls) in inspect.getmembers(self.modelresource_module, inspect.isclass):
if cls.__module__ == self.modelresource_module.__name__:
self.exportable_classes.append(cls)
self.exportable_classes = self.modelresource_module.IMPORT_ORDER


class PulpAppConfig(PulpPluginAppConfig):
Expand Down
@@ -0,0 +1,85 @@
# Generated by Django 2.2.11 on 2020-04-13 16:39

import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import uuid


class Migration(migrations.Migration):

dependencies = [
('core', '0027_export_backend'),
]

operations = [
migrations.CreateModel(
name='Import',
fields=[
('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('pulp_created', models.DateTimeField(auto_now_add=True)),
('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
('params', django.contrib.postgres.fields.jsonb.JSONField(null=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Importer',
fields=[
('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('pulp_created', models.DateTimeField(auto_now_add=True)),
('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
('pulp_type', models.TextField(default=None)),
('name', models.TextField(db_index=True, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='PulpImport',
fields=[
('import_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='core_pulp_export', serialize=False, to='core.Import')),
],
options={
'default_related_name': '%(app_label)s_pulp_export',
},
bases=('core.import',),
),
migrations.CreateModel(
name='PulpImporter',
fields=[
('importer_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='core_pulp_importer', serialize=False, to='core.Importer')),
],
options={
'default_related_name': '%(app_label)s_pulp_importer',
},
bases=('core.importer',),
),
migrations.AddField(
model_name='import',
name='importer',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.Importer'),
),
migrations.AddField(
model_name='import',
name='task',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='core.Task'),
),
migrations.CreateModel(
name='PulpImporterRepository',
fields=[
('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('pulp_created', models.DateTimeField(auto_now_add=True)),
('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
('source_repo', models.TextField()),
('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.Repository')),
('pulp_importer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='repo_map', to='core.PulpImporter')),
],
options={
'abstract': False,
},
),
]
20 changes: 13 additions & 7 deletions pulpcore/app/modelresource.py
Expand Up @@ -26,9 +26,16 @@ class QueryModelResource(resources.ModelResource):
queryset (django.db.models.query.QuerySet): filtering queryset for this resource
(driven by repo_version)
"""
def set_up_queryset(self):
return None

def __init__(self, repo_version=None):
self.repo_version = repo_version
self.queryset = None
if repo_version:
self.queryset = self.set_up_queryset()

class Meta:
import_id_fields = ('pulp_id',)


#
Expand All @@ -52,18 +59,17 @@ class Meta:
# follow the same pattern as a plugin writer would follow
#
class ContentResource(QueryModelResource):
def __init__(self, repo_version):
QueryModelResource.__init__(repo_version)
self.queryset = repo_version.content
def set_up_queryset(self):
return self.repo_version.content

class Meta:
model = Content
fields = ('pulp_id', 'pulp_type')


class ContentArtifactResource(QueryModelResource):
def __init__(self, repo_version):
QueryModelResource.__init__(repo_version)
self.queryset = ContentArtifact.objects.filter(content__in=repo_version.content)
def set_up_queryset(self):
return ContentArtifact.objects.filter(content__in=self.repo_version.content)

class Meta:
model = ContentArtifact
6 changes: 6 additions & 0 deletions pulpcore/app/models/__init__.py
Expand Up @@ -18,6 +18,12 @@
PulpExport,
PulpExporter,
)
from .importer import ( # noqa
Import,
Importer,
PulpImport,
PulpImporter,
)
from .publication import ( # noqa
BaseDistribution,
ContentGuard,
Expand Down
84 changes: 84 additions & 0 deletions pulpcore/app/models/importer.py
@@ -0,0 +1,84 @@
from django.contrib.postgres.fields import JSONField
from django.db import models

from pulpcore.app.models import (
BaseModel,
MasterModel,
)
from .repository import Repository


class Import(BaseModel):
"""
A model that represents imports into Pulp.
Fields:
params (models.JSONField): A set of parameters used to run the import
Relations:
task (models.ForeignKey): The Task that ran the import
importer (models.ForeignKey): The Importer that imported the export
"""
params = JSONField(null=True)
task = models.ForeignKey("Task", on_delete=models.PROTECT)
importer = models.ForeignKey("Importer", on_delete=models.CASCADE)


class Importer(MasterModel):
"""
A base model that provides logic to import data into Pulp.
Can be extended by plugins to provide import functionality.
Fields:
name (models.TextField): The importer unique name.
"""
name = models.TextField(db_index=True, unique=True)


class PulpImporter(Importer):
"""
A model that can be used to import exports from other Pulp instances.
"""
TYPE = 'pulp'

@property
def repo_mapping(self):
return {repo.source_repo: repo.repository.name for repo in self.repo_map.all()}

@repo_mapping.setter
def repo_mapping(self, mapping):
self.repo_map.all().delete()
for source, repo_name in mapping.items():
repo = Repository.objects.get(name=repo_name)
self.repo_map.create(source_repo=source,
repository=repo)

class Meta:
default_related_name = '%(app_label)s_pulp_importer'


class PulpImporterRepository(BaseModel):
"""
A model that maps repo names in an export to repos in Pulp.
Fields:
source_repo (models.TextField): The name of the repo in the export
Relations:
pulp_importer (models.ForeignKey): The associated Pulp importer
repository (models.ForeignKey): The repository in Pulp
"""
source_repo = models.TextField()
pulp_importer = models.ForeignKey(
PulpImporter,
related_name="repo_map",
on_delete=models.CASCADE)
repository = models.ForeignKey(Repository, on_delete=models.CASCADE)


class PulpImport(Import):
"""A model that represents imports into Pulp from another Pulp instance."""

class Meta:
default_related_name = '%(app_label)s_pulp_export'
14 changes: 14 additions & 0 deletions pulpcore/app/models/repository.py
Expand Up @@ -612,6 +612,20 @@ def remove_content(self, content):
version_removed=None)
q_set.update(version_removed=self)

def set_content(self, content):
"""
Sets the repo version content by calling remove_content() then add_content().
Args:
content (django.db.models.QuerySet): Set of desired content
Raise:
pulpcore.exception.ResourceImmutableError: if set_content is called on a
complete RepositoryVersion
"""
self.remove_content(self.content.exclude(pk__in=content))
self.add_content(content.exclude(pk__in=self.content))

def next(self):
"""
Returns:
Expand Down
9 changes: 9 additions & 0 deletions pulpcore/app/serializers/__init__.py
Expand Up @@ -25,6 +25,9 @@
ExportsIdentityFromExporterField,
ExportRelatedField,
ExportIdentityField,
ImportsIdentityFromImporterField,
ImportRelatedField,
ImportIdentityField,
LatestVersionField,
SecretCharField,
SingleContentArtifactField,
Expand All @@ -42,6 +45,12 @@
PulpExporterSerializer,
PulpExportSerializer,
)
from .importer import ( # noqa
ImportSerializer,
ImporterSerializer,
PulpImporterSerializer,
PulpImportSerializer,
)
from .progress import ProgressReportSerializer # noqa
from .publication import ( # noqa
BaseDistributionSerializer,
Expand Down
2 changes: 1 addition & 1 deletion pulpcore/app/serializers/exporter.py
Expand Up @@ -24,7 +24,7 @@ class ExporterSerializer(ModelSerializer):
pulp_href = DetailIdentityField()
name = serializers.CharField(
help_text=_("Unique name of the file system exporter."),
validators=[UniqueValidator(queryset=models.BaseDistribution.objects.all())]
validators=[UniqueValidator(queryset=models.Exporter.objects.all())]
)

@staticmethod
Expand Down
42 changes: 42 additions & 0 deletions pulpcore/app/serializers/fields.py
Expand Up @@ -320,6 +320,48 @@ def get_object(self, view_name, view_args, view_kwargs):
return self.get_queryset().get(**lookup_kwargs)


class ImportsIdentityFromImporterField(DetailIdentityField):
view_name = 'importers-detail'

def __init__(self, view_name=None, **kwargs):
assert view_name is None, 'The `view_name` must not be set.'
super().__init__(view_name=self.view_name, **kwargs)

def get_url(self, obj, view_name, request, *args, **kwargs):
return super().get_url(obj, self.view_name, request, *args, **kwargs) + "imports/"


class ImportFieldGetURLMixin:
view_name = 'imports-detail'

def __init__(self, view_name=None, **kwargs):
assert view_name is None, 'The `view_name` must not be set.'
super().__init__(view_name=self.view_name, **kwargs)

def get_url(self, obj, view_name, request, *args, **kwargs):
imports_field = ImportsIdentityFromImporterField()
importer_url = imports_field.get_url(obj.importer, None, request, *args, **kwargs)
return f"{importer_url}{obj.pk}/"

def use_pk_only_optimization(self):
return False


class ImportIdentityField(ImportFieldGetURLMixin, IdentityField):
pass


class ImportRelatedField(ImportFieldGetURLMixin, RelatedField):
queryset = models.Import.objects.all()

def get_object(self, view_name, view_args, view_kwargs):
lookup_kwargs = {
'importer__pk': view_kwargs['importer_pk'],
'pk': view_kwargs['pk']
}
return self.get_queryset().get(**lookup_kwargs)


class SecretCharField(serializers.CharField):
"""
Serializer field for secrets.
Expand Down

0 comments on commit be696b2

Please sign in to comment.