Migrate Pulp 2 ISO content into Pulp 3

Create hard links (of copy if not possible) and create Artifacts,
ContentArtifacts and Content in pulp 3.

closes #3810
https://pulp.plan.io/issues/3810

Required PR: pulp/pulpcore#254

README.md

@@ -69,14 +69,13 @@ All the commands should be run on Pulp 3 machine.
 1. Create a Migration Plan
 ```
 $ # migrate content for Pulp 2 ISO plugin
-$ http POST :24817/pulp/api/v3/migration-plans/ plan='{ "plugins": [{"type": "iso", "content": 
-true}]}'
+$ http POST :24817/pulp/api/v3/migration-plans/ plan='{"plugins": [{"type": "iso"}]}'
 
 HTTP/1.1 201 Created
 {
     "_created": "2019-07-23T08:18:12.927007Z",
     "_href": "/pulp/api/v3/migration-plans/59f8a786-c7d7-4e2b-ad07-701479d403c5/",
-    "plan": "{ \"plugins\": [{\"type\": \"iso\", \"content\": \ntrue}]}"
+    "plan": "{ \"plugins\": [{\"type\": \"iso\"}]}"
 }
 
 ```
@@ -97,18 +96,34 @@ HTTP/1.1 202 Accepted
 If you are extending this migration tool to be able to migrate the content type of your interest
 from Pulp 2 to Pulp 3, here are some guidelines.
 
-1. Add the necessary mappings to the constants.py.
+
+1. Layout of the files/directories is important.
+ - Create a plugin directory in `pulp_2to3_migrate.app.plugin` if it doesn't exist. Directory name
+  has to have the same name as you specified your plugin name in PULP2_SUPPORTED_PLUGINS in step 1.
+ - This directory has to have a `pulp2` and a `pulp3` with a module named `models.py` in each.
+ In `pulp2/models.py` define your Content model to access Pulp 2 data. See step 3.
+ In `pulp3/models.py` define your Content model to pre-migrate Pulp 3 content to. See step 4.
+
+2. Add the necessary mappings to the constants.py.
  - to PULP2_SUPPORTED_PLUGINS
- - to PULP_2TO3_MAP
+ - to PULP_2TO3_CONTENT_MODEL_MAP
 
-2. Add a Content model to communicate with Pulp 2.
+3. Add a Content model to communicate with Pulp 2.
  - It has to have a field `type` which will correspond to the `_content_type_id` of your Content
- in Pulp 2. Don't forget to add it to PULP_2TO3_MAP in step 1.
- - It has to have a ForeignKey to the `pulp_2to3_migrate.app.models.Pulp2Content` model with the
- `related_name` set to `'pulp3content'`
-
-3. Layout of the files/directories is important.
- - Create a plugin directory in `pulp_2to3_migrate.pulp2` if it doesn't exist. Directory name
-  has to have the same name as you specified your plugin name in PULP2_SUPPORTED_PLUGINS in step 1.
- - This directory has to have a module named `models.py` where you define you Content model to
-   for Pulp 2. Don't forget to put its name into PULP2_SUPPORTED_PLUGINS in step 1.
+ in Pulp 2. Don't forget to add it to PULP_2TO3_CONTENT_MODEL_MAP in step 1.
+
+4. Add a Content model to pre-migrate Pulp 2 content to (subclass the provided `Pulp2to3Content` 
+class). It has to have:
+ - a field `type` which will correspond to the `_content_type_id` of your Content in Pulp 2.
+ - a ForeignKey to the `pulp_2to3_migrate.app.models.Pulp2Content` model with the `related_name` 
+ set to `'detail_model'` (provided by `Pulp2to3Content`).
+ - a classmethod `pre_migrate_content_detail` (see `Pulp2to3Content` for more details)
+ - a classmethod `migrate_content_to_pulp3` (see `Pulp2to3Content` for more details)
+ - a method `create_pulp3_content` (see `Pulp2to3Content` for more details)
+
+ If your content has one artifact and if you are willing to use the default implementation of the 
+ first stage of DeclarativeContentMigration, on your Content model you also need:
+ - an `expected_digests` property to provide expected digests for artifact creation/validation
+ - an `expected_size` property to provide the expected size for artifact creation/validation
+ - a `relative_path_for_content_artifact` property to provide the relative path for content 
+ artifact creation.

pulp_2to3_migrate/app/constants.py

@@ -2,7 +2,7 @@
 PULP_2TO3_MIGRATION_RESOURCE = 'pulp_2to3_migration'
 
 # Pulp2 plugins and their content types which can be migrated
-# 'pulp2_plugin': 'pulp2 model class name'
+# 'pulp2_plugin': 'pulp_2to3_migrate pulp2 model class name'
 SUPPORTED_PULP2_PLUGINS = {
     'iso': ['ISO'],
     # 'rpm': [
@@ -26,10 +26,10 @@
     #     'Tag'],
 }
 
-# Mapping Pulp 2 content type to Pulp 3 plugin and content type
-# 'pulp2 content type id' -> ('pulp3 plugin', 'pulp3 content type')
-PULP_2TO3_TYPE_MAP = {
-    'iso': ('file', 'file'),
-    # 'erratum': ('rpm', 'advisory'),
-    # 'rpm': ('rpm': 'package'),
+# 'pulp2_content_type_id': 'pulp_2to3_migrate pulp3 model class name '
+PULP_2TO3_CONTENT_MODEL_MAP = {
+    'iso': 'Pulp2ISO',
 }
+
+# For manfatory parameters which won't be used by migration tool, e.g. in Declarative Version
+NOT_USED = 'Not Used'

pulp_2to3_migrate/app/models/__init__.py

@@ -0,0 +1,8 @@
+from .base import (  # noqa
+    MigrationPlan,
+    Pulp2Content,
+    Pulp2to3Content,
+)
+
+# import all pulp_2to3 detail plugin models here
+from pulp_2to3_migrate.app.plugin.iso.pulp3.models import Pulp2ISO  # noqa

pulp_2to3_migrate/app/models/base.py

@@ -0,0 +1,99 @@
+from django.db import models
+
+from pulpcore.plugin.models import (
+    Content,
+    Model
+)
+
+
+class MigrationPlan(Model):
+    """
+    Migration Plans that have been created and maybe even run.
+
+    Fields:
+        plan (models.TextField): The migration plan in the JSON format
+    """
+    plan = models.TextField()
+    # TODO: convert to JSONField
+
+
+class Pulp2Content(Model):
+    """
+    General info about Pulp 2 content.
+
+    Pulp3 plugin models should create a Foreign key to this model.
+
+    Fields:
+        pulp2_id (models.UUIDField): Content ID in Pulp 2
+        pulp2_content_type_id (models.CharField): Content type in Pulp 2
+        pulp2_last_updated (models.PositiveIntegerField): Content creation or update time in Pulp 2
+        pulp2_storage_path (models.TextField): Content storage path on Pulp 2 system
+        downloaded (models.BooleanField): Flag to identify if content is on a filesystem or not
+
+    Relations:
+        pulp3_content (models.ForeignKey): Pulp 3 content which Pulp 2 content was migrated to
+    """
+    pulp2_id = models.CharField(max_length=255)
+    pulp2_content_type_id = models.CharField(max_length=255)
+    pulp2_last_updated = models.PositiveIntegerField()
+    pulp2_storage_path = models.TextField()
+    downloaded = models.BooleanField(default=True)
+    pulp3_content = models.ForeignKey(Content, on_delete=models.SET_NULL, null=True)
+
+    class Meta:
+        unique_together = ('pulp2_id', 'pulp2_content_type_id')
+
+
+class Pulp2to3Content(Model):
+    """
+    Pulp 2to3 detail content model to store pulp 2 content details for Pulp 3 content creation.
+    """
+    pulp2content = models.ForeignKey(Pulp2Content,
+                                     related_name='detail_model',
+                                     on_delete=models.CASCADE)
+    type = '<your pulp 2 content type>'
+
+    class Meta:
+        abstract = True
+
+    @classmethod
+    async def pre_migrate_content_detail(cls, content_batch):
+        """
+        Pre-migrate Pulp 2 content with all the fields needed to create a Pulp 3 Content
+
+        Args:
+             content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content.
+
+        Example for ISO content:
+        >>> pulp2_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch}
+        >>> pulp2_ids = pulp2_id_obj_map.keys()
+        >>> pulp2_iso_content_batch = ISO.objects.filter(id__in=pulp2_ids)
+        >>> pulp2iso_to_save = [Pulp2ISO(name=iso.name,
+        >>>                              checksum=iso.checksum,
+        >>>                              size=iso.size,
+        >>>                              pulp2content=pulp2_map[iso.id])
+        >>>                     for iso in pulp2_iso_content_batch]
+        >>> cls.objects.bulk_create(pulp2iso_to_save, ignore_conflicts=True)
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    async def migrate_content_to_pulp3(cls):
+        """
+        Migrate pre-migrated Pulp 2 content.
+
+        Create a DeclatativeContentMigration pipeline here and instantiate it with your first stage.
+        Here the default implementation of the first stage is used:
+        >>> first_stage = ContentMigrationFirstStage(cls)
+        >>> dv = DeclarativeContentMigration(first_stage=first_stage)
+        >>> await dv.create()
+        """
+        raise NotImplementedError()
+
+    def create_pulp3_content(self):
+        """
+        Create a Pulp 3 detail Content unit for saving it later in a bulk operation.
+
+        Return an unsaved Pulp 3 Content
+        """
+        raise NotImplementedError()

pulp_2to3_migrate/app/plugin/api.py

@@ -0,0 +1,5 @@
+from .content import (  # noqa
+    DeclarativeContentMigration,
+    ContentMigrationFirstStage,
+    RelatePulp2to3Content,
+)