Discover plugins via enrty_points

Stop using mappings for plugin specific items, move everyting
to a plugin migrator class.

closes #5491
https://pulp.plan.io/issues/5491

CHANGES/5491.misc

@@ -0,0 +1 @@
+Refactor to have plugin discovery via entry_points.

README.md

@@ -93,20 +93,16 @@ HTTP/1.1 202 Accepted
 
 ### Plugin Writer's Guide
 
-If you are extending this migration tool to be able to migrate the content type of your interest
+If you are extending this migration tool to be able to migrate the plugin of your interest
 from Pulp 2 to Pulp 3, here are some guidelines.
 
 
-1. Layout of the files/directories is important.
- - Create a plugin directory in `pulp_2to3_migration.app.plugin` if it doesn't exist. Directory name
-  has to have the same name as you specified your plugin name in PULP2_SUPPORTED_PLUGINS in step 1.
- - This directory has to have a `pulp2` and a `pulp3` with a module named `models.py` in each.
- In `pulp2/models.py` define your Content model to access Pulp 2 data. See step 3.
- In `pulp3/models.py` define your Content model to pre-migrate Pulp 3 content to. See step 4.
+1. Create a migrator class (subclass the provided `Pulp2to3PluginMigrator` class). There should be
+ one migrator class per plugin. Define all the necessary attributes and methods for it (see
+  `Pulp2to3PluginMigrator` for more details)
 
-2. Add the necessary mappings to the constants.py.
- - to PULP2_SUPPORTED_PLUGINS
- - to PULP_2TO3_CONTENT_MODEL_MAP
+2. Discovery of the plugins is done via entry_points. Add your migrator class defined in step 1
+ to the list of the "migrators" entry_points in setup.py.
 
 3. Add a Content model to communicate with Pulp 2.
  - It has to have a field `type` which will correspond to the `_content_type_id` of your Content
@@ -117,7 +113,6 @@ class). It has to have:
  - a field `type` which will correspond to the `_content_type_id` of your Content in Pulp 2.
  - on a Meta class a `default_related_name` set to `<your pulp 2 content type>_detail_model`
  - a classmethod `pre_migrate_content_detail` (see `Pulp2to3Content` for more details)
- - a classmethod `migrate_content_to_pulp3` (see `Pulp2to3Content` for more details)
  - a method `create_pulp3_content` (see `Pulp2to3Content` for more details)
 
  If your content has one artifact and if you are willing to use the default implementation of the 

pulp_2to3_migration/app/constants.py

@@ -1,69 +1,10 @@
-# temporary hack, to move gradually to one migrator
-from pulp_2to3_migration.app.plugin.iso.pulp3.migrator import IsoMigrator
-
 # for tasking system to ensure only one migration is run at a time
 PULP_2TO3_MIGRATION_RESOURCE = 'pulp_2to3_migration'
 
-# Pulp2 plugins and their content types which can be migrated
-# 'pulp2_plugin': 'pulp_2to3_migration pulp2 model class name'
-SUPPORTED_PULP2_PLUGINS = {
-    'iso': ['ISO'],
-    # 'rpm': [
-    #     'Distribution',
-    #     'Drpm',
-    #     'Erratum',
-    #     'Modulemd',
-    #     'ModulemdDefaults',
-    #     'PackageCategory',
-    #     'PackageEnvironment',
-    #     'PackageGroup',
-    #     'PackageLangpacks',
-    #     'Rpm',
-    #     'Srpm',
-    #     'YumRepoMetadataFile'],
-    # 'docker': [
-    #     'Blob',
-    #     'Image',
-    #     'Manifest',
-    #     'ManifestList',
-    #     'Tag'],
-}
-
-# 'pulp2_content_type_id': 'pulp_2to3_migration pulp3 model class name '
-PULP_2TO3_CONTENT_MODEL_MAP = {
-    'iso': 'Pulp2ISO',
-}
-
-
-PULP_2TO3_PLUGIN_MAP = {
-    'iso': 'pulp_file',
-    # 'docker' : 'pulp_container',
-    # 'rpm' : 'pulp_rpm',
-}
-
-PULP2_COLLECTION_MAP = {
-    'iso': 'units_iso',
-    # 'docker': 'units_docker_manifest',
-    # 'rpm': 'units_rpm',
-}
-
 PULP_2TO3_POLICIES = {
     'immediate': 'immediate',
     'on_demand': 'on_demand',
     'background': 'on_demand',
 }
 
-# 'pulp2 plugin': [('pulp2 importer_type_id', 'pulp_2to3 plugin importer migration model'), ... ]
-PULP_2TO3_IMPORTER_TYPE_MODEL_MAP = {
-    'iso': [('iso_importer', 'IsoImporter')]
-}
-
-# 'pulp2 plugin': [('pulp2 distributor_type_id', 'pulp_2to3 plugin distributor migration model'),
-#                  ... ]
-# PULP_2TO3_DISTRIBUTOR_TYPE_MODEL_MAP = {
-#     'iso': [('iso_distributor', 'IsoDistributor'),]
-# }
-
-PLUGIN_MIGRATORS_MAP = {
-    'iso': IsoMigrator
-}
+NOT_USED = 'Not Used'

pulp_2to3_migration/app/migration.py

@@ -1,23 +1,18 @@
 import asyncio
 import hashlib
-import importlib
 import logging
 
 from pulpcore.plugin.models import (
     ProgressReport,
     Repository,
 )
 
-from pulp_2to3_migration.app.constants import (
-    PLUGIN_MIGRATORS_MAP,
-    PULP_2TO3_IMPORTER_TYPE_MODEL_MAP,
-)
 from pulp_2to3_migration.app.models import (
     Pulp2Content,
     Pulp2Importer,
     Pulp2Repository,
 )
-
+from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS
 
 _logger = logging.getLogger(__name__)
 
@@ -35,11 +30,11 @@ async def migrate_content(plugins_to_migrate):
     with ProgressReport(**progress_data) as pb:
         # schedule content migration into Pulp 3 using pre-migrated Pulp 2 content
         for plugin in plugins_to_migrate:
-            plugin_migrator = PLUGIN_MIGRATORS_MAP.get(plugin)
+            plugin_migrator = PLUGIN_MIGRATORS.get(plugin)
             content_migration_coros.append(plugin_migrator.migrate_content_to_pulp3())
 
             # only used for progress bar counters
-            content_types = [model.type for model in plugin_migrator.content_models]
+            content_types = plugin_migrator.content_models.keys()
             pulp2content_qs = Pulp2Content.objects.filter(pulp2_content_type_id__in=content_types,
                                                           pulp3_content=None)
             pb.total += pulp2content_qs.count()
@@ -86,19 +81,12 @@ async def migrate_importers(plugins_to_migrate):
     Args:
         plugins_to_migrate(list): A list of plugins which are being migrated.
     """
-    # import all needed plugin importer migration models
-    importer_models = {}
-    for plugin, importer_info in PULP_2TO3_IMPORTER_TYPE_MODEL_MAP.items():
+    # gather all needed plugin importer migrators
+    importer_migrators = {}
+    for plugin, plugin_migrator in PLUGIN_MIGRATORS.items():
         if plugin not in plugins_to_migrate:
             continue
-        module_path = 'pulp_2to3_migration.app.plugin.{plugin}.pulp3.repository'.format(
-            plugin=plugin
-        )
-        plugin_module = importlib.import_module(module_path)
-        for record in importer_info:
-            importer_type_id, model_name = record
-            importer_model = getattr(plugin_module, model_name)
-            importer_models[importer_type_id] = importer_model
+        importer_migrators.update(**plugin_migrator.importer_migrators)
 
     progress_data = dict(
         message='Migrating importers to Pulp 3', code='migrating.importers', total=0
@@ -109,8 +97,8 @@ async def migrate_importers(plugins_to_migrate):
         pb.save()
 
         for pulp2importer in pulp2importers_qs:
-            importer_model = importer_models.get(pulp2importer.pulp2_type_id)
-            remote, created = await importer_model.migrate_to_pulp3(pulp2importer)
+            importer_migrator = importer_migrators.get(pulp2importer.pulp2_type_id)
+            remote, created = await importer_migrator.migrate_to_pulp3(pulp2importer)
             pulp2importer.pulp3_remote = remote
             pulp2importer.is_migrated = True
             pulp2importer.save()

pulp_2to3_migration/app/models/__init__.py

@@ -10,5 +10,3 @@
     Pulp2RepoContent,
     Pulp2Repository,
 )
-# import all pulp_2to3 detail plugin models here
-from pulp_2to3_migration.app.plugin.iso.pulp3.models import Pulp2ISO  # noqa

pulp_2to3_migration/app/plugin/__init__.py

@@ -0,0 +1,8 @@
+import pkg_resources
+
+# { plugin_name: PluginMigratorClass }
+PLUGIN_MIGRATORS = {}
+
+if not PLUGIN_MIGRATORS:
+    for entry_point in pkg_resources.iter_entry_points(group='migrators'):
+        PLUGIN_MIGRATORS[entry_point.name] = entry_point.load()

pulp_2to3_migration/app/plugin/content.py

@@ -29,14 +29,14 @@
 )
 from pulpcore.plugin.tasking import WorkingDirectory
 
+from pulp_2to3_migration.app.constants import NOT_USED
 from pulp_2to3_migration.app.models import (
     Pulp2Content,
     Pulp2Importer,
     Pulp2LazyCatalog,
 )
 
 _logger = logging.getLogger(__name__)
-NOT_USED = 'Not Used'
 
 
 class DeclarativeContentMigration:
@@ -156,7 +156,7 @@ async def run(self):
         If a plugin needs to have more control over the order of content migration, it should
         override this method.
         """
-        content_types = [model.type for model in self.migrator.content_models]
+        content_types = self.migrator.content_models.keys()
         pulp2content_qs = Pulp2Content.objects.filter(pulp2_content_type_id__in=content_types,
                                                       pulp3_content=None)
         total_pulp2content = pulp2content_qs.count()
@@ -170,8 +170,8 @@ async def run(self):
         batch_count = math.ceil(total_pulp2content / batch_size)
 
         with ProgressReport(
-            message='Migrating {} content to Pulp 3'.format(self.migrator.type),
-            code='migrating.{}.content'.format(self.migrator.type),
+            message='Migrating {} content to Pulp 3'.format(self.migrator.pulp2_plugin),
+            code='migrating.{}.content'.format(self.migrator.pulp2_plugin),
             total=total_pulp2content
         ) as pb:
             # schedule content migration

pulp_2to3_migration/app/plugin/iso/migrator.py

@@ -0,0 +1,46 @@
+from pulp_2to3_migration.app.plugin.api import (
+    ContentMigrationFirstStage,
+    DeclarativeContentMigration,
+    Pulp2to3PluginMigrator,
+)
+
+from .pulp2_models import ISO
+from .pulp_2to3_models import Pulp2ISO
+from .repository import IsoImporter
+
+
+class IsoMigrator(Pulp2to3PluginMigrator):
+    """
+    An entry point for migration the Pulp 2 ISO plugin to Pulp 3.
+
+    Attributes:
+        pulp2_plugin(str): Pulp 2 plugin name
+        pulp2_content_models(dict): {'pulp2 content_type_id': 'content class to access MongoDB'}
+        pulp2_collection(str): a pulp2 collection which existence signifies that a plugin
+                               is installed in pulp2
+        pulp3_plugin(str): Pulp 3 plugin name
+        content_models(dict): {'pulp2 content_type_id': 'detail content class to pre-migrate to'}
+        importer_migrators(dict): {'importer_type_id': 'pulp_2to3 importer interface/migrator'}
+
+    """
+    pulp2_plugin = 'iso'
+    pulp2_content_models = {
+        'iso': ISO,
+    }
+    pulp2_collection = 'units_iso'
+    pulp3_plugin = 'pulp_file'
+    content_models = {
+        'iso': Pulp2ISO,
+    }
+    importer_migrators = {
+        'iso_importer': IsoImporter,
+    }
+
+    @classmethod
+    async def migrate_content_to_pulp3(cls):
+        """
+        Migrate pre-migrated Pulp 2 ISO content.
+        """
+        first_stage = ContentMigrationFirstStage(cls)
+        dm = DeclarativeContentMigration(first_stage=first_stage)
+        await dm.create()

pulp_2to3_migration/app/plugin/iso/pulp3/models.py → pulp_2to3_migration/app/plugin/iso/pulp_2to3_models.py

@@ -1,10 +1,11 @@
 from django.db import models
 
 from pulp_2to3_migration.app.models import Pulp2to3Content
-from pulp_2to3_migration.app.plugin.iso.pulp2.models import ISO
 
 from pulp_file.app.models import FileContent
 
+from .pulp2_models import ISO
+
 
 class Pulp2ISO(Pulp2to3Content):
     """

pulp_2to3_migration/app/plugin/migrator.py

@@ -2,14 +2,18 @@ class Pulp2to3PluginMigrator:
     """
     Class to serve as a plugin interface for migration to Pulp 3.
 
+    The attributes described below are expected to be defined by plugin writers.
+
     Attributes:
-        type(str): migrator type which corresponds to a Pulp 2 plugin name
-        content_models(tuple): Pulp2to3Content models this migrator is responsible for
+        pulp2_plugin(str): Pulp 2 plugin name
+        pulp2_content_models(dict): {'pulp2 content_type_id': 'content class to access MongoDB'}
+        pulp2_collection(str): a pulp2 collection which existence signifies that a plugin
+                               is installed in pulp2
+        pulp3_plugin(str): Pulp 3 plugin name
+        content_models(dict): {'pulp2 content_type_id': 'detail content class to pre-migrate to'}
+        importer_migrators(dict): {'importer_type_id': 'pulp_2to3 importer interface/migrator'}
 
     """
-    type = 'pulp2 plugin name'
-    content_models = ()
-
     @classmethod
     async def migrate_to_pulp3(cls):
         """

pulp_2to3_migration/app/pre_migration.py

@@ -1,5 +1,4 @@
 import asyncio
-import importlib
 import logging
 
 from collections import namedtuple
@@ -14,10 +13,6 @@
 from pulpcore.constants import TASK_STATES
 from pulpcore.plugin.models import ProgressReport
 
-from pulp_2to3_migration.app.constants import (
-    PULP_2TO3_CONTENT_MODEL_MAP,
-    SUPPORTED_PULP2_PLUGINS,
-)
 from pulp_2to3_migration.app.models import (
     Pulp2Content,
     Pulp2Distributor,
@@ -26,6 +21,7 @@
     Pulp2RepoContent,
     Pulp2Repository,
 )
+from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS
 from pulp_2to3_migration.pulp2.base import (
     Distributor,
     Importer,
@@ -48,23 +44,16 @@ async def pre_migrate_all_content(plugins_to_migrate):
     """
     pre_migrators = []
 
-    # import all pulp 2 content models
-    # (for each content type: one works with mongo and other - with postgresql)
-    for plugin, model_names in SUPPORTED_PULP2_PLUGINS.items():
+    # get all the content models for the migrating plugins
+    for plugin, plugin_migrator in PLUGIN_MIGRATORS.items():
         if plugin not in plugins_to_migrate:
             continue
-        pulp2_module_path = 'pulp_2to3_migration.app.plugin.{plugin}.pulp2.models'.format(
-            plugin=plugin)
-        pulp2_module = importlib.import_module(pulp2_module_path)
-        pulp_2to3_module = importlib.import_module('pulp_2to3_migration.app.models')
-        for pulp2_content_model_name in model_names:
+        for content_type in plugin_migrator.pulp2_content_models:
             # mongodb model
-            pulp2_content_model = getattr(pulp2_module, pulp2_content_model_name)
+            pulp2_content_model = plugin_migrator.pulp2_content_models[content_type]
 
             # postgresql model
-            content_type = pulp2_content_model.type
-            pulp_2to3_detail_model_name = PULP_2TO3_CONTENT_MODEL_MAP[content_type]
-            pulp_2to3_detail_model = getattr(pulp_2to3_module, pulp_2to3_detail_model_name)
+            pulp_2to3_detail_model = plugin_migrator.content_models[content_type]
 
             content_model = ContentModel(pulp2=pulp2_content_model,
                                          pulp_2to3_detail=pulp_2to3_detail_model)