Fix multiple bugs related to modularity

Additionally, refactor the module publish step to be more similar to the
others.

closes #4252
https://pulp.plan.io/issues/4252
closes #4268
https://pulp.plan.io/issues/4268
closes #4253
https://pulp.plan.io/issues/4253

plugins/pulp_rpm/plugins/distributors/yum/metadata/modules.py

@@ -0,0 +1,34 @@
+import hashlib
+import os
+
+from pulp.plugins.util.metadata_writer import MetadataFileContext
+from pulp.server.exceptions import PulpCodedException
+
+from pulp_rpm.common.constants import CONFIG_DEFAULT_CHECKSUM
+from pulp_rpm.plugins import error_codes
+from pulp_rpm.plugins.distributors.yum.metadata.metadata import REPO_DATA_DIR_NAME
+from pulp_rpm.yum_plugin import util
+
+
+_LOG = util.getLogger(__name__)
+
+MODULES_FILE_NAME = 'modules.yaml.gz'
+
+
+class ModulesFileContext(MetadataFileContext):
+    def __init__(self, working_dir, checksum_type=CONFIG_DEFAULT_CHECKSUM):
+        metadata_file_path = os.path.join(working_dir, REPO_DATA_DIR_NAME, MODULES_FILE_NAME)
+        super(ModulesFileContext, self).__init__(metadata_file_path, checksum_type)
+
+    # def initialize(self):
+    #     super(ModulesFileContext, self).initialize()
+
+    # def finalize(self):
+    #     super(ModulesFileContext, self).finalize()
+
+    def add_document(self, document, precomputed_checksum):
+        checksum = hashlib.sha256(document).hexdigest()
+        if checksum == precomputed_checksum:
+            self.metadata_file_handle.write(document)
+        else:
+            raise PulpCodedException(error_codes.RPM1017)

plugins/pulp_rpm/plugins/distributors/yum/publish.py

@@ -1,7 +1,5 @@
 import copy
 import datetime
-import hashlib
-import gzip
 import itertools
 import os
 import subprocess
@@ -23,20 +21,21 @@
 from pulp_rpm.common import constants, ids, file_utils
 from pulp_rpm.yum_plugin import util
 from pulp_rpm.plugins import error_codes
+from pulp_rpm.plugins.importers.yum.repomd import modules
 from pulp_rpm.plugins.db import models
 from pulp_rpm.plugins.distributors.export_distributor import export_utils
 from pulp_rpm.plugins.distributors.export_distributor import generate_iso
-from pulp_rpm.plugins.importers.yum.repomd import modules
 from pulp_rpm.plugins.importers.yum.parse.treeinfo import KEY_PACKAGEDIR
 from . import configuration
 from .metadata.filelists import FilelistsXMLFileContext
 from .metadata.metadata import REPO_DATA_DIR_NAME
+from .metadata.modules import ModulesFileContext
 from .metadata.other import OtherXMLFileContext
 from .metadata.prestodelta import PrestodeltaXMLFileContext
 from .metadata.primary import PrimaryXMLFileContext
+from .metadata.package import PackageXMLFileContext
 from .metadata.repomd import RepomdXMLFileContext
 from .metadata.updateinfo import UpdateinfoXMLFileContext
-from .metadata.package import PackageXMLFileContext
 
 
 logger = util.getLogger(__name__)
@@ -810,6 +809,7 @@ class PublishErrataStepIncremental(platform_steps.UnitModelPluginStep):
     """
     Publish all incremental errata
     """
+
     def __init__(self, **kwargs):
         super(PublishErrataStepIncremental, self).__init__(constants.PUBLISH_ERRATA_STEP,
                                                            [models.Errata], **kwargs)
@@ -839,61 +839,60 @@ class PublishModulesStep(platform_steps.UnitModelPluginStep):
     :type fp_out: file-like
     """
 
-    FILE_NAME = 'modules.yaml.gz'
-
     def __init__(self):
         super(PublishModulesStep, self).__init__(
             constants.PUBLISH_MODULES_STEP,
             [
                 models.Modulemd,
                 models.ModulemdDefaults
             ])
+        self.context = None
         self.description = _('Publishing Modules')
-        self.fp_out = None
+
+    def is_skipped(self):
+        """
+        Test to find out if the step should be skipped.
+
+        :return: whether or not the step should be skipped
+        :rtype:  bool
+        """
+        # skip if there are no modules files.
+        if self.get_total() == 0:
+            return True
+
+        return super(PublishModulesStep, self).is_skipped()
 
     def initialize(self):
         """
         Create the directory and open fp_out for writing.
         """
-        path = os.path.join(
-            self.get_working_dir(),
-            REPO_DATA_DIR_NAME,
-            self.FILE_NAME)
-        plugin_misc.mkdir(os.path.dirname(path))
-        self.fp_out = gzip.open(path, 'wb')
+        checksum_type = self.parent.get_checksum_type()
+        self.context = ModulesFileContext(self.get_working_dir(), checksum_type=checksum_type)
+        self.context.initialize()
 
     def process_main(self, item=None):
         """
         Append each yaml document to the modules.yaml.
 
         :param item: Either Modulemd or ModulemdDefaults.
-        :type item: pulp.server.db.model.FileContent
+        :type item: pulp.server.db.model.FileContent (baseclass)
         """
         with open(item.storage_path) as fp_in:
             document = fp_in.read()
-            checksum = hashlib.sha256(document).hexdigest()
-            if checksum == item.checksum:
-                self.fp_out.write(document)
-            else:
-                raise PulpCodedException(error_codes.RPM1017)
+            self.context.add_document(document, precomputed_checksum=item.checksum)
 
     def finalize(self):
         """
         Close the fp_out; rename the file to include the checksum;
         and add to the repomd.
         """
-        if self.fp_out:
-            self.fp_out.close()
-        with open(self.fp_out.name, 'rb') as fp_in:
-            checksum = file_utils.calculate_checksum(fp_in)
-        _dir, name = os.path.split(self.fp_out.name)
-        path = os.path.join(_dir, '-'.join([checksum, name]))
-        os.rename(self.fp_out.name, path)
-        repomd = self.parent.repomd_file_context
-        repomd.add_metadata_file_metadata(
-            data_type=modules.METADATA_FILE_NAME,
-            file_path=path,
-            precalculated_checksum=checksum)
+        if self.context:
+            self.context.finalize()
+            self.parent.repomd_file_context.add_metadata_file_metadata(
+                data_type=modules.METADATA_FILE_NAME,
+                file_path=self.context.metadata_file_path,
+                precalculated_checksum=self.context.checksum
+            )
 
 
 class PublishCompsStep(platform_steps.UnitModelPluginStep):