Merge pull request #112 from phracek/prep_section

Prep section is ready to master.

rebasehelper/application.py

@@ -45,6 +45,7 @@ class Application(object):
     temp_dir = ""
     kwargs = {}
     old_sources = ""
+    rest_sources = []
     new_sources = ""
     spec_file = None
     spec_file_path = None
@@ -89,13 +90,14 @@ def __init__(self, cli_conf=None):
         self._get_spec_file()
         self._prepare_spec_objects()
 
+        # check the workspace dir
+        if not self.conf.cont:
+            self._check_workspace_dir()
+
         # TODO: Remove the value from kwargs and use only CLI attribute!
         self.kwargs['continue'] = self.conf.cont
         self._initialize_data()
 
-        # check the workspace dir
-        if not self.conf.cont:
-            self._check_workspace_dir()
         if self.conf.cont or self.conf.build_only:
             self._delete_old_builds()
 
@@ -140,7 +142,8 @@ def _prepare_spec_objects(self):
             logger.debug("argument passed as a new source is a version")
             version, extra_version = SpecFile.split_version_string(self.conf.sources)
             self.rebase_spec_file.set_version(version)
-            self.rebase_spec_file.set_extra_version(extra_version)
+            if extra_version:
+                self.rebase_spec_file.set_extra_version(extra_version)
 
     def _initialize_data(self):
         """
@@ -158,6 +161,9 @@ def _initialize_data(self):
             raise RebaseHelperError('You have to define new sources.')
         else:
             self.new_sources = os.path.abspath(self.conf.sources)
+        # Contains all source except the Source0
+        self.rest_sources = self.spec_file.get_sources()[1:]
+        self.rest_sources = [os.path.abspath(x) for x in self.rest_sources]
 
     def _get_rebase_helper_log(self):
         return os.path.join(self.results_dir, settings.REBASE_HELPER_RESULTS_LOG)
@@ -238,7 +244,7 @@ def extract_archive(archive_path, destination):
             raise RebaseHelperError('%s. Supported archives are %s', ni_e.message, Archive.get_supported_archives())
 
         try:
-            archive.extract(destination)
+            archive.extract_archive(destination)
         except IOError:
             raise RebaseHelperError("Archive '%s' can not be extracted", archive_path)
         except (EOFError, SystemError):
@@ -282,23 +288,32 @@ def prepare_sources(self):
         new_dir = Application.extract_sources(self.new_sources,
                                               os.path.join(self.execution_dir, settings.NEW_SOURCES_DIR))
 
+        # This copies other sources to extracted sources marked as 0
+        for rest in self.rest_sources:
+            for source_dir in [old_dir, new_dir]:
+                archive = [x for x in Archive.get_supported_archives() if rest.endswith(x)]
+                # if the source is a remote file, download it
+                if archive:
+                    Application.extract_sources(rest, os.path.join(self.execution_dir, source_dir))
+
         return [old_dir, new_dir]
 
     def patch_sources(self, sources):
         # Patch sources
         git_helper = GitHelper(sources[0])
         git_helper.check_git_config()
         patch = Patcher(self.conf.patchtool)
-
         self.rebase_spec_file.update_changelog(self.rebase_spec_file.get_new_log(git_helper))
         try:
             self.rebased_patches = patch.patch(sources[0],
                                                sources[1],
+                                               self.rest_sources,
                                                git_helper,
                                                self.spec_file.get_applied_patches(),
+                                               self.spec_file.get_prep_section(),
                                                **self.kwargs)
         except RuntimeError as run_e:
-            raise RebaseHelperError(run_e.message)
+            raise RebaseHelperError('Patching failed')
         self.rebase_spec_file.write_updated_patches(self.rebased_patches)
         if self.conf.non_interactive:
             OutputLogger.set_patch_output('Unapplied patches:', self.rebased_patches['unapplied'])

rebasehelper/archive.py

@@ -19,9 +19,11 @@
 #
 # Authors: Petr Hracek <phracek@redhat.com>
 #          Tomas Hozza <thozza@redhat.com>
-
+from __future__ import print_function
 import tarfile
 import zipfile
+import bz2
+import os
 try:
     import lzma
 except ImportError:
@@ -59,6 +61,15 @@ def open(cls, filename=None, *args, **kwargs):
         """
         raise NotImplementedError()
 
+    @classmethod
+    def extract(cls, filename=None, *args, **kwargs):
+        """
+        Extracts the archive into the given path
+
+        :param path: Path where to extract the archive to.
+        :return:
+        """
+        raise NotImplementedError()
 
 @register_archive_type
 class TarXzArchiveType(ArchiveTypeBase):
@@ -82,13 +93,18 @@ def open(cls, filename=None):
 
         return tarfile.open(mode='r', fileobj=xz_file)
 
+    @classmethod
+    def extract(cls, archive=None, filename=None, path=None, *args, **kwargs):
+        if archive is None:
+            raise TypeError("Expected argument 'archive' (pos 1) is missing")
+        archive.extractall(path)
 
 @register_archive_type
 class TarBz2ArchiveType(ArchiveTypeBase):
 
-    """ .tar.bz2 archive type """
+    """ .bz2 archive type """
 
-    EXTENSION = ".tar.bz2"
+    EXTENSION = ".bz2"
 
     @classmethod
     def match(cls, filename=None):
@@ -102,7 +118,23 @@ def open(cls, filename=None):
         if filename is None:
             raise TypeError("Expected argument 'filename' (pos 1) is missing")
 
-        return tarfile.TarFile.open(filename)
+        if filename.endswith('.tar.bz2'):
+            return tarfile.TarFile.open(filename)
+        else:
+            return bz2.BZ2File(filename)
+
+    @classmethod
+    def extract(cls, archive=None, filename=None, path=None, *args, **kwargs):
+        if archive is None:
+            raise TypeError("Expected argument 'archive' (pos 1) is missing")
+        if filename.endswith('tar.bz2'):
+            archive.extractall(path)
+        else:
+            data = archive.read()
+            if not os.path.exists(path):
+                os.mkdir(path)
+            with open(os.path.join(path, filename[:-4]), 'w') as f:
+                f.write(data)
 
 
 @register_archive_type
@@ -126,6 +158,11 @@ def open(cls, filename=None):
 
         return tarfile.TarFile.open(filename)
 
+    @classmethod
+    def extract(cls, archive=None, filename=None, path=None, *args, **kwargs):
+        if archive is None:
+            raise TypeError("Expected argument 'archive' (pos 1) is missing")
+        archive.extractall(path)
 
 @register_archive_type
 class TgzArchiveType(TarGzArchiveType):
@@ -159,6 +196,12 @@ def open(cls, filename=None):
 
         return zipfile.ZipFile(filename, "r")
 
+    @classmethod
+    def extract(cls, archive=None, filename=None, path=None, *args, **kwargs):
+        if archive is None:
+            raise TypeError("Expected argument 'archive' (pos 1) is missing")
+        archive.extractall(path)
+
 
 class Archive(object):
 
@@ -177,7 +220,7 @@ def __init__(self, filename=None):
         if self._archive_type is None:
             raise NotImplementedError("Unsupported archive type")
 
-    def extract(self, path=None):
+    def extract_archive(self, path=None):
         """
         Extracts the archive into the given path
 
@@ -190,7 +233,7 @@ def extract(self, path=None):
         logger.debug("Extracting '%s' into '%s'", self._filename, path)
 
         archive = self._archive_type.open(self._filename)
-        archive.extractall(path)
+        self._archive_type.extract(archive, self._filename, path)
         archive.close()
 
     @classmethod

rebasehelper/patch_helper.py

@@ -22,8 +22,10 @@
 
 from __future__ import print_function
 import os
+import re
 from rebasehelper.logger import logger
 from rebasehelper.utils import ConsoleHelper, defenc
+from rebasehelper.utils import ProcessHelper
 from rebasehelper.utils import GitHelper, GitRebaseError
 
 #from git import Repo
@@ -50,7 +52,7 @@ def match(cls, cmd=None, *args, **kwargs):
         return NotImplementedError()
 
     @classmethod
-    def run_patch(cls, old_dir, new_dir, git_helper, patches, *args, **kwargs):
+    def run_patch(cls, old_dir, new_dir, rest_sources, git_helper, patches, *args, **kwargs):
         """Method will check all patches in relevant package"""
         return NotImplementedError()
 
@@ -70,6 +72,9 @@ class GitPatchTool(PatchBase):
     git_helper = None
     non_interactive = False
     patches = None
+    prep_section = False
+    exec_prep_script = False
+    patch_sources_by_prep_script = False
 
     @classmethod
     def match(cls, cmd=None):
@@ -79,7 +84,7 @@ def match(cls, cmd=None):
             return False
 
     @staticmethod
-    def apply_patch(git_helper, patch_name):
+    def apply_patch(git_helper, patch_object):
         """
         Function applies patches to old sources
         It tries apply patch with am command and if it fails
@@ -88,13 +93,15 @@ def apply_patch(git_helper, patch_name):
         """
         logger.debug('Applying patch with am')
 
+        patch_name = patch_object.get_path()
+        patch_option = patch_object.get_option()
         ret_code = git_helper.command_am(input_file=patch_name)
         if int(ret_code) != 0:
             git_helper.command_am(parameters='--abort', input_file=patch_name)
             logger.debug('Applying patch with git am failed.')
-            ret_code = git_helper.command_apply(input_file=patch_name)
+            ret_code = git_helper.command_apply(input_file=patch_name, option=patch_option)
             if int(ret_code) != 0:
-                ret_code = git_helper.command_apply(input_file=patch_name, ignore_space=True)
+                ret_code = git_helper.command_apply(input_file=patch_name, option=patch_option, ignore_space=True)
             ret_code = GitPatchTool.commit_patch(git_helper, patch_name)
         return ret_code
 
@@ -104,7 +111,10 @@ def _prepare_git(cls, upstream_name):
         cls.git_helper.command_fetch(upstream_name)
         cls.output_data = cls.git_helper.command_log(parameters='--pretty=oneline')
         logger.debug('Outputdata from git log %s', cls.output_data)
-        last_hash = GitHelper.get_commit_hash_log(cls.output_data, number=0)
+        number = 0
+        if cls.prep_section:
+            number = 1
+        last_hash = GitHelper.get_commit_hash_log(cls.output_data, number=number)
         init_hash = GitHelper.get_commit_hash_log(cls.output_data, len(cls.output_data)-1)
         return init_hash, last_hash
 
@@ -193,14 +203,67 @@ def commit_patch(git_helper, patch_name):
     def apply_old_patches(cls):
         """Function applies a patch to a old/new sources"""
         for patch in cls.patches:
-            patch_path = patch.get_path()
-            logger.info("Applying patch '%s' to '%s'", os.path.basename(patch_path), os.path.basename(cls.source_dir))
-            ret_code = GitPatchTool.apply_patch(cls.git_helper, patch_path)
+            logger.info("Applying patch '%s' to '%s'", os.path.basename(patch.get_path()), os.path.basename(cls.source_dir))
+            ret_code = GitPatchTool.apply_patch(cls.git_helper, patch)
             # unexpected
             if int(ret_code) != 0:
                 if cls.source_dir == cls.old_sources:
                     raise RuntimeError('Failed to patch old sources')
 
+    @classmethod
+    def _prepare_prep_script(cls, sources, prep):
+        for src in sources:
+            file_name = os.path.join('SOURCES', os.path.basename(src))
+            for index, row in enumerate(prep):
+                if file_name in row:
+                    src_path = [x for x in row.split() if x.endswith(file_name)]
+                    prep[index] = row.replace(src_path[0], src)
+
+        return prep
+
+    @classmethod
+    def create_prep_script(cls, prep):
+
+        """Function abstract special things from prep section and apply them to old sources"""
+
+        logger.debug('Extract prep script')
+        # Check whether patch or git am is used inside %prep section
+        # If yes then execute whole %prep section
+        logger.debug("prep section '%s'", prep)
+        found_patching = [x for x in prep if ' patch ' in x]
+        if found_patching:
+            cls.exec_prep_script = True
+        found_git_am = [x for x in prep if 'git am' in x]
+        if found_git_am:
+            cls.patch_sources_by_prep_script = True
+
+        logger.debug('Fix %SOURCES tags in prep script')
+        prep = cls._prepare_prep_script(cls.rest_sources, prep)
+        logger.debug('Fix %PATCH tags in prep script')
+        prep = cls._prepare_prep_script([x.get_path() for x in cls.patches], prep)
+        prep_script_path = os.path.join(cls.kwargs['results_dir'], 'prep_script')
+        logger.debug("Writing Prep script '%s' to the disc", prep_script_path)
+        try:
+            with open(prep_script_path, "w") as f:
+                f.write("#!/bin/bash\n\n")
+                f.writelines('\n'.join(prep))
+            os.chmod(prep_script_path, 0755)
+        except IOError:
+            logger.debug("Unable to write prep script file to '%s'", prep_script_path)
+            return None
+
+        return prep_script_path
+
+    @classmethod
+    def call_prep_script(cls, prep_script_path):
+        cwd = os.getcwd()
+        os.chdir(cls.old_sources)
+        ProcessHelper.run_subprocess(prep_script_path, output=os.path.join(cls.kwargs['results_dir'], 'prep_script.log'))
+        if not cls.patch_sources_by_prep_script:
+            cls.git_helper.command_add_files(parameters=["--all"])
+            cls.git_helper.command_commit(message="prep_script prep_corrections")
+        os.chdir(cwd)
+
     @classmethod
     def init_git(cls, directory):
 
@@ -212,7 +275,7 @@ def init_git(cls, directory):
         gh.command_commit(message='Initial Commit')
 
     @classmethod
-    def run_patch(cls, old_dir, new_dir, git_helper, patches, **kwargs):
+    def run_patch(cls, old_dir, new_dir, rest_sources, git_helper, patches, prep, **kwargs):
 
         """
         The function can be used for patching one
@@ -223,14 +286,20 @@ def run_patch(cls, old_dir, new_dir, git_helper, patches, **kwargs):
         cls.new_sources = new_dir
         cls.output_data = []
         cls.cont = cls.kwargs['continue']
+        cls.rest_sources = rest_sources
         cls.git_helper = git_helper
         cls.patches = patches
         cls.non_interactive = kwargs.get('non_interactive')
         if not os.path.isdir(os.path.join(cls.old_sources, '.git')):
             cls.init_git(old_dir)
             cls.init_git(new_dir)
             cls.source_dir = cls.old_sources
-            cls.apply_old_patches()
+            prep_path = cls.create_prep_script(prep)
+            if not cls.patch_sources_by_prep_script:
+                cls.apply_old_patches()
+            if cls.exec_prep_script or cls.patch_sources_by_prep_script:
+                logger.info('Executing prep script')
+                cls.call_prep_script(prep_path)
             cls.cont = False
 
         return cls._git_rebase()
@@ -261,7 +330,7 @@ def __init__(self, tool=None):
         if self._tool is None:
             raise NotImplementedError("Unsupported patch tool")
 
-    def patch(self, old_dir, new_dir, git_helper, patches, **kwargs):
+    def patch(self, old_dir, new_dir, rest_sources, git_helper, patches, prep, **kwargs):
         """
         Apply patches and generate rebased patches if needed
 
@@ -273,7 +342,7 @@ def patch(self, old_dir, new_dir, git_helper, patches, **kwargs):
         :return:
         """
         logger.debug("Patching source by patch tool %s", self._path_tool_name)
-        return self._tool.run_patch(old_dir, new_dir, git_helper, patches, **kwargs)
+        return self._tool.run_patch(old_dir, new_dir, rest_sources, git_helper, patches, prep, **kwargs)