many: detect local source changes

snapcraft/_baseplugin.py

@@ -120,6 +120,10 @@ def __init__(self, name, options, project=None):
         else:
             self.builddir = self.build_basedir
 
+        # By default, snapcraft does an in-source build. Set this property to
+        # True if that's not desired.
+        self.out_of_source_build = False
+
     # The API
     def pull(self):
         """Pull the source code and/or internal prereqs to build the part."""

snapcraft/file_utils.py

@@ -106,18 +106,18 @@ def link_or_copy(source: str, destination: str,
     """
 
     try:
-        _link(source, destination, follow_symlinks)
+        link(source, destination, follow_symlinks)
     except OSError as e:
         if e.errno == errno.EEXIST and not os.path.isdir(destination):
             # os.link will fail if the destination already exists, so let's
             # remove it and try again.
             os.remove(destination)
             link_or_copy(source, destination, follow_symlinks)
         else:
-            _copy(source, destination, follow_symlinks)
+            copy(source, destination, follow_symlinks)
 
 
-def _link(source: str, destination: str, follow_symlinks: bool=False) -> None:
+def link(source: str, destination: str, follow_symlinks: bool=False) -> None:
     # Note that follow_symlinks doesn't seem to work for os.link, so we'll
     # implement this logic ourselves using realpath.
     source_path = source
@@ -137,7 +137,7 @@ def _link(source: str, destination: str, follow_symlinks: bool=False) -> None:
         raise SnapcraftCopyFileNotFoundError(source)
 
 
-def _copy(source: str, destination: str, follow_symlinks: bool=False) -> None:
+def copy(source: str, destination: str, follow_symlinks: bool=False) -> None:
     # If os.link raised an I/O error, it may have left a file behind. Skip on
     # OSError in case it doesn't exist or is a directory.
     with suppress(OSError):

snapcraft/internal/errors.py

@@ -61,12 +61,16 @@ class StepOutdatedError(SnapcraftError):
         '`snapcraft clean {parts_names} -s {step.name}`.'
     )
 
-    def __init__(self, *, step, part, dirty_report=None, dependents=None):
+    def __init__(self, *, step, part, dirty_report=None, outdated_report=None,
+                 dependents=None):
         messages = []
 
         if dirty_report:
             messages.append(dirty_report.get_report())
 
+        if outdated_report:
+            messages.append(outdated_report.get_report())
+
         if dependents:
             humanized_dependents = formatting_utils.humanize_list(
                 dependents, 'and')

snapcraft/internal/lifecycle/_runner.py

@@ -202,10 +202,17 @@ def run(self, step: steps.Step, part_names=None):
                             getattr(self, '_re{}'.format(
                                 current_step.name))(part)
                         else:
-                            notify_part_progress(
-                                part,
-                                'Skipping {}'.format(current_step.name),
-                                '(already ran)')
+                            outdated_report = self._cache.get_outdated_report(
+                                part, current_step)
+                            if outdated_report:
+                                self._handle_outdated(
+                                    part, current_step, outdated_report,
+                                    cli_config)
+                            else:
+                                notify_part_progress(
+                                    part,
+                                    'Skipping {}'.format(current_step.name),
+                                    '(already ran)')
                     else:
                         getattr(self, '_run_{}'.format(
                             current_step.name))(part)
@@ -244,7 +251,8 @@ def _reprime(self, part, hint=''):
         self._rerun_step(
             step=steps.PRIME, part=part, progress='Re-priming', hint=hint)
 
-    def _run_step(self, *, step: steps.Step, part, progress, hint=''):
+    def _prepare_to_run(self, *, step: steps.Step,
+                        part: pluginhandler.PluginHandler):
         common.reset_env()
         all_dependencies = self.parts_config.get_dependencies(part.name)
 
@@ -277,11 +285,18 @@ def _run_step(self, *, step: steps.Step, part, progress, hint=''):
 
         part = _replace_in_part(part)
 
+    def _run_step(self, *, step: steps.Step, part, progress, hint=''):
+        self._prepare_to_run(step=step, part=part)
+
         notify_part_progress(part, progress, hint)
         getattr(part, step.name)()
 
         # We know we just ran this step, so rather than check, manually twiddle
         # the cache
+        self._step_complete(part, step)
+
+    def _step_complete(self, part, step):
+        self._cache.clear_step(part, step)
         self._cache.add_step_run(part, step)
         self.steps_were_run = True
 
@@ -316,6 +331,28 @@ def _handle_dirty(self, part, step, dirty_report, cli_config):
         getattr(self, '_re{}'.format(step.name))(part, hint='({})'.format(
             dirty_report.get_summary()))
 
+    def _handle_outdated(self, part, step, outdated_report, cli_config):
+        dirty_action = cli_config.get_outdated_step_action()
+        if not step.clean_if_dirty:
+            if dirty_action == config.OutdatedStepAction.ERROR:
+                raise errors.StepOutdatedError(
+                    step=step, part=part.name, outdated_report=outdated_report)
+
+        update_function = getattr(part, 'update_{}'.format(step.name), None)
+        if update_function:
+            self._prepare_to_run(step=step, part=part)
+            notify_part_progress(
+                part, 'Updating {} step for'.format(step.name), '({})'.format(
+                    outdated_report.get_summary()))
+            update_function()
+
+            # We know we just ran this step, so rather than check, manually
+            # twiddle the cache
+            self._step_complete(part, step)
+        else:
+            getattr(self, '_re{}'.format(step.name))(part, '({})'.format(
+                outdated_report.get_summary()))
+
 
 def notify_part_progress(part, progress, hint='', debug=False):
     if debug:

snapcraft/internal/lifecycle/_status_cache.py

@@ -34,6 +34,7 @@ def __init__(self, config: _config.Config) -> None:
         """
         self.config = config
         self._steps_run = dict()  # type: Dict[str, Set[steps.Step]]
+        self._outdated_reports = collections.defaultdict(dict)  # type: _Report
         self._dirty_reports = collections.defaultdict(dict)  # type: _Report
 
     def should_step_run(self, part: pluginhandler.PluginHandler,
@@ -48,9 +49,12 @@ def should_step_run(self, part: pluginhandler.PluginHandler,
         A given step should run if it:
             1. Hasn't yet run
             2. Is dirty
-            3. Either (1) or (2) apply to any earlier steps in its lifecycle
+            3. Is outdated
+            4. Either (1), (2), or (3) apply to any earlier steps in the part's
+               lifecycle
         """
         if (not self.has_step_run(part, step) or
+                self.get_outdated_report(part, step) is not None or
                 self.get_dirty_report(part, step) is not None):
             return True
 
@@ -82,6 +86,17 @@ def has_step_run(self, part: pluginhandler.PluginHandler,
         self._ensure_steps_run(part)
         return step in self._steps_run[part.name]
 
+    def get_outdated_report(self, part, step):
+        """Obtain the outdated report for a given step of the given part.
+
+        :param pluginhandler.PluginHandler part: Part in question.
+        :param steps.Step step: Step in question.
+        :return: Outdated report (could be None)
+        :rtype: pluginhandler.OutdatedReport
+        """
+        self._ensure_outdated_report(part, step)
+        return self._outdated_reports[part.name][step]
+
     def get_dirty_report(self, part: pluginhandler.PluginHandler,
                          step: steps.Step) -> pluginhandler.DirtyReport:
         """Obtain the dirty report for a given step of the given part.
@@ -107,6 +122,9 @@ def clear_step(self, part: pluginhandler.PluginHandler,
             _remove_key(self._steps_run[part.name], step)
             if not self._steps_run[part.name]:
                 _del_key(self._steps_run, part.name)
+        _del_key(self._outdated_reports[part.name], step)
+        if not self._outdated_reports[part.name]:
+            _del_key(self._outdated_reports, part.name)
         _del_key(self._dirty_reports[part.name], step)
         if not self._dirty_reports[part.name]:
             _del_key(self._dirty_reports, part.name)
@@ -115,6 +133,12 @@ def _ensure_steps_run(self, part: pluginhandler.PluginHandler) -> None:
         if part.name not in self._steps_run:
             self._steps_run[part.name] = _get_steps_run(part)
 
+    def _ensure_outdated_report(self, part: pluginhandler.PluginHandler,
+                                step: steps.Step) -> None:
+        if step not in self._outdated_reports[part.name]:
+            self._outdated_reports[part.name][step] = part.get_outdated_report(
+                step)
+
     def _ensure_dirty_report(self, part: pluginhandler.PluginHandler,
                              step: steps.Step) -> None:
         # If we already have a dirty report, bail

snapcraft/internal/pluginhandler/__init__.py

@@ -46,6 +46,7 @@
 from ._runner import Runner
 from ._patchelf import PartPatcher
 from ._dirty_report import Dependency, DirtyReport  # noqa
+from ._outdated_report import OutdatedReport
 
 logger = logging.getLogger(__name__)
 
@@ -249,6 +250,32 @@ def is_clean(self, step):
         except errors.NoLatestStepError:
             return True
 
+    def is_outdated(self, step):
+        """Return true if the given step needs to be updated (no cleaning)."""
+
+        return self.get_outdated_report(step) is not None
+
+    def get_outdated_report(self, step: steps.Step):
+        """Return an OutdatedReport class describing why step is outdated.
+
+        Returns None if step is not outdated.
+        """
+
+        try:
+            return getattr(self, 'check_{}'.format(step.name))()
+        except AttributeError:
+            with contextlib.suppress(errors.StepHasNotRunError):
+                timestamp = self.step_timestamp(step)
+
+                for previous_step in reversed(step.previous_steps()):
+                    # Has a previous step run since this one ran? Then this
+                    # step needs to be updated.
+                    with contextlib.suppress(errors.StepHasNotRunError):
+                        if timestamp < self.step_timestamp(previous_step):
+                            return OutdatedReport(
+                                previous_step_modified=previous_step)
+            return None
+
     def is_dirty(self, step):
         """Return true if the given step needs to be cleaned and run again."""
 
@@ -301,11 +328,6 @@ def mark_done(self, step, state=None):
                 self.plugin.statedir, step), 'w') as f:
             f.write(yaml.dump(state))
 
-        # We know we've only just completed this step, so make sure any later
-        # steps don't have a saved state.
-        for step in step.next_steps():
-                self.mark_cleaned(step)
-
     def mark_cleaned(self, step):
         state_file = states.get_step_state_file(self.plugin.statedir, step)
         if os.path.exists(state_file):
@@ -349,6 +371,21 @@ def pull(self, force=False):
         self._runner.pull()
         self.mark_pull_done()
 
+    def check_pull(self):
+        # Check to see if pull needs to be updated
+        state_file = states.get_step_state_file(
+            self.plugin.statedir, steps.PULL)
+
+        # Not all sources support checking for updates
+        with contextlib.suppress(NotImplementedError):
+            if self.source_handler.check(state_file):
+                return OutdatedReport(source_updated=True)
+        return None
+
+    def update_pull(self):
+        self.source_handler.update()
+        self.mark_pull_done()
+
     def _do_pull(self):
         if self.source_handler:
             self.source_handler.pull()
@@ -406,28 +443,48 @@ def prepare_build(self, force=False):
     def build(self, force=False):
         self.makedirs()
 
-        if os.path.exists(self.plugin.build_basedir):
-            shutil.rmtree(self.plugin.build_basedir)
-
-        # FIXME: It's not necessary to ignore here anymore since it's now done
-        # in the Local source. However, it's left here so that it continues to
-        # work on old snapcraft trees that still have src symlinks.
-        def ignore(directory, files):
-            if directory == self.plugin.sourcedir:
-                snaps = glob(os.path.join(directory, '*.snap'))
-                if snaps:
-                    snaps = [os.path.basename(s) for s in snaps]
-                    return common.SNAPCRAFT_FILES + snaps
+        if not self.plugin.out_of_source_build:
+            if os.path.exists(self.plugin.build_basedir):
+                shutil.rmtree(self.plugin.build_basedir)
+
+            # FIXME: It's not necessary to ignore here anymore since it's now
+            # done in the Local source. However, it's left here so that it
+            # continues to work on old snapcraft trees that still have src
+            # symlinks.
+            def ignore(directory, files):
+                if directory == self.plugin.sourcedir:
+                    snaps = glob(os.path.join(directory, '*.snap'))
+                    if snaps:
+                        snaps = [os.path.basename(s) for s in snaps]
+                        return common.SNAPCRAFT_FILES + snaps
+                    else:
+                        return common.SNAPCRAFT_FILES
                 else:
-                    return common.SNAPCRAFT_FILES
-            else:
-                return []
+                    return []
+
+            # No hard-links being used here in case the build process modifies
+            # these files.
+            shutil.copytree(self.plugin.sourcedir, self.plugin.build_basedir,
+                            symlinks=True, ignore=ignore)
+
+        self._do_build()
+
+    def update_build(self):
+        if not self.plugin.out_of_source_build:
+            # Use the local source to update. It's important to use
+            # file_utils.copy instead of link_or_copy, as the build process
+            # may modify these files
+            source = sources.Local(
+                self.plugin.sourcedir, self.plugin.build_basedir,
+                copy_function=file_utils.copy)
+            if not source.check(states.get_step_state_file(
+                    self.plugin.statedir, steps.BUILD)):
+                return
+            source.update()
 
-        # No hard-links being used here in case the build process modifies
-        # these files.
-        shutil.copytree(self.plugin.sourcedir, self.plugin.build_basedir,
-                        symlinks=True, ignore=ignore)
+        self._do_build()
 
+    def _do_build(self):
         self._runner.prepare()
         self._runner.build()
         self._runner.install()