many: detect local source changes

snapcraft/_baseplugin.py

@@ -120,6 +120,10 @@ def __init__(self, name, options, project=None):
         else:
             self.builddir = self.build_basedir
 
+        # By default, snapcraft does an in-source build. Set this property to
+        # True if that's not desired.
+        self.out_of_source_build = False
+
     # The API
     def pull(self):
         """Pull the source code and/or internal prereqs to build the part."""

snapcraft/file_utils.py

@@ -106,18 +106,27 @@ def link_or_copy(source: str, destination: str,
     """
 
     try:
-        _link(source, destination, follow_symlinks)
+        link(source, destination, follow_symlinks=follow_symlinks)
     except OSError as e:
         if e.errno == errno.EEXIST and not os.path.isdir(destination):
             # os.link will fail if the destination already exists, so let's
             # remove it and try again.
             os.remove(destination)
             link_or_copy(source, destination, follow_symlinks)
         else:
-            _copy(source, destination, follow_symlinks)
+            copy(source, destination, follow_symlinks=follow_symlinks)
 
 
-def _link(source: str, destination: str, follow_symlinks: bool=False) -> None:
+def link(source: str, destination: str, *,
+         follow_symlinks: bool=False) -> None:
+    """Hard-link source and destination files.
+
+    :param str source: The source to which destination will be linked.
+    :param str destination: The destination to be linked to source.
+    :param bool follow_symlinks: Whether or not symlinks should be followed.
+
+    :raises SnapcraftCopyFileNotFoundError: If source doesn't exist.
+    """
     # Note that follow_symlinks doesn't seem to work for os.link, so we'll
     # implement this logic ourselves using realpath.
     source_path = source
@@ -137,7 +146,19 @@ def _link(source: str, destination: str, follow_symlinks: bool=False) -> None:
         raise SnapcraftCopyFileNotFoundError(source)
 
 
-def _copy(source: str, destination: str, follow_symlinks: bool=False) -> None:
+def copy(source: str, destination: str, *,
+         follow_symlinks: bool=False) -> None:
+    """Copy source and destination files.
+
+    This function overwrites the destination if it already exists, and also
+    tries to copy ownership information.
+
+    :param str source: The source to be copied to destination.
+    :param str destination: Where to put the copy.
+    :param bool follow_symlinks: Whether or not symlinks should be followed.
+
+    :raises SnapcraftCopyFileNotFoundError: If source doesn't exist.
+    """
     # If os.link raised an I/O error, it may have left a file behind. Skip on
     # OSError in case it doesn't exist or is a directory.
     with suppress(OSError):

snapcraft/internal/errors.py

@@ -61,12 +61,16 @@ class StepOutdatedError(SnapcraftError):
         '`snapcraft clean {parts_names} -s {step.name}`.'
     )
 
-    def __init__(self, *, step, part, dirty_report=None, dependents=None):
+    def __init__(self, *, step, part, dirty_report=None, outdated_report=None,
+                 dependents=None):
         messages = []
 
         if dirty_report:
             messages.append(dirty_report.get_report())
 
+        if outdated_report:
+            messages.append(outdated_report.get_report())
+
         if dependents:
             humanized_dependents = formatting_utils.humanize_list(
                 dependents, 'and')

snapcraft/internal/lifecycle/_runner.py

@@ -184,34 +184,56 @@ def run(self, step: steps.Step, part_names=None):
                     # already been built --elopio - 20170713
                     pluginhandler.check_for_collisions(self.config.all_parts)
                 for part in parts:
-                    dirty_report = self._cache.get_dirty_report(
-                        part, current_step)
-                    if dirty_report:
-                        self._handle_dirty(
-                            part, current_step, dirty_report, cli_config)
-                    elif self._cache.has_step_run(part, current_step):
-                        # By default, if a step has already run, don't run it
-                        # again. However, automatically clean and re-run the
-                        # step if all the following conditions apply:
-                        #
-                        #   1. The step is the exact step that was requested
-                        #      (not an earlier one)
-                        #   2. The part was explicitly specified
-                        if (part_names and current_step == step and
-                                part.name in part_names):
-                            getattr(self, '_re{}'.format(
-                                current_step.name))(part)
-                        else:
-                            notify_part_progress(
-                                part,
-                                'Skipping {}'.format(current_step.name),
-                                '(already ran)')
-                    else:
-                        getattr(self, '_run_{}'.format(
-                            current_step.name))(part)
+                    self._handle_step(
+                        part_names, part, step, current_step, cli_config)
 
         self._create_meta(step, processed_part_names)
 
+    def _handle_step(self, requested_part_names: Sequence[str],
+                     part: pluginhandler.PluginHandler,
+                     requested_step: steps.Step,
+                     current_step: steps.Step, cli_config) -> None:
+        # If this step hasn't yet run, all we need to do is run it
+        if not self._cache.has_step_run(part, current_step):
+            getattr(self, '_run_{}'.format(current_step.name))(part)
+            return
+
+        # Alright, this step has already run. In that case, a few different
+        # things need to happen:
+
+        # 1. By default, if a step has already run, don't run it again.
+        #    However, automatically clean and re-run the step if all the
+        #    following conditions apply:
+        #
+        #      1.1. The step is the exact step that was requested (not an
+        #           earlier one).
+        #      1.2. The part was explicitly specified.
+        if (requested_part_names and current_step == requested_step and
+                part.name in requested_part_names):
+            getattr(self, '_re{}'.format(current_step.name))(part)
+            return
+
+        # 2. If a step has already run, it might be dirty, in which case we
+        #    need to clean and run it again.
+        dirty_report = self._cache.get_dirty_report(part, current_step)
+        if dirty_report:
+            self._handle_dirty(part, current_step, dirty_report, cli_config)
+            return
+
+        # 3. If a step has already run, it might be outdated, in which case we
+        #    need to update it (without cleaning if possible).
+        outdated_report = self._cache.get_outdated_report(
+            part, current_step)
+        if outdated_report:
+            self._handle_outdated(
+                part, current_step, outdated_report, cli_config)
+            return
+
+        # 4. The step has already run, and is up-to-date, no need to run it
+        #    again.
+        notify_part_progress(
+            part, 'Skipping {}'.format(current_step.name), '(already ran)')
+
     def _run_pull(self, part):
         self._run_step(step=steps.PULL, part=part, progress='Pulling')
 
@@ -244,7 +266,8 @@ def _reprime(self, part, hint=''):
         self._rerun_step(
             step=steps.PRIME, part=part, progress='Re-priming', hint=hint)
 
-    def _run_step(self, *, step: steps.Step, part, progress, hint=''):
+    def _prepare_step(self, *, step: steps.Step,
+                      part: pluginhandler.PluginHandler):
         common.reset_env()
         all_dependencies = self.parts_config.get_dependencies(part.name)
 
@@ -277,11 +300,18 @@ def _run_step(self, *, step: steps.Step, part, progress, hint=''):
 
         part = _replace_in_part(part)
 
+    def _run_step(self, *, step: steps.Step, part, progress, hint=''):
+        self._prepare_step(step=step, part=part)
+
         notify_part_progress(part, progress, hint)
         getattr(part, step.name)()
 
         # We know we just ran this step, so rather than check, manually twiddle
         # the cache
+        self._complete_step(part, step)
+
+    def _complete_step(self, part, step):
+        self._cache.clear_step(part, step)
         self._cache.add_step_run(part, step)
         self.steps_were_run = True
 
@@ -316,6 +346,28 @@ def _handle_dirty(self, part, step, dirty_report, cli_config):
         getattr(self, '_re{}'.format(step.name))(part, hint='({})'.format(
             dirty_report.get_summary()))
 
+    def _handle_outdated(self, part, step, outdated_report, cli_config):
+        dirty_action = cli_config.get_outdated_step_action()
+        if not step.clean_if_dirty:
+            if dirty_action == config.OutdatedStepAction.ERROR:
+                raise errors.StepOutdatedError(
+                    step=step, part=part.name, outdated_report=outdated_report)
+
+        update_function = getattr(part, 'update_{}'.format(step.name), None)
+        if update_function:
+            self._prepare_step(step=step, part=part)
+            notify_part_progress(
+                part, 'Updating {} step for'.format(step.name), '({})'.format(
+                    outdated_report.get_summary()))
+            update_function()
+
+            # We know we just ran this step, so rather than check, manually
+            # twiddle the cache
+            self._complete_step(part, step)
+        else:
+            getattr(self, '_re{}'.format(step.name))(part, '({})'.format(
+                outdated_report.get_summary()))
+
 
 def notify_part_progress(part, progress, hint='', debug=False):
     if debug:

snapcraft/internal/lifecycle/_status_cache.py

@@ -34,6 +34,7 @@ def __init__(self, config: _config.Config) -> None:
         """
         self.config = config
         self._steps_run = dict()  # type: Dict[str, Set[steps.Step]]
+        self._outdated_reports = collections.defaultdict(dict)  # type: _Report
         self._dirty_reports = collections.defaultdict(dict)  # type: _Report
 
     def should_step_run(self, part: pluginhandler.PluginHandler,
@@ -48,9 +49,12 @@ def should_step_run(self, part: pluginhandler.PluginHandler,
         A given step should run if it:
             1. Hasn't yet run
             2. Is dirty
-            3. Either (1) or (2) apply to any earlier steps in its lifecycle
+            3. Is outdated
+            4. Either (1), (2), or (3) apply to any earlier steps in the part's
+               lifecycle
         """
         if (not self.has_step_run(part, step) or
+                self.get_outdated_report(part, step) is not None or
                 self.get_dirty_report(part, step) is not None):
             return True
 
@@ -82,6 +86,17 @@ def has_step_run(self, part: pluginhandler.PluginHandler,
         self._ensure_steps_run(part)
         return step in self._steps_run[part.name]
 
+    def get_outdated_report(self, part, step):
+        """Obtain the outdated report for a given step of the given part.
+
+        :param pluginhandler.PluginHandler part: Part in question.
+        :param steps.Step step: Step in question.
+        :return: Outdated report (could be None)
+        :rtype: pluginhandler.OutdatedReport
+        """
+        self._ensure_outdated_report(part, step)
+        return self._outdated_reports[part.name][step]
+
     def get_dirty_report(self, part: pluginhandler.PluginHandler,
                          step: steps.Step) -> pluginhandler.DirtyReport:
         """Obtain the dirty report for a given step of the given part.
@@ -107,6 +122,9 @@ def clear_step(self, part: pluginhandler.PluginHandler,
             _remove_key(self._steps_run[part.name], step)
             if not self._steps_run[part.name]:
                 _del_key(self._steps_run, part.name)
+        _del_key(self._outdated_reports[part.name], step)
+        if not self._outdated_reports[part.name]:
+            _del_key(self._outdated_reports, part.name)
         _del_key(self._dirty_reports[part.name], step)
         if not self._dirty_reports[part.name]:
             _del_key(self._dirty_reports, part.name)
@@ -115,6 +133,12 @@ def _ensure_steps_run(self, part: pluginhandler.PluginHandler) -> None:
         if part.name not in self._steps_run:
             self._steps_run[part.name] = _get_steps_run(part)
 
+    def _ensure_outdated_report(self, part: pluginhandler.PluginHandler,
+                                step: steps.Step) -> None:
+        if step not in self._outdated_reports[part.name]:
+            self._outdated_reports[part.name][step] = part.get_outdated_report(
+                step)
+
     def _ensure_dirty_report(self, part: pluginhandler.PluginHandler,
                              step: steps.Step) -> None:
         # If we already have a dirty report, bail