Add test verification to ansible-test.

test/runner/Makefile

@@ -0,0 +1,12 @@
+.PHONY: all compile sanity units
+
+all: compile sanity units
+
+compile:
+	./ansible-test compile test/runner/ ${TEST_FLAGS}
+
+sanity:
+	./ansible-test sanity test/runner/ ${TEST_FLAGS}
+
+units:
+	PYTHONPATH=. pytest units ${TEST_FLAGS}

test/runner/lib/changes.py

@@ -69,15 +69,18 @@ def __init__(self, args, git):
 
         if self.is_pr:
             self.paths = sorted(git.get_diff_names([self.branch]))
+            self.diff = git.get_diff([self.branch])
         else:
             merge_runs = self.get_merge_runs(self.project_id, self.branch)
             last_successful_commit = self.get_last_successful_commit(merge_runs)
 
             if last_successful_commit:
                 self.paths = sorted(git.get_diff_names([last_successful_commit, self.commit]))
+                self.diff = git.get_diff([last_successful_commit, self.commit])
             else:
                 # tracked files (including unchanged)
                 self.paths = sorted(git.get_file_names(['--cached']))
+                self.diff = None
 
     def get_merge_runs(self, project_id, branch):
         """
@@ -159,6 +162,8 @@ def __init__(self, args, git):
         self.staged = sorted(git.get_diff_names(['--cached']))
         # tracked changes (including deletions) which are not staged
         self.unstaged = sorted(git.get_diff_names([]))
+        # diff of all tracked files from fork point to working copy
+        self.diff = git.get_diff([self.fork_point])
 
     @staticmethod
     def is_official_branch(name):

test/runner/lib/delegation.py

@@ -4,6 +4,7 @@
 
 import os
 import sys
+import tempfile
 import time
 
 import lib.pytar
@@ -46,6 +47,27 @@ def delegate(args, exclude, require):
     :type args: EnvironmentConfig
     :type exclude: list[str]
     :type require: list[str]
+    :rtype: bool
+    """
+    if isinstance(args, TestConfig):
+        with tempfile.NamedTemporaryFile(prefix='metadata-', suffix='.json', dir=os.getcwd()) as metadata_fd:
+            args.metadata_path = os.path.basename(metadata_fd.name)
+            args.metadata.to_file(args.metadata_path)
+
+            try:
+                return delegate_command(args, exclude, require)
+            finally:
+                args.metadata_path = None
+    else:
+        return delegate_command(args, exclude, require)
+
+
+def delegate_command(args, exclude, require):
+    """
+    :type args: EnvironmentConfig
+    :type exclude: list[str]
+    :type require: list[str]
+    :rtype: bool
     """
     if args.tox:
         delegate_tox(args, exclude, require)
@@ -396,6 +418,7 @@ def filter_options(args, argv, options, exclude, require):
             '--ignore-unstaged': 0,
             '--changed-from': 1,
             '--changed-path': 1,
+            '--metadata': 1,
         })
     elif isinstance(args, SanityConfig):
         options.update({
@@ -427,3 +450,7 @@ def filter_options(args, argv, options, exclude, require):
     for target in require:
         yield '--require'
         yield target
+
+    if args.metadata_path:
+        yield '--metadata'
+        yield args.metadata_path

test/runner/lib/diff.py

@@ -0,0 +1,253 @@
+"""Diff parsing functions and classes."""
+from __future__ import absolute_import, print_function
+
+import re
+import textwrap
+import traceback
+
+from lib.util import (
+    ApplicationError,
+)
+
+
+def parse_diff(lines):
+    """
+    :type lines: list[str]
+    :rtype: list[FileDiff]
+    """
+    return DiffParser(lines).files
+
+
+class FileDiff(object):
+    """Parsed diff for a single file."""
+    def __init__(self, old_path, new_path):
+        """
+        :type old_path: str
+        :type new_path: str
+        """
+        self.old = DiffSide(old_path, new=False)
+        self.new = DiffSide(new_path, new=True)
+        self.headers = []  # list [str]
+        self.binary = False
+
+    def append_header(self, line):
+        """
+        :type line: str
+        """
+        self.headers.append(line)
+
+    @property
+    def is_complete(self):
+        """
+        :rtype: bool
+        """
+        return self.old.is_complete and self.new.is_complete
+
+
+class DiffSide(object):
+    """Parsed diff for a single 'side' of a single file."""
+    def __init__(self, path, new):
+        """
+        :type path: str
+        :type new: bool
+        """
+        self.path = path
+        self.new = new
+        self.prefix = '+' if self.new else '-'
+        self.eof_newline = True
+        self.exists = True
+
+        self.lines = []  # type: list [tuple[int, str]]
+        self.lines_and_context = []  # type: list [tuple[int, str]]
+        self.ranges = []  # type: list [tuple[int, int]]
+
+        self._next_line_number = 0
+        self._lines_remaining = 0
+        self._range_start = 0
+
+    def set_start(self, line_start, line_count):
+        """
+        :type line_start: int
+        :type line_count: int
+        """
+        self._next_line_number = line_start
+        self._lines_remaining = line_count
+        self._range_start = 0
+
+    def append(self, line):
+        """
+        :type line: str
+        """
+        if self._lines_remaining <= 0:
+            raise Exception('Diff range overflow.')
+
+        entry = self._next_line_number, line
+
+        if line.startswith(' '):
+            pass
+        elif line.startswith(self.prefix):
+            self.lines.append(entry)
+
+            if not self._range_start:
+                self._range_start = self._next_line_number
+        else:
+            raise Exception('Unexpected diff content prefix.')
+
+        self.lines_and_context.append(entry)
+
+        self._lines_remaining -= 1
+
+        if self._range_start:
+            if self.is_complete:
+                range_end = self._next_line_number
+            elif line.startswith(' '):
+                range_end = self._next_line_number - 1
+            else:
+                range_end = 0
+
+            if range_end:
+                self.ranges.append((self._range_start, range_end))
+                self._range_start = 0
+
+        self._next_line_number += 1
+
+    @property
+    def is_complete(self):
+        """
+        :rtype: bool
+        """
+        return self._lines_remaining == 0
+
+    def format_lines(self, context=True):
+        """
+        :type context: bool
+        :rtype: list[str]
+        """
+        if context:
+            lines = self.lines_and_context
+        else:
+            lines = self.lines
+
+        return ['%s:%4d %s' % (self.path, line[0], line[1]) for line in lines]
+
+
+class DiffParser(object):
+    """Parse diff lines."""
+    def __init__(self, lines):
+        """
+        :type lines: list[str]
+        """
+        self.lines = lines
+        self.files = []  # type: list [FileDiff]
+
+        self.action = self.process_start
+        self.line_number = 0
+        self.previous_line = None  # type: str
+        self.line = None  # type: str
+        self.file = None  # type: FileDiff
+
+        for self.line in self.lines:
+            self.line_number += 1
+
+            try:
+                self.action()
+            except Exception as ex:
+                message = textwrap.dedent('''
+                %s
+
+                     Line: %d
+                 Previous: %s
+                  Current: %s
+                %s
+                ''').strip() % (
+                    ex,
+                    self.line_number,
+                    self.previous_line or '',
+                    self.line or '',
+                    traceback.format_exc(),
+                )
+
+                raise ApplicationError(message.strip())
+
+            self.previous_line = self.line
+
+        self.complete_file()
+
+    def process_start(self):
+        """Process a diff start line."""
+        self.complete_file()
+
+        match = re.search(r'^diff --git a/(?P<old_path>.*) b/(?P<new_path>.*)$', self.line)
+
+        if not match:
+            raise Exception('Unexpected diff start line.')
+
+        self.file = FileDiff(match.group('old_path'), match.group('new_path'))
+        self.action = self.process_continue
+
+    def process_range(self):
+        """Process a diff range line."""
+        match = re.search(r'^@@ -((?P<old_start>[0-9]+),)?(?P<old_count>[0-9]+) \+((?P<new_start>[0-9]+),)?(?P<new_count>[0-9]+) @@', self.line)
+
+        if not match:
+            raise Exception('Unexpected diff range line.')
+
+        self.file.old.set_start(int(match.group('old_start') or 1), int(match.group('old_count')))
+        self.file.new.set_start(int(match.group('new_start') or 1), int(match.group('new_count')))
+        self.action = self.process_content
+
+    def process_continue(self):
+        """Process a diff start, range or header line."""
+        if self.line.startswith('diff '):
+            self.process_start()
+        elif self.line.startswith('@@ '):
+            self.process_range()
+        else:
+            self.process_header()
+
+    def process_header(self):
+        """Process a diff header line."""
+        if self.line.startswith('Binary files '):
+            self.file.binary = True
+        elif self.line == '--- /dev/null':
+            self.file.old.exists = False
+        elif self.line == '+++ /dev/null':
+            self.file.new.exists = False
+        else:
+            self.file.append_header(self.line)
+
+    def process_content(self):
+        """Process a diff content line."""
+        if self.line == r'\ No newline at end of file':
+            if self.previous_line.startswith(' '):
+                self.file.old.eof_newline = False
+                self.file.new.eof_newline = False
+            elif self.previous_line.startswith('-'):
+                self.file.old.eof_newline = False
+            elif self.previous_line.startswith('+'):
+                self.file.new.eof_newline = False
+            else:
+                raise Exception('Unexpected previous diff content line.')
+
+            return
+
+        if self.file.is_complete:
+            self.process_continue()
+            return
+
+        if self.line.startswith(' '):
+            self.file.old.append(self.line)
+            self.file.new.append(self.line)
+        elif self.line.startswith('-'):
+            self.file.old.append(self.line)
+        elif self.line.startswith('+'):
+            self.file.new.append(self.line)
+        else:
+            raise Exception('Unexpected diff content line.')
+
+    def complete_file(self):
+        """Complete processing of the current file, if any."""
+        if not self.file:
+            return
+
+        self.files.append(self.file)