parser: support remote dependencies (#930)

Ensure that we re-parse the parts that have missing dependencies after we've added all the valid ones to the master_parts_list

I would have fixed this in a cleaner way (allowing to include parts of a snapcraft.yaml that are good, and ignoring the bad ones only) but it would have caused a major refactor of the parser which probably is better to avoid.

LP: #1645350

snapcraft/internal/parser.py

@@ -216,7 +216,9 @@ def _process_entry(data):
     return parts_list, after_parts
 
 
-def _process_wiki_entry(entry, master_parts_list):
+def _process_wiki_entry(
+        entry, master_parts_list, master_missing_parts,
+        pending_validation_entries):
     """Add valid wiki entries to the master parts list"""
     # return the number of errors encountered
     try:
@@ -238,14 +240,42 @@ def _process_wiki_entry(entry, master_parts_list):
 
     parts_list, after_parts = _process_entry(data)
 
-    if is_valid_parts_list(parts_list, after_parts):
+    known_parts = list(parts_list.keys()) + list(master_parts_list.keys())
+    missing_parts = missing_parts_set(after_parts, known_parts)
+
+    if not len(missing_parts):
         master_parts_list.update(parts_list)
+        master_missing_parts -= set(parts_list.keys())
+    else:
+        pending_validation_entries.append(entry)
+        master_missing_parts.update(missing_parts)
+        logging.debug('Parts {!r} are missing'.format(
+            ",".join(missing_parts)))
+
+
+def _try_process_entry(
+        entry, master_parts_list, missing_parts,
+        pending_validation_entries):
+    wiki_errors = 0
+
+    try:
+        _process_wiki_entry(
+            entry, master_parts_list, missing_parts,
+            pending_validation_entries)
+    except SnapcraftError as e:
+        logger.warning(e)
+        wiki_errors += 1
+
+    return wiki_errors
 
 
 def _process_index(output):
     # XXX: This can't remain in memory if the list gets very large, but it
     # should be okay for now.
     master_parts_list = OrderedDict()
+    pending_validation_entries = []
+    missing_parts = set()
+
     wiki_errors = 0
 
     output = output.replace(b'{{{', b'').replace(b'}}}', b'')
@@ -257,22 +287,25 @@ def _process_index(output):
     for line in output.decode().splitlines():
         if line == '---':
             if entry:
-                try:
-                    _process_wiki_entry(entry, master_parts_list)
-                except SnapcraftError as e:
-                    logger.warning(e)
-                    wiki_errors += 1
-
+                wiki_errors += _try_process_entry(
+                    entry, master_parts_list, missing_parts,
+                    pending_validation_entries)
                 entry = ''
         else:
             entry = '\n'.join([entry, line])
 
     if entry:
-        try:
-            _process_wiki_entry(entry, master_parts_list)
-        except SnapcraftError as e:
-            logger.warning(e)
-            wiki_errors += 1
+        wiki_errors += _try_process_entry(
+            entry, master_parts_list,  missing_parts,
+            pending_validation_entries)
+
+    for entry in pending_validation_entries:
+        wiki_errors += _try_process_entry(
+            entry, master_parts_list, missing_parts, [])
+
+    if len(missing_parts):
+        logging.warning('Parts {!r} are not defined in the parts entry'.format(
+                ",".join(missing_parts)))
 
     return {'master_parts_list': master_parts_list,
             'wiki_errors': wiki_errors}
@@ -308,14 +341,13 @@ def run(args):
     return wiki_errors
 
 
-def is_valid_parts_list(parts_list, parts):
+def missing_parts_set(parts, known_parts):
+    missing_parts = set()
     for partname in parts:
-        if partname not in parts_list.keys():
-            logging.error('Part {!r} is missing from the parts entry'.format(
-                partname))
-            return False
+        if partname not in known_parts:
+            missing_parts.add(partname)
 
-    return True
+    return missing_parts
 
 
 def _write_parts_list(path, master_parts_list):

snapcraft/tests/test_parser.py

@@ -968,6 +968,80 @@ def test_parsed_output_matches_wiki_order(self, mock_get_origin_data):
         self.assertEqual(parts,
                          _get_part_list())
 
+    @mock.patch('snapcraft.internal.parser._get_origin_data')
+    def test_remote_after_parts(self, mock_get_origin_data):
+        _create_example_output("""
+---
+maintainer: John Doe <john.doe@example.com>
+origin: lp:snapcraft-parser-example
+description: example part on which parent depends on
+parts: [child]
+---
+maintainer: Marco Trevisan <marco@ubuntu.com>
+origin: lp:snapcraft-parser-example
+description: parent part that depends on child
+parts: [parent]
+""")
+        parts = OrderedDict()
+
+        child_part = OrderedDict()
+        child_part['description'] = 'parent part that depends on child'
+        child_part['maintainer'] = 'John Doe <john.doe@example.com>'
+        child_part['plugin'] = 'dump'
+        child_part['source'] = 'lp:project'
+        parts['child'] = child_part
+
+        parent_part = OrderedDict()
+        parent_part['description'] = 'example part on which parent depends on'
+        parent_part['maintainer'] = 'Marco Trevisan <marco@ubuntu.com>'
+        parent_part['plugin'] = 'dump'
+        parent_part['source'] = 'lp:project'
+        parent_part['after'] = ['child']
+        parts['parent'] = parent_part
+
+        mock_get_origin_data.return_value = {
+            'parts': parts,
+        }
+        main(['--index', TEST_OUTPUT_PATH])
+        self.assertEqual(2, _get_part_list_count())
+
+    @mock.patch('snapcraft.internal.parser._get_origin_data')
+    def test_remote_after_parts_unordered(self, mock_get_origin_data):
+        _create_example_output("""
+---
+maintainer: Marco Trevisan <marco@ubuntu.com>
+origin: lp:snapcraft-parser-example
+description: parent part that depends on child
+parts: [parent]
+---
+maintainer: John Doe <john.doe@example.com>
+origin: lp:snapcraft-parser-example
+description: example part on which parent depends on
+parts: [child]
+""")
+        parts = OrderedDict()
+
+        parent_part = OrderedDict()
+        parent_part['description'] = 'example part on which parent depends on'
+        parent_part['maintainer'] = 'Marco Trevisan <marco@ubuntu.com>'
+        parent_part['plugin'] = 'dump'
+        parent_part['source'] = 'lp:project'
+        parent_part['after'] = ['child']
+        parts['parent'] = parent_part
+
+        child_part = OrderedDict()
+        child_part['description'] = 'parent part that depends on child'
+        child_part['maintainer'] = 'John Doe <john.doe@example.com>'
+        child_part['plugin'] = 'dump'
+        child_part['source'] = 'lp:project'
+        parts['child'] = child_part
+
+        mock_get_origin_data.return_value = {
+            'parts': parts,
+        }
+        main(['--index', TEST_OUTPUT_PATH])
+        self.assertEqual(2, _get_part_list_count())
+
     def test__get_origin_data_both(self):
         with open(os.path.join(self.tempdir_path,
                   '.snapcraft.yaml'), 'w') as fp: