Integrate with new remote parts

demos/mosquitto/snapcraft.yaml

@@ -28,4 +28,4 @@ parts:
     files:
       subscribe.py: bin/subscribe
       publish.py: bin/publish
-    after: [mqtt-paho-py3]
+    after: [mqtt-paho/python3]

integration_tests/test_wiki.py

@@ -21,4 +21,5 @@ class WikiTestCase(integration_tests.TestCase):
 
     def test_pull_wiki_part(self):
         project_dir = 'wiki'
+        self.run_snapcraft('update', project_dir)
         self.run_snapcraft('pull', project_dir)

snapcraft/internal/parts.py

@@ -98,5 +98,41 @@ def _save_headers(self):
             headers_file.write(yaml.dump(headers))
 
 
+class _RemoteParts(_Base):
+
+    def __init__(self):
+        super().__init__()
+
+        if os.path.exists(self.parts_yaml):
+            with open(self.parts_yaml) as parts_file:
+                self._parts = yaml.load(parts_file)
+        else:
+            self._parts = {}
+
+    def get_part(self, part_name):
+        remote_part = self._parts[part_name].copy()
+        for key in ['description', 'maintainer']:
+            remote_part.pop(key)
+        return remote_part
+
+    def compose(self, part_name, properties):
+        """Return properties composed with the ones from part name in the wiki.
+        :param str part_name: The name of the part to query from the wiki
+        :param dict properties: The current set of properties
+        :return: Part properties from the wiki composed with the properties
+                 passed as a parameter.
+        :rtype: dict
+        :raises KeyError: if part_name is not found in the wiki.
+        """
+        remote_part = self.get_part(part_name)
+        remote_part.update(properties)
+
+        return remote_part
+
+
 def update():
     _Update().execute()
+
+
+def get_remote_parts():
+    return _RemoteParts()

snapcraft/internal/wiki.py

@@ -1,71 +0,0 @@
-# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
-#
-# Copyright (C) 2015 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import logging
-import requests
-import yaml
-
-PARTS_URI = 'https://wiki.ubuntu.com/Snappy/Parts'
-PARTS_URI_PARAMS = {'action': 'raw'}
-
-_WIKI_OPEN = '{{{'
-_WIKI_CLOSE = '}}}'
-
-logging.getLogger("urllib3").setLevel(logging.CRITICAL)
-
-
-class Wiki:
-    wiki_parts = None
-
-    def _fetch(self):
-        if self.wiki_parts is None:
-            raw_content = requests.get(PARTS_URI, params=PARTS_URI_PARAMS)
-            content = raw_content.text.strip()
-
-            if content.startswith(_WIKI_OPEN):
-                content = content[len(_WIKI_OPEN):].strip()
-            if content.endswith(_WIKI_CLOSE):
-                content = content[:-len(_WIKI_CLOSE)]
-
-            self.wiki_parts = yaml.load(content)
-
-    def get_part(self, name):
-        self._fetch()
-
-        if name in self.wiki_parts:
-            if 'plugin' and 'type' in self.wiki_parts[name]:
-                del self.wiki_parts[name]['type']
-            return self.wiki_parts[name]
-
-    def compose(self, name, properties):
-        """Return properties composed with the ones from part name in the wiki.
-
-        :param str name: The name of the part to query from the wiki
-        :param dict properties: The current set of properties
-        :return: Part properties from the wiki composed with the properties
-                 passed as a parameter. If there is no wiki part named name,
-                 properties will be returned.
-        :rtype: dict
-        :raises KeyError: if the part named name is not found in the wiki.
-        """
-        self._fetch()
-
-        wiki_properties = self.wiki_parts[name]
-        for key in wiki_properties:
-            properties[key] = properties.get(key, wiki_properties[key])
-        properties['plugin'] = wiki_properties.get('plugin', None)
-
-        return properties

snapcraft/internal/yaml.py

@@ -15,7 +15,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import codecs
-import contextlib
 import logging
 import os
 import os.path
@@ -29,9 +28,9 @@
 from snapcraft.internal import (
     common,
     libraries,
+    parts,
     pluginhandler,
     sources,
-    wiki,
 )
 from snapcraft._schema import Validator, SnapcraftSchemaError
 
@@ -134,7 +133,7 @@ def __init__(self, project_options=None):
         self.build_tools = self.data.get('build-packages', [])
         self.build_tools.extend(project_options.additional_build_packages)
 
-        self._wiki = wiki.Wiki()
+        self._remote_parts = parts.get_remote_parts()
         self._process_parts()
 
     def _process_parts(self):
@@ -145,17 +144,18 @@ def _process_parts(self):
             plugin_name = properties.pop('plugin', None)
             if not plugin_name:
                 logger.info(
-                    'Searching the wiki to compose part "{}"'.format(
-                        part_name))
-                with contextlib.suppress(KeyError):
-                    properties = self._wiki.compose(part_name, properties)
+                    'Searching in the remote parts cache for part '
+                    '{!r}'.format(part_name))
+                try:
+                    properties = self._remote_parts.compose(
+                        part_name, properties)
                     plugin_name = properties.pop('plugin', None)
-                    # The wiki still supports using 'type' for snapcraft 1.x
-                    if 'type' in properties:
-                        del properties['type']
-
-            if not plugin_name:
-                raise PluginNotDefinedError(part_name)
+                except KeyError as e:
+                    raise SnapcraftLogicError(
+                        '{!r} is missing the `plugin` entry and is not '
+                        'defined in the current remote parts cache, try to '
+                        'run `snapcraft update` to '
+                        'refresh'.format(part_name)) from e
 
             if 'after' in properties:
                 self.after_requests[part_name] = properties.pop('after')
@@ -187,16 +187,18 @@ def _compute_part_dependencies(self):
                         found = True
                         break
                 if not found:
-                    wiki_part = self._wiki.get_part(dep)
-                    found = True if wiki_part else False
-                    if found:
-                        plugin_name = wiki_part.pop('plugin')
-                        part.deps.append(self.load_plugin(
-                            dep, plugin_name, wiki_part))
-                        self._part_names.append(dep)
-                if not found:
-                    raise SnapcraftLogicError(
-                        'part name missing {}'.format(dep))
+                    try:
+                        remote_part = self._remote_parts.get_part(dep)
+                    except KeyError as e:
+                        raise SnapcraftLogicError(
+                            'Cannot find definition for part {!r}. '
+                            'It may be a remote part, run `snapcraft update` '
+                            'to refresh the remote parts '
+                            'cache'.format(dep)) from e
+                    plugin_name = remote_part.pop('plugin')
+                    part.deps.append(self.load_plugin(
+                        dep, plugin_name, remote_part))
+                    self._part_names.append(dep)
 
     def _sort_parts(self):
         '''Performs an inneficient but easy to follow sorting of parts.'''

snapcraft/tests/fake_servers.py

@@ -55,10 +55,19 @@ def do_GET(self):
                 'curl': {
                     'source': 'http://curl.org',
                     'plugin': 'autotools',
+                    'description': 'test entry for curl',
+                    'maintainer': 'none',
+                },
+                'part1': {
+                    'plugin': 'go',
+                    'source': 'http://source.tar.gz',
+                    'description': 'test entry for part1',
+                    'maintainer': 'none',
                 },
             }
+        self.send_header('Content-Type', 'text/plain')
         if 'NO_CONTENT_LENGTH' not in os.environ:
-            self.send_header('Content-Length', '100')
+            self.send_header('Content-Length', '300')
         self.send_header('ETag', '1111')
         self.end_headers()
         self.wfile.write(yaml.dump(response).encode())

snapcraft/tests/test_commands_update.py

@@ -51,8 +51,16 @@ def test_update(self):
         expected_parts = {
             'curl': {
                 'source': 'http://curl.org',
-                'plugin': 'autotools'
-            }
+                'plugin': 'autotools',
+                'description': 'test entry for curl',
+                'maintainer': 'none',
+            },
+            'part1': {
+                'plugin': 'go',
+                'source': 'http://source.tar.gz',
+                'description': 'test entry for part1',
+                'maintainer': 'none',
+            },
         }
         expected_headers = {
             'If-None-Match': '1111',

snapcraft/tests/test_wiki.py

@@ -1,76 +0,0 @@
-# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
-#
-# Copyright (C) 2015 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import unittest.mock
-
-from snapcraft.internal import wiki
-
-from snapcraft.tests import TestCase
-
-
-class TestYaml(TestCase):
-
-    def setUp(self):
-        super().setUp()
-
-        class Content:
-
-            @property
-            def text(self):
-                return '''{{{part-in-wiki:
-  plugin: go
-  source: http://somesource
-}}}'''
-
-        patcher = unittest.mock.patch('requests.get')
-        self.mock_requests = patcher.start()
-        self.mock_requests.return_value = Content()
-        self.addCleanup(patcher.stop)
-
-        self.w = wiki.Wiki()
-
-    def tearDown(self):
-        self.mock_requests.assert_called_once_with(
-            'https://wiki.ubuntu.com/Snappy/Parts',
-            params={'action': 'raw'})
-
-    def test_get_part(self):
-        self.assertEqual(self.w.get_part('part-in-wiki'), {
-            'plugin': 'go', 'source': 'http://somesource'})
-        self.assertEqual(self.w.get_part('part-not-in-wiki'), None)
-
-    def test_compose_part_with_properties_from_the_wiki(self):
-        properties = self.w.compose(
-            'part-in-wiki', {'source': '.', 'another': 'different'})
-        expected_properties = {
-            'plugin': 'go', 'source': '.', 'another': 'different'}
-
-        self.assertEqual(properties, expected_properties)
-
-    def test_compose_part_with_properties_from_the_wiki_using_source(self):
-        properties = self.w.compose(
-            'part-in-wiki', {'another': 'different'})
-        expected_properties = {
-            'plugin': 'go', 'source': 'http://somesource',
-            'another': 'different'}
-
-        self.assertEqual(properties, expected_properties)
-
-    def test_compose_part_for_part_not_in_wiki_raises_exception(self):
-        with self.assertRaises(KeyError) as raised:
-            self.w.compose('part-not-in-wiki',
-                           {'source': '.', 'another': 'different'})
-        self.assertEqual(raised.exception.args, ('part-not-in-wiki',))