Implement `snapcraft update` for parts

integration_tests/__init__.py

@@ -46,6 +46,8 @@ def setUp(self):
 
         self.useFixture(fixtures.EnvironmentVariable(
             'XDG_CONFIG_HOME', os.path.join(self.path, '.config')))
+        self.useFixture(fixtures.EnvironmentVariable(
+            'XDG_DATA_HOME', os.path.join(self.path, 'data')))
 
     def run_snapcraft(self, command, project_dir=None):
         if isinstance(command, str):

integration_tests/test_parts.py

@@ -0,0 +1,46 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2016 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+import yaml
+
+import integration_tests
+
+
+class PartsTestCase(integration_tests.TestCase):
+
+    def setUp(self):
+        super().setUp()
+
+        self.parts_dir = os.path.join('data', 'snapcraft')
+        self.parts_yaml = os.path.join(self.parts_dir, 'parts.yaml')
+        self.headers_yaml = os.path.join(self.parts_dir, 'headers.yaml')
+
+    def test_update(self):
+        self.run_snapcraft('update')
+
+        self.assertTrue(os.path.exists(self.parts_yaml))
+        self.assertTrue(os.path.exists(self.headers_yaml))
+
+    def test_curl_exists(self):
+        """Curl is used in most of the demos so we test for its existence."""
+        self.run_snapcraft('update')
+
+        with open(self.parts_yaml) as parts_file:
+            parts = yaml.load(parts_file)
+
+        self.assertTrue('curl' in parts, parts)

snapcraft/internal/parts.py

@@ -0,0 +1,91 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2016 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+
+import requests
+import yaml
+from progressbar import (ProgressBar, Percentage, Bar)
+from xdg import BaseDirectory
+
+PARTS_URI = 'https://parts.snapcraft.io/v1/parts.yaml'
+
+logging.getLogger("urllib3").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+
+class _Base:
+
+    def __init__(self):
+        self.parts_dir = os.path.join(BaseDirectory.xdg_data_home, 'snapcraft')
+        os.makedirs(self.parts_dir, exist_ok=True)
+        self.parts_yaml = os.path.join(self.parts_dir, 'parts.yaml')
+
+
+class _Update(_Base):
+
+    def __init__(self):
+        super().__init__()
+        self._headers_yaml = os.path.join(self.parts_dir, 'headers.yaml')
+        self._parts_uri = os.environ.get('SNAPCRAFT_PARTS_URI', PARTS_URI)
+
+    def execute(self):
+        headers = self._load_headers()
+        self._request = requests.get(self._parts_uri, stream=True,
+                                     headers=headers)
+
+        if self._request.status_code == 304:
+            logger.info('The parts cache is already up to date.')
+            return
+        self._request.raise_for_status()
+
+        self._download()
+        self._save_headers()
+
+    def _download(self):
+        total_length = int(self._request.headers.get('Content-Length'))
+        progress_bar = ProgressBar(
+            widgets=['Updating parts list',
+                     Bar(marker='=', left='[', right=']'),
+                     ' ', Percentage()],
+            maxval=total_length)
+
+        total_read = 0
+        progress_bar.start()
+        with open(self.parts_yaml, 'wb') as parts_file:
+            for buf in self._request.iter_content(1):
+                parts_file.write(buf)
+                total_read += len(buf)
+                progress_bar.update(total_read)
+        progress_bar.finish()
+
+    def _load_headers(self):
+        if not os.path.exists(self._headers_yaml):
+            return None
+
+        with open(self._headers_yaml) as headers_file:
+            return yaml.load(headers_file)
+
+    def _save_headers(self):
+        headers = {'If-None-Match': self._request.headers.get('ETag')}
+
+        with open(self._headers_yaml, 'w') as headers_file:
+            headers_file.write(yaml.dump(headers))
+
+
+def update():
+    _Update().execute()

snapcraft/main.py

@@ -35,6 +35,7 @@
   snapcraft [options] upload <snap-file>
   snapcraft [options] list-plugins
   snapcraft [options] tour [<directory>]
+  snapcraft [options] update
   snapcraft [options] help (topics | <plugin> | <topic>) [--devel]
   snapcraft (-h | --help)
   snapcraft --version
@@ -88,6 +89,9 @@
   prime        Final copy and preparation for the snap.
   snap         Create a snap.
 
+Parts ecosystem commands
+  update       Updates the parts listing from the cloud.
+
 Calling snapcraft without a COMMAND will default to 'snap'
 
 The cleanbuild command requires a properly setup lxd environment that
@@ -113,7 +117,7 @@
 from docopt import docopt
 
 import snapcraft
-from snapcraft.internal import lifecycle, log
+from snapcraft.internal import lifecycle, log, parts
 from snapcraft.internal.common import (
     format_output_in_columns, MAX_CHARACTERS_WRAP, get_tourdir)
 
@@ -250,6 +254,8 @@ def run(args, project_options):
     elif args['help']:
         snapcraft.topic_help(args['<topic>'] or args['<plugin>'],
                              args['--devel'], args['topics'])
+    elif args['update']:
+        parts.update()
     else:  # snap by default:
         lifecycle.snap(project_options, args['<directory>'], args['--output'])
 

snapcraft/tests/__init__.py

@@ -19,6 +19,7 @@
 from unittest import mock
 
 import fixtures
+import progressbar
 import testscenarios
 
 from snapcraft.internal import common
@@ -43,7 +44,7 @@ def setUp(self):
         temp_cwd_fixture = fixture_setup.TempCWD()
         self.useFixture(temp_cwd_fixture)
         self.path = temp_cwd_fixture.path
-        self.useFixture(fixture_setup.TempConfig(self.path))
+        self.useFixture(fixture_setup.TempXDG(self.path))
         # Some tests will directly or indirectly change the plugindir, which
         # is a module variable. Make sure that it is returned to the original
         # value when a test ends.
@@ -81,3 +82,16 @@ def verify_state(self, part_name, state_dir, expected_step):
             self.assertTrue(os.path.exists(os.path.join(state_dir, step)),
                             'Expected {!r} to be run for {}'.format(
                                 step, part_name))
+
+
+class SilentProgressBar(progressbar.ProgressBar):
+    """A progress bar causing no spurious output during tests."""
+
+    def start(self):
+        pass
+
+    def update(self, value=None):
+        pass
+
+    def finish(self):
+        pass

snapcraft/tests/fake_servers.py

@@ -20,6 +20,8 @@
 import os
 import urllib.parse
 
+import yaml
+
 import snapcraft.tests
 from snapcraft.storeapi import macaroons
 
@@ -33,6 +35,35 @@ def log_message(*args):
         logger.debug(args)
 
 
+class FakePartsServer(http.server.HTTPServer):
+
+    def __init__(self, server_address):
+        super().__init__(
+            server_address, FakePartsRequestHandler)
+
+
+class FakePartsRequestHandler(BaseHTTPRequestHandler):
+
+    def do_GET(self):
+        logger.debug('Handling getting parts')
+        if self.headers.get('If-None-Match') == '1111':
+            self.send_response(304)
+            response = {}
+        else:
+            self.send_response(200)
+            response = {
+                'curl': {
+                    'source': 'http://curl.org',
+                    'plugin': 'autotools',
+                },
+            }
+        self.send_header('Content-Type', 'text/plain')
+        self.send_header('Content-Length', '100')
+        self.send_header('ETag', '1111')
+        self.end_headers()
+        self.wfile.write(yaml.dump(response).encode())
+
+
 class FakeSSOServer(http.server.HTTPServer):
 
     def __init__(self, server_address):

snapcraft/tests/fixture_setup.py

@@ -35,7 +35,7 @@ def setUp(self):
         os.chdir(self.path)
 
 
-class TempConfig(fixtures.Fixture):
+class TempXDG(fixtures.Fixture):
     """Isolate a test from xdg so a private temp config is used."""
 
     def __init__(self, path):
@@ -44,17 +44,29 @@ def __init__(self, path):
 
     def setUp(self):
         super().setUp()
-        patcher_home = mock.patch(
+        patcher = mock.patch(
             'xdg.BaseDirectory.xdg_config_home',
             new=os.path.join(self.path, '.config'))
-        patcher_home.start()
-        self.addCleanup(patcher_home.stop)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        patcher = mock.patch(
+            'xdg.BaseDirectory.xdg_data_home',
+            new=os.path.join(self.path, '.local'))
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
         patcher_dirs = mock.patch(
             'xdg.BaseDirectory.xdg_config_dirs',
             new=[xdg.BaseDirectory.xdg_config_home])
         patcher_dirs.start()
         self.addCleanup(patcher_dirs.stop)
 
+        patcher_dirs = mock.patch(
+            'xdg.BaseDirectory.xdg_data_dirs',
+            new=[xdg.BaseDirectory.xdg_data_home])
+        patcher_dirs.start()
+        self.addCleanup(patcher_dirs.stop)
+
 
 class CleanEnvironment(fixtures.Fixture):
 
@@ -67,6 +79,21 @@ def setUp(self):
         self.addCleanup(os.environ.update, current_environment)
 
 
+class FakeParts(fixtures.Fixture):
+
+    def setUp(self):
+        super().setUp()
+
+        self.fake_parts_server_fixture = FakePartsServerRunning()
+        self.useFixture(self.fake_parts_server_fixture)
+        self.useFixture(fixtures.EnvironmentVariable(
+            'SNAPCRAFT_PARTS_URI',
+            urllib.parse.urljoin(
+                self.fake_parts_server_fixture.url, 'parts.yaml')))
+        self.useFixture(fixtures.EnvironmentVariable(
+            'no_proxy', 'localhost,127.0.0.1'))
+
+
 class FakeStore(fixtures.Fixture):
 
     def setUp(self):
@@ -125,6 +152,11 @@ def _stop_fake_server(self, thread):
         thread.join()
 
 
+class FakePartsServerRunning(_FakeServerRunning):
+
+    fake_server = fake_servers.FakePartsServer
+
+
 class FakeSSOServerRunning(_FakeServerRunning):
 
     fake_server = fake_servers.FakeSSOServer

snapcraft/tests/store/test_store_client.py

@@ -20,7 +20,6 @@
 from unittest import mock
 
 import fixtures
-import progressbar
 
 from snapcraft import (
     config,
@@ -179,19 +178,6 @@ def test_registration_failed(self):
         self.assertFalse(response.ok)
 
 
-class SilentProgressBar(progressbar.ProgressBar):
-    """A progress bar causing no spurious output during tests."""
-
-    def start(self):
-        pass
-
-    def update(self, value=None):
-        pass
-
-    def finish(self):
-        pass
-
-
 class UploadTestCase(tests.TestCase):
 
     def setUp(self):
@@ -203,7 +189,7 @@ def setUp(self):
             'test-snap.snap')
         patcher = mock.patch(
             'snapcraft.storeapi._upload.ProgressBar',
-            new=SilentProgressBar)
+            new=tests.SilentProgressBar)
         patcher.start()
         self.addCleanup(patcher.stop)