Construct CartAPI URL from OS environment variables (#15)

* Construct CartAPI URL from OS environment variables

* Fix environment variables prefix for CartAPI class

* Add logger debug statements to CartAPI instance methods

* Remove unused import

* Revert changes to Downloader class

Commit 3449fb7 did not resolve merge conflicts.

* Resolve pylint "protected-access" warning

* Resolve pylint D200 and D401 warnings

* Resolve pylint pep257 warning

* Resolve flake8 F401 warning

* resolve autopep8 warnings

* CartAPI response is not JSON data

* Add coverage test for CartAPI URL

* Resolve pylint "protected-access" warning

docs/exampleusage.md

@@ -14,7 +14,7 @@ download the data to.
 from tempfile import mkdtemp
 
 down_path = mkdtemp()
-down = Downloader('http://127.0.0.1:8081')
+down = Downloader(cart_api_url='http://127.0.0.1:8081')
 resp = requests.get('http://127.0.0.1:8181/status/transactions/by_id/67')
 assert resp.status_code == 200
 down.transactioninfo(down_path, resp.json())
@@ -39,7 +39,7 @@ class Root(object):
     def POST(self):
         """Accept the cloud event data and return the local download path."""
         down_path = mkdtemp()
-        down = Downloader('http://127.0.0.1:8081')
+        down = Downloader(cart_api_url='http://127.0.0.1:8081')
         down.cloudevent(down_path, cherrpy.request.json)
         return { 'download_path': down_path }
 

pacifica/downloader/__init__.py

@@ -8,6 +8,6 @@
 Cartd service.
 """
 from __future__ import absolute_import
-from .downloader import Downloader
+from .downloader import Downloader  # noqa: F401
 
-__all__ = ['Downloader']
+__all__ = ('Downloader')

pacifica/downloader/cartapi.py

@@ -5,18 +5,27 @@
 from uuid import uuid4 as uuid
 import time
 from json import dumps
-import requests
+from .common import CommonBase
 
 
-class CartAPI(object):
+LOGGER = logging.getLogger(__name__)
+
+
+class CartAPI(CommonBase):
     """
     Cart api object for manipulating carts.
 
     This class has two methods used for setting up a cart and waiting
     for completion.
     """
 
-    def __init__(self, cart_api_url, **kwargs):
+    _proto = None
+    _addr = None
+    _port = None
+    _cart_api_url = None
+    _auth = None
+
+    def __init__(self, **kwargs):
         """
         Constructor for cart api.
 
@@ -26,13 +35,38 @@ def __init__(self, cart_api_url, **kwargs):
         keyword arguments. Also, an optional requests authentication
         dictionary can be passed via keyword arguments.
         """
-        self.cart_api_url = cart_api_url
-        adapter = requests.adapters.HTTPAdapter(max_retries=5)
-        session = requests.Session()
-        session.mount('http://', adapter)
-        session.mount('https://', adapter)
-        self.session = kwargs.get('session', session)
-        self.auth = kwargs.get('auth', {})
+        super(CartAPI, self).__init__()
+
+        self._setup_requests_session()
+        self.session = kwargs.get('session', self.session)
+
+        self._server_url(
+            [
+                ('proto', 'http'),
+                ('port', '8081'),
+                ('addr', '127.0.0.1'),
+                ('cart_api_url', None),
+            ],
+            'CARTD',
+            kwargs
+        )
+
+        if self._cart_api_url is None:
+            self._cart_api_url = '{}://{}:{}'.format(self._proto, self._addr, self._port)
+
+        self._auth = kwargs.get('auth', {})
+
+        LOGGER.debug('CartAPI URL %s auth %s', self._cart_api_url, self._auth)
+
+    @property
+    def auth(self):
+        """Return the requests authentication dictionary."""
+        return self._auth
+
+    @property
+    def cart_api_url(self):
+        """Return the CartAPI URL."""
+        return self._cart_api_url
 
     def setup_cart(self, yield_files):
         """
@@ -43,15 +77,16 @@ def setup_cart(self, yield_files):
         the `Cartd API <https://github.com/pacifica/pacifica-cartd>`_.
         This method returns the full url to the cart created.
         """
-        cart_url = '{}/{}'.format(self.cart_api_url, uuid())
+        cart_url = '{}/{}'.format(self._cart_api_url, uuid())
         resp = self.session.post(
             cart_url,
             data=dumps({
                 'fileids': [file_obj for file_obj in yield_files()]
             }),
             headers={'Content-Type': 'application/json'},
-            **self.auth
+            **self._auth
         )
+        LOGGER.debug('CartAPI Setup code %s', resp.status_code)
         assert resp.status_code == 201
         return cart_url
 
@@ -64,10 +99,12 @@ def wait_for_cart(self, cart_url, timeout=120):
         ready to download.
         """
         while timeout > 0:
-            resp = self.session.head(cart_url, **self.auth)
+            resp = self.session.head(cart_url, **self._auth)
             resp_status = resp.headers['X-Pacifica-Status']
             resp_message = resp.headers['X-Pacifica-Message']
             resp_code = resp.status_code
+            LOGGER.debug('CartAPI Wait code %s status %s message %s',
+                         resp_code, resp_status, resp_message.replace('"', '\\"'))
             if resp_code == 204 and resp_status != 'staging':
                 break
             if resp_code == 500:  # pragma: no cover

pacifica/downloader/common/__init__.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""Common downloader functionality."""
+from os import getenv
+import requests
+
+
+# pylint: disable=too-few-public-methods
+class CommonBase(object):
+    """Contains methods to implement common functionality."""
+
+    session = None
+
+    def _setup_requests_session(self):
+        """Setup a requests retry session so we can talk to http services."""
+        self.session = requests.session()
+        retry_adapter = requests.adapters.HTTPAdapter(max_retries=5)
+        self.session.mount('https://', retry_adapter)
+        self.session.mount('http://', retry_adapter)
+
+    def _server_url(self, parts, env_prefix, kwargs):
+        """Server URL parsing for init class method."""
+        for part, default in parts:
+            attr_name = '_{}'.format(part)
+            setattr(self, attr_name, kwargs.get(part))
+            if not getattr(self, attr_name):
+                setattr(self, attr_name, getenv('{}{}'.format(
+                    env_prefix, attr_name.upper()), default))
+# pylint: enable=too-few-public-methods

pacifica/downloader/downloader.py

@@ -12,19 +12,19 @@ class Downloader(object):
     """
     Downloader Class.
 
-    The constructor takes one argument: `cart_api_url`.
-    The `cart_api_url` is the endpoint for creating carts.
-
     The other methods in this class are the supported
     download methods. Each method takes appropriate input for that
     method and the method will download the data to the location
     specified in the method's arguments.
     """
 
-    def __init__(self, cart_api_url, **kwargs):
-        """Create the downloader given directory location."""
-        self.auth = kwargs.get('auth', {})
-        self.cart_api = CartAPI(cart_api_url, auth=self.auth)
+    def __init__(self, **kwargs):
+        """
+        Create the downloader.
+
+        Keyword arguments are delegated to the CartAPI.
+        """
+        self.cart_api = CartAPI(**kwargs)
 
     def _download_from_url(self, location, cart_url, filename):
         """
@@ -34,7 +34,7 @@ def _download_from_url(self, location, cart_url, filename):
         """
         resp = requests.get(
             '{}?filename={}'.format(cart_url, filename),
-            stream=True, **self.auth
+            stream=True, **self.cart_api.auth
         )
         cart_tar = tarfile.open(name=None, mode='r|', fileobj=resp.raw)
         cart_tar.extractall(location)

tests/downloader_test.py

@@ -13,10 +13,15 @@
 class TestDownloader(TestCase):
     """Test the Downloader class."""
 
+    def test_cart_api_url(self):
+        """Test the CartAPI URL."""
+        down = Downloader(proto='http', addr='127.0.0.1', port=8081)
+        self.assertEqual(down.cart_api.cart_api_url, 'http://127.0.0.1:8081')
+
     def test_download_policy(self):
         """Test the download with policy."""
         down_path = mkdtemp()
-        down = Downloader('http://127.0.0.1:8081')
+        down = Downloader(cart_api_url='http://127.0.0.1:8081')
         resp = requests.get('http://127.0.0.1:8181/status/transactions/by_id/67')
         self.assertEqual(resp.status_code, 200)
         down.transactioninfo(down_path, resp.json())
@@ -44,7 +49,7 @@ def sha1sum(text_data):
             ]
         }
         down_path = mkdtemp()
-        down = Downloader('http://127.0.0.1:8081')
+        down = Downloader(cart_api_url='http://127.0.0.1:8081')
         down.cloudevent(down_path, cloud_event_stub)
         for file_id in range(1100, 1110):
             self.assertTrue(