feat: Implemented Local and Remote Stub Sources (#18)

* feat(stubs): Initial Setup for stubs.source Module

* feat(utils): Url Utility Functions

Added Url related utility functions

* test(stubs): Initial Tests for source module

* feat: Resolve Path to both Local and Remote Source

Added ready method to resolve path to a local or remote stub. In the case of a remote, the source is
first downloaded/unpacked into a temporary directory.

micropy/stubs/__init__.py

@@ -1,7 +1,14 @@
 # -*- coding: utf-8 -*-
 
-"""Module for stub handling."""
+"""
+micropy.stubs
+~~~~~~~~~~~~~~
 
+This module contains all functionality relating
+to stub files/frozen modules and their usage in MicropyCli
+"""
+
+from . import source
 from .stubs import StubManager
 
-__all__ = ['StubManager']
+__all__ = ['StubManager', 'source']

micropy/stubs/source.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+
+"""
+micropy.stubs.source
+~~~~~~~~~~~~~~
+
+This module contains abstractions for handling stub sources
+and their location.
+"""
+
+
+import io
+import shutil
+import tarfile
+import tempfile
+from contextlib import contextmanager
+from functools import partial
+from pathlib import Path
+
+import requests
+
+from micropy import utils
+from micropy.logger import Log
+
+
+class StubSource:
+    """Abstract Base Class for Stub Sources"""
+
+    def __init__(self, location):
+        self.location = location
+        _name = self.__class__.__name__
+        self.log = Log.add_logger(_name)
+
+    @contextmanager
+    def ready(self, path=None, teardown=None):
+        """Yields prepared Stub Source
+
+        Allows StubSource subclasses to have a preperation
+        method before providing a local path to itself.
+
+        Args:
+            path (str, optional): path to stub source.
+                Defaults to location.
+            teardown (func, optional): callback to execute on exit.
+                Defaults to None.
+
+        Yields:
+            Resolved PathLike object to stub source
+        """
+        _path = path or self.location
+        path = Path(_path).resolve()
+        yield path
+        if teardown:
+            teardown()
+
+
+class LocalStubSource(StubSource):
+    """Stub Source Subclass for local locations
+
+    Args:
+        path (str): Path to Stub Source
+
+    Returns:
+        obj: Instance of LocalStubSource
+    """
+
+    def __init__(self, path):
+        location = utils.ensure_existing_dir(path)
+        return super().__init__(location)
+
+
+class RemoteStubSource(StubSource):
+    """Stub Source for remote locations
+
+    Args:
+        url (str): URL to Stub Source
+
+    Returns:
+        obj: Instance of RemoteStubSource
+    """
+
+    def __init__(self, url):
+        location = utils.ensure_valid_url(url)
+        return super().__init__(location)
+
+    def _unpack_archive(self, file_bytes, path):
+        """Unpack archive from bytes buffer
+
+        Args:
+            file_bytes (bytes): Byte array to extract from
+                Must be from tarfile with gzip compression
+            path (str): path to extract file to
+
+        Returns:
+            path: path extracted to
+        """
+        tar_bytes_obj = io.BytesIO(file_bytes)
+        with tarfile.open(fileobj=tar_bytes_obj, mode="r:gz") as tar:
+            tar.extractall(path)
+        return path
+
+    def ready(self):
+        """Retrieves and unpacks source
+
+        Prepares remote stub resource by downloading and
+        unpacking it into a temporary directory.
+        This directory is removed on exit of the superclass
+        context manager
+
+        Returns:
+            callable: StubSource.ready parent method
+        """
+        tmp_dir = tempfile.mkdtemp()
+        tmp_path = Path(tmp_dir)
+        filename = utils.get_url_filename(self.location).split(".tar.gz")[0]
+        outpath = tmp_path / filename
+        resp = requests.get(self.location)
+        source_path = self._unpack_archive(resp.content, outpath)
+        teardown = partial(shutil.rmtree, tmp_path)
+        return super().ready(path=source_path, teardown=teardown)
+
+
+def get_source(location, **kwargs):
+    """Factory for StubSource Instance
+
+    Args:
+        location (str): PathLike object or valid URL
+
+    Returns:
+        obj: Either Local or Remote StubSource Instance
+    """
+    if utils.is_url(location):
+        return RemoteStubSource(location, **kwargs)
+    return LocalStubSource(location, **kwargs)

micropy/utils/__init__.py

@@ -8,9 +8,8 @@
 MicropyCli.
 """
 
+from .helpers import *  # noqa
 from .pybwrapper import PyboardWrapper
-from .utils import ensure_existing_dir, ensure_valid_url, is_url
 from .validate import Validator
 
-__all__ = ["Validator", "PyboardWrapper", "is_url",
-           "ensure_existing_dir", "ensure_valid_url"]
+__all__ = ["Validator", "PyboardWrapper"]

micropy/utils/utils.py → micropy/utils/helpers.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-micropy.utils.utils
+micropy.utils.helpers
 ~~~~~~~~~~~~~~
 
 This module contains generic utility helpers
@@ -11,8 +11,12 @@
 from pathlib import Path
 
 import requests
-from requests.compat import urlparse
-from requests.exceptions import ConnectionError, HTTPError, InvalidURL
+from requests import exceptions as reqexc
+from requests import utils as requtil
+
+__all__ = ["is_url", "get_url_filename",
+           "ensure_existing_dir", "ensure_valid_url",
+           "is_downloadable"]
 
 
 def is_url(url):
@@ -24,7 +28,7 @@ def is_url(url):
     Returns:
         bool: True if arg url is a valid url
     """
-    scheme = urlparse(str(url)).scheme
+    scheme = requtil.urlparse(str(url)).scheme
     return scheme in ('http', 'https',)
 
 
@@ -43,16 +47,13 @@ def ensure_valid_url(url):
         str: valid url
     """
     if not is_url(url):
-        raise InvalidURL(f"{url} is not a valid url!")
+        raise reqexc.InvalidURL(f"{url} is not a valid url!")
     try:
         resp = requests.head(url)
-    except ConnectionError as e:
+    except reqexc.ConnectionError as e:
         raise e
     else:
-        code = resp.status_code
-        if not code == 200:
-            msg = f"{url} ({code}) did not respond with OK <200>"
-            raise HTTPError(msg)
+        resp.raise_for_status()
     return url
 
 
@@ -79,3 +80,34 @@ def ensure_existing_dir(path):
     if not path.is_dir():
         raise NotADirectoryError(f"{_path} is not a directory!")
     return path
+
+
+def is_downloadable(url):
+    """Checks if the url can be downloaded from
+
+    Args:
+        url (str): url to check
+
+    Returns:
+        bool: True if contains a downloadable resource
+    """
+    headers = requests.head(url).headers
+    content_type = headers.get("content-type").lower()
+    ctype = content_type.split("/")
+    if any(t in ('text', 'html',) for t in ctype):
+        return False
+    return True
+
+
+def get_url_filename(url):
+    """Parse filename from url
+
+    Args:
+        url (str): url to parse
+
+    Returns:
+        str: filename of url
+    """
+    path = requtil.urlparse(url).path
+    file_name = Path(path).name
+    return file_name

tests/conftest.py

@@ -50,3 +50,22 @@ def _mock_resolve(self, **kwargs):
     monkeypatch.setattr(Path, 'cwd', _mock_cwd)
     monkeypatch.setattr(Path, 'resolve', _mock_resolve)
     return tmp_path
+
+
+@pytest.fixture(scope="session")
+def test_urls():
+    def test_headers(type): return {
+        "content-type": type
+    }
+    return {
+        "valid": "http://www.google.com",
+        "valid_https": "https://www.google.com",
+        "invalid": "/foobar/bar/foo",
+        "invalid_file": "file:///foobar/bar/foo",
+        "bad_resp": "http://www.google.com/XYZ/ABC/BADRESP",
+        "download": "https://www.somewebsite.com/archive_test_stub.tar.gz",
+        "headers": {
+            "can_download": test_headers("application/gzip"),
+            "not_download": test_headers("text/plain")
+        }
+    }

tests/test_stub_source.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+import pytest
+
+from micropy.stubs import source
+
+
+@pytest.yield_fixture
+def test_archive(shared_datadir):
+    archive = shared_datadir / 'archive_test_stub.tar.gz'
+    file_obj = archive.open('rb')
+    file_bytes = file_obj.read()
+    yield file_bytes
+    file_obj.close()
+
+
+def test_get_source(shared_datadir, test_urls):
+    """should return correct subclass"""
+    test_path = shared_datadir / 'esp8266_test_stub'
+    local_stub = source.get_source(test_path)
+    assert isinstance(local_stub, source.LocalStubSource)
+    remote_stub = source.get_source(test_urls['valid'])
+    assert isinstance(remote_stub, source.RemoteStubSource)
+
+
+def test_source_ready(shared_datadir, test_urls, tmp_path, mocker,
+                      test_archive):
+    """should prepare and resolve stub"""
+    # Test LocalStub ready
+    test_path = shared_datadir / 'esp8266_test_stub'
+    local_stub = source.get_source(test_path)
+    expected_path = local_stub.location.resolve()
+    with local_stub.ready() as source_path:
+        assert source_path == expected_path
+
+    # Setup RemoteStub
+    test_parent = tmp_path / 'tmpdir'
+    test_parent.mkdir()
+    expected_path = (test_parent / 'archive_test_stub').resolve()
+    mocker.patch.object(source.utils, "ensure_valid_url",
+                        return_value=test_urls['download'])
+    mocker.patch.object(source.tempfile, "mkdtemp", return_value=test_parent)
+    get_mock = mocker.patch.object(source.requests, "get")
+    content_mock_val = mocker.PropertyMock(return_value=test_archive)
+    type(get_mock.return_value).content = content_mock_val
+    # Test Remote Stub
+    remote_stub = source.get_source(test_urls['download'])
+    with remote_stub.ready() as source_path:
+        print(list(source_path.parent.iterdir()))
+        assert source_path.exists()
+        assert str(source_path) == str(expected_path)

tests/test_utils.py

@@ -7,14 +7,6 @@
 
 from micropy import utils
 
-test_urls = {
-    "valid": "http://www.google.com",
-    "valid_https": "https://www.google.com",
-    "invalid": "/foobar/bar/foo",
-    "invalid_file": "file:///foobar/bar/foo",
-    "bad_resp": "http://www.google.com/XYZ/ABC/BADRESP"
-}
-
 
 @pytest.fixture
 def schema(datadir):
@@ -39,7 +31,7 @@ def test_fail_validate(schema):
         val.validate(fail_file)
 
 
-def test_is_url():
+def test_is_url(test_urls):
     """should respond true/false for url"""
     u = test_urls
     assert utils.is_url(u['valid'])
@@ -48,7 +40,7 @@ def test_is_url():
     assert not utils.is_url(u['invalid_file'])
 
 
-def test_ensure_valid_url(mocker):
+def test_ensure_valid_url(mocker, test_urls):
     """should ensure url is valid"""
     u = test_urls
     with pytest.raises(InvalidURL):
@@ -78,3 +70,23 @@ def test_ensure_existing_dir(tmp_path):
     assert result == tmp_path
     assert result.exists()
     assert result.is_dir()
+
+
+def test_is_downloadable(mocker, test_urls):
+    """should check if url can be downloaded from"""
+    u = test_urls
+    uheaders = u["headers"]
+    mock_head = mocker.patch.object(requests, "head")
+    head_mock_val = mocker.PropertyMock(
+        side_effect=[uheaders["not_download"],
+                     uheaders["can_download"]])
+    type(mock_head.return_value).headers = head_mock_val
+    assert not utils.is_downloadable(u["valid"])
+    assert utils.is_downloadable(u["valid"])
+
+
+def test_get_url_filename(test_urls):
+    """should return filename"""
+    filename = "archive_test_stub.tar.gz"
+    result = utils.get_url_filename(test_urls["download"])
+    assert result == filename