Add test cases for non-file uri schemes

datacube/compat.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 # We're using references that don't exist in python 3 (unicode, long):
 # pylint: skip-file
+
 """
 Compatibility helpers for Python 2 and 3.
 
@@ -22,18 +23,27 @@
     range = range
 
     import configparser
+
     NoOptionError = configparser.NoOptionError
 
+
     def read_config(default_text=None):
         config = configparser.ConfigParser()
         if default_text:
             config.read_string(default_text)
         return config
 
 
+    # noinspection PyUnresolvedReferences
     from urllib.parse import urlparse, urljoin
+    # noinspection PyUnresolvedReferences
     from urllib.request import url2pathname
+    # noinspection PyUnresolvedReferences
     from itertools import zip_longest
+    # noinspection PyUnresolvedReferences
+    import urllib.parse as url_parse_module
+
+
 else:
     text_type = unicode
     string_types = (str, unicode)
@@ -46,17 +56,25 @@ def read_config(default_text=None):
 
     import ConfigParser
     from io import StringIO
+
     NoOptionError = ConfigParser.NoOptionError
 
+
     def read_config(default_text=None):
         config = ConfigParser.SafeConfigParser()
         if default_text:
             config.readfp(StringIO(default_text))
         return config
 
+
+    # noinspection PyUnresolvedReferences
     from urlparse import urlparse, urljoin
+    # noinspection PyUnresolvedReferences
     from urllib import url2pathname
+    # noinspection PyUnresolvedReferences
     from itertools import izip_longest as zip_longest
+    # noinspection PyUnresolvedReferences
+    import urlparse as url_parse_module
 
 
 def with_metaclass(meta, *bases):

datacube/storage/storage.py

@@ -9,7 +9,7 @@
 from contextlib import contextmanager
 from pathlib import Path
 
-from datacube.compat import urlparse, urljoin
+from datacube.compat import urlparse, urljoin, url_parse_module
 from datacube.config import OPTIONS
 from datacube.model import Dataset
 from datacube.storage import netcdf_writer
@@ -416,6 +416,19 @@ def get_crs(self):
         return self.crs
 
 
+def register_scheme(*schemes):
+    """
+    Register additional uri schemes as supporting relative offsets (etc), so that band/measurement paths can be
+    calculated relative to the base uri.
+    """
+    url_parse_module.uses_netloc.extend(schemes)
+    url_parse_module.uses_relative.extend(schemes)
+    url_parse_module.uses_params.extend(schemes)
+
+# Not recognised by python by default. Doctests below will fail without it.
+register_scheme('s3')
+
+
 def _resolve_url(base_url, path):
     """
     If path is a URL or an absolute path return URL
@@ -429,6 +442,12 @@ def _resolve_url(base_url, path):
     'file:///foo/abc'
     >>> _resolve_url('file:///foo/abc', '/bar')
     'file:///bar'
+    >>> _resolve_url('http://foo.com/abc/odc-metadata.yaml', 'band-5.tif')
+    'http://foo.com/abc/band-5.tif'
+    >>> _resolve_url('s3://foo.com/abc/odc-metadata.yaml', 'band-5.tif')
+    's3://foo.com/abc/band-5.tif'
+    >>> _resolve_url('s3://foo.com/abc/odc-metadata.yaml?something', 'band-5.tif')
+    's3://foo.com/abc/band-5.tif'
     """
     if path:
         if is_url(path):