Enable support for google storage buckets

datacube/utils/uris.py

@@ -21,6 +21,8 @@ def is_url(url_str: str) -> bool:
     True
     >>> is_url('s3:///etc/blah')
     True
+    >>> is_url('gs://data/etc/blah.yaml')
+    True
     >>> is_url('/etc/blah')
     False
     >>> is_url('C:/etc/blah')
@@ -218,3 +220,7 @@ def register_scheme(*schemes):
 # s3:// not recognised by python by default
 #  without this `urljoin` might be broken for s3 urls
 register_scheme('s3')
+
+# gs:// not recognised by python by default
+#  without this `urljoin` might be broken for google storage urls
+register_scheme('gs')

integration_tests/index/test_index_data.py

@@ -122,7 +122,7 @@ def telemetry_dataset(index: Index, initialised_postgres_db: PostgresDb, default
 
 def test_index_duplicate_dataset(index: Index, initialised_postgres_db: PostgresDb,
                                  local_config,
-                                 default_metadata_type)->None:
+                                 default_metadata_type) -> None:
     dataset_type = index.products.add_document(_pseudo_telemetry_dataset_type)
     assert not index.datasets.has(_telemetry_uuid)
 

tests/test_utils.py

@@ -103,12 +103,18 @@ def test_uri_to_local_path():
 def test_uri_resolve():
     abs_path = '/abs/path/to/something'
     some_uri = 'http://example.com/file.txt'
-    base = 's3://foo'
-    assert uri_resolve(base, abs_path) == "file://" + abs_path
-    assert uri_resolve(base, some_uri) is some_uri
-    assert uri_resolve(base, None) is base
-    assert uri_resolve(base, '') is base
-    assert uri_resolve(base, 'relative/path') == base + '/relative/path'
+    s3_base = 's3://foo'
+    gs_base = 'gs://foo'
+    assert uri_resolve(s3_base, abs_path) == "file://" + abs_path
+    assert uri_resolve(s3_base, some_uri) is some_uri
+    assert uri_resolve(s3_base, None) is s3_base
+    assert uri_resolve(s3_base, '') is s3_base
+    assert uri_resolve(s3_base, 'relative/path') == s3_base + '/relative/path'
+    assert uri_resolve(gs_base, abs_path) == "file://" + abs_path
+    assert uri_resolve(gs_base, some_uri) is some_uri
+    assert uri_resolve(gs_base, None) is gs_base
+    assert uri_resolve(gs_base, '') is gs_base
+    assert uri_resolve(gs_base, 'relative/path') == gs_base + '/relative/path'
 
 
 def test_pick_uri():