Test adding metadata types, products and datasets over HTTP

datacube/scripts/metadata.py

@@ -26,15 +26,15 @@ def this_group():
 @click.option('--allow-exclusive-lock/--forbid-exclusive-lock', is_flag=True, default=False,
               help='Allow index to be locked from other users while updating (default: false)')
 @click.argument('files',
-                type=click.Path(exists=True, readable=True, writable=False),
+                type=str,
                 nargs=-1)
 @ui.pass_index()
 def add_metadata_types(index, allow_exclusive_lock, files):
     # type: (Index, bool, list) -> None
     """
     Add or update metadata types in the index
     """
-    for descriptor_path, parsed_doc in read_documents(*(Path(f) for f in files)):
+    for descriptor_path, parsed_doc in read_documents(*files):
         try:
             type_ = index.metadata_types.from_doc(parsed_doc)
             index.metadata_types.add(type_, allow_table_lock=allow_exclusive_lock)

datacube/scripts/product.py

@@ -29,15 +29,15 @@ def product_cli():
 @click.option('--allow-exclusive-lock/--forbid-exclusive-lock', is_flag=True, default=False,
               help='Allow index to be locked from other users while updating (default: false)')
 @click.argument('files',
-                type=click.Path(exists=True, readable=True, writable=False),
+                type=str,
                 nargs=-1)
 @ui.pass_index()
 def add_products(index, allow_exclusive_lock, files):
     # type: (Index, bool, list) -> None
     """
     Add or update products in the generic index.
     """
-    for descriptor_path, parsed_doc in read_documents(*(Path(f) for f in files)):
+    for descriptor_path, parsed_doc in read_documents(*files):
         try:
             type_ = index.products.from_doc(parsed_doc)
             index.products.add(type_, allow_table_lock=allow_exclusive_lock)

integration_tests/index/test_index_data.py

@@ -13,9 +13,9 @@
 import pytest
 from dateutil import tz
 
-from datacube.index.index import Index
-from datacube.index.exceptions import MissingRecordError
 from datacube.drivers.postgres import PostgresDb
+from datacube.index.exceptions import MissingRecordError
+from datacube.index.index import Index
 from datacube.model import Dataset, MetadataType
 
 _telemetry_uuid = UUID('4ec8fe97-e8b9-11e4-87ff-1040f381a756')
@@ -248,6 +248,7 @@ def test_index_dataset_with_sources(index, default_metadata_type):
     for p in ('skip', 'ensure', 'verify'):
         index.datasets.add(child, sources_policy=p)
 
+
 # Make sure that both normal and s3aio index can handle normal data locations correctly
 @pytest.mark.parametrize('datacube_env_name', ('datacube', 's3aio_env',), indirect=True)
 def test_index_dataset_with_location(index: Index, default_metadata_type: MetadataType):

integration_tests/test_dataset_add.py

@@ -1,10 +1,12 @@
 import math
-import yaml
+
 import toolz
+import yaml
 
+from datacube.index import Index
+from datacube.model import MetadataType
+from datacube.testutils import gen_dataset_test_dag, load_dataset_definition, write_files, dataset_maker
 from datacube.utils import SimpleDocNav
-from datacube.testutils import gen_dataset_test_dag, load_dataset_definition, write_files
-from datacube.testutils import dataset_maker
 
 
 def check_skip_lineage_test(clirunner, index):
@@ -70,21 +72,19 @@ def check_with_existing_lineage(clirunner, index):
 
     child_docs = [ds.sources[x].doc for x in ('ab', 'ac', 'ae')]
 
-    prefix = write_files({'lineage.yml':
-                          yaml.safe_dump_all(child_docs),
-                          'main.yml':
-                          yaml.safe_dump(ds.doc),
+    prefix = write_files({'lineage.yml': yaml.safe_dump_all(child_docs),
+                          'main.yml': yaml.safe_dump(ds.doc),
                           })
 
-    clirunner(['dataset', 'add', str(prefix/'lineage.yml')])
+    clirunner(['dataset', 'add', str(prefix / 'lineage.yml')])
     assert index.datasets.get(ds.sources['ae'].id) is not None
     assert index.datasets.get(ds.sources['ab'].id) is not None
     assert index.datasets.get(ds.sources['ac'].id) is not None
 
     clirunner(['dataset', 'add',
                '--no-auto-add-lineage',
                '--product', 'A',
-               str(prefix/'main.yml')])
+               str(prefix / 'main.yml')])
 
     assert index.datasets.get(ds.id) is not None
 
@@ -106,17 +106,15 @@ def check_inconsistent_lineage(clirunner, index):
     child_docs = [ds.sources[x].doc for x in ('ae',)]
     modified_doc = toolz.assoc_in(ds.doc, 'lineage.source_datasets.ae.label'.split('.'), 'modified')
 
-    prefix = write_files({'lineage.yml':
-                          yaml.safe_dump_all(child_docs),
-                          'main.yml':
-                          yaml.safe_dump(modified_doc),
+    prefix = write_files({'lineage.yml': yaml.safe_dump_all(child_docs),
+                          'main.yml': yaml.safe_dump(modified_doc),
                           })
 
-    clirunner(['dataset', 'add', str(prefix/'lineage.yml')])
+    clirunner(['dataset', 'add', str(prefix / 'lineage.yml')])
     assert index.datasets.get(ds.sources['ae'].id) is not None
 
     r = clirunner(['dataset', 'add',
-                   str(prefix/'main.yml')])
+                   str(prefix / 'main.yml')])
 
     assert 'ERROR Inconsistent lineage dataset' in r.output
 
@@ -127,7 +125,7 @@ def check_inconsistent_lineage(clirunner, index):
 
     # now again but skipping verification check
     r = clirunner(['dataset', 'add', '--no-verify-lineage',
-                   str(prefix/'main.yml')])
+                   str(prefix / 'main.yml')])
 
     assert index.datasets.has(ds.id)
     assert index.datasets.has(ds.sources['ab'].id)
@@ -155,24 +153,24 @@ def check_missing_lineage(clirunner, index):
 
     r = clirunner(['dataset', 'add',
                    '--no-auto-add-lineage',
-                   str(prefix/'main.yml')])
+                   str(prefix / 'main.yml')])
 
     assert 'ERROR Following lineage datasets are missing' in r.output
     assert index.datasets.has(ds.id) is False
 
     # now add lineage and try again
-    clirunner(['dataset', 'add', str(prefix/'lineage.yml')])
+    clirunner(['dataset', 'add', str(prefix / 'lineage.yml')])
     assert index.datasets.has(ds.sources['ae'].id)
     r = clirunner(['dataset', 'add',
                    '--no-auto-add-lineage',
-                   str(prefix/'main.yml')])
+                   str(prefix / 'main.yml')])
 
     assert index.datasets.has(ds.id)
 
 
 def check_missing_metadata_doc(clirunner):
     prefix = write_files({'im.tiff': ''})
-    r = clirunner(['dataset', 'add', str(prefix/'im.tiff')])
+    r = clirunner(['dataset', 'add', str(prefix / 'im.tiff')])
     assert "ERROR No supported metadata docs found for dataset" in r.output
 
 
@@ -184,7 +182,7 @@ def check_no_confirm(clirunner, path):
 
 def check_bad_yaml(clirunner, index):
     prefix = write_files({'broken.yml': '"'})
-    r = clirunner(['dataset', 'add', str(prefix/'broken.yml')])
+    r = clirunner(['dataset', 'add', str(prefix / 'broken.yml')])
     assert 'ERROR Failed reading documents from ' in r.output
 
 
@@ -278,13 +276,13 @@ def test_dataset_add_ambgious_products(dataset_add_configs, index_empty, clirunn
     })
 
     clirunner(['metadata', 'add', p.metadata])
-    clirunner(['product', 'add', str(prefix/'products.yml')])
+    clirunner(['product', 'add', str(prefix / 'products.yml')])
 
     pp = list(index.products.get_all())
     assert len(pp) == 2
 
     for ds, i in zip(dss, (1, 2)):
-        r = clirunner(['dataset', 'add', str(prefix/('dataset%d.yml' % i))])
+        r = clirunner(['dataset', 'add', str(prefix / ('dataset%d.yml' % i))])
         assert 'ERROR Auto match failed' in r.output
         assert 'matches several products' in r.output
         assert index.datasets.has(ds.id) is False
@@ -293,15 +291,15 @@ def test_dataset_add_ambgious_products(dataset_add_configs, index_empty, clirunn
     ds, fname = dss[0], 'dataset1.yml'
     r = clirunner(['dataset', 'add',
                    '--product', 'A',
-                   str(prefix/fname)])
+                   str(prefix / fname)])
 
     assert index.datasets.has(ds.id) is True
 
     # check that forcing via exclude works
     ds, fname = dss[1], 'dataset2.yml'
     r = clirunner(['dataset', 'add',
                    '--exclude-product', 'B',
-                   str(prefix/fname)])
+                   str(prefix / fname)])
 
     assert index.datasets.has(ds.id) is True
 
@@ -330,7 +328,7 @@ def test_dataset_add_with_nans(dataset_add_configs, index_empty, clirunner):
     r = clirunner(['dataset', 'add',
                    '--auto-add-lineage',
                    '--verify-lineage',
-                   str(prefix/'dataset.yml')])
+                   str(prefix / 'dataset.yml')])
 
     assert "ERROR" not in r.output
 
@@ -351,7 +349,7 @@ def test_dataset_add_inconsistent_measurements(dataset_add_configs, index_empty,
     mk = dataset_maker(0)
 
     # not set, empty, subset, full set, super-set
-    ds1 = SimpleDocNav(mk('A', product_type='eo',))
+    ds1 = SimpleDocNav(mk('A', product_type='eo', ))
     ds2 = SimpleDocNav(mk('B', product_type='eo', measurements={}))
     ds3 = SimpleDocNav(mk('C', product_type='eo', measurements={
         'red': {}
@@ -392,12 +390,12 @@ def test_dataset_add_inconsistent_measurements(dataset_add_configs, index_empty,
     })
 
     clirunner(['metadata', 'add', p.metadata])
-    r = clirunner(['product', 'add', str(prefix/'products.yml')])
+    r = clirunner(['product', 'add', str(prefix / 'products.yml')])
 
     pp = list(index.products.get_all())
     assert len(pp) == 1
 
-    r = clirunner(['dataset', 'add', str(prefix/'dataset.yml')])
+    r = clirunner(['dataset', 'add', str(prefix / 'dataset.yml')])
     print(r.output)
 
     r = clirunner(['dataset', 'search', '-f', 'csv'])
@@ -459,3 +457,20 @@ def test_dataset_archive_restore(dataset_add_configs, index_empty, clirunner):
     r = clirunner(['dataset', 'info', ds.id, ds.sources['ab'].id, ds.sources['ac'].id])
     assert 'status: active' in r.output
     assert 'status: archived' not in r.output
+
+
+def test_dataset_add_http(dataset_add_configs, index: Index, default_metadata_type: MetadataType, httpserver,
+                          clirunner):
+    # pytest-localserver also looks good, it's been around for ages, and httpserver is the new cool
+    p = dataset_add_configs
+
+    httpserver.expect_request("/metadata_types.yaml").respond_with_data(open(p.metadata).read())
+    httpserver.expect_request("/products.yaml").respond_with_data(open(p.products).read())
+    httpserver.expect_request("/datasets.yaml").respond_with_data(open(p.datasets).read())
+    # check that the request is served
+    #    assert requests.get(httpserver.url_for("/dataset.yaml")).yaml() == {'foo': 'bar'}
+
+    clirunner(['metadata', 'add', httpserver.url_for("/metadata_types.yaml")])
+    clirunner(['product', 'add', httpserver.url_for("/products.yaml")])
+    # clirunner(['dataset', 'add', p.datasets])
+    clirunner(['dataset', 'add', httpserver.url_for("/datasets.yaml")])

requirements-test.txt

@@ -64,6 +64,7 @@ pyPEG2==2.15.2
 pyproj==1.9.5.1
 pytest==3.5.0
 pytest-cov==2.5.1
+pytest-httpserver==0.3.0
 pytest-timeout==1.2.1
 python-dateutil==2.7.2
 pytz==2018.4

setup.cfg

@@ -10,3 +10,6 @@ tag_prefix = datacube-
 
 [bdist_wheel]
 universal=1
+
+[flake8]
+max-line-length = 120

setup.py

@@ -13,6 +13,7 @@
     'pytest',
     'pytest-cov',
     'pytest-timeout',
+    'pytest-httpserver',
     'moto',
 ]