diff --git a/docs/config_samples/dem/dem_mappings.yaml b/docs/config_samples/dem/dem_mappings.yaml new file mode 100644 index 0000000000..112f257dc4 --- /dev/null +++ b/docs/config_samples/dem/dem_mappings.yaml @@ -0,0 +1,111 @@ + +name: DEM-S +description: DEM-S 25 metre, 1 degree tile + +match: + metadata: + platform: + code: ENDEAVOUR + instrument: + name: SRTM + product_type: DEM-S + + +location_name: eotiles + +file_path_template: '{platform[code]}_{instrument[name]}_{tile_index[0]}_{tile_index[1]}_DEMS_{start_time:%Y-%m-%dT%H-%M-%S.%f}.nc' + +global_attributes: + title: Experimental Data files From the Australian Geoscience Data Cube - DO NOT USE + summary: These files are experimental, short lived, and the format will change. + source: This data is a reprojection and retile of Landsat surface reflectance scene data available from /g/data/rs0/scenes/ + product_version: '0.0.0' + license: Creative Commons Attribution 4.0 International CC BY 4.0 + +storage: + driver: NetCDF CF + crs: | + GEOGCS["WGS 84", + DATUM["WGS_1984", + SPHEROID["WGS 84",6378137,298.257223563, + AUTHORITY["EPSG","7030"]], + AUTHORITY["EPSG","6326"]], + PRIMEM["Greenwich",0, + AUTHORITY["EPSG","8901"]], + UNIT["degree",0.0174532925199433, + AUTHORITY["EPSG","9122"]], + AUTHORITY["EPSG","4326"]] + tile_size: + longitude: 1.0 + latitude: 1.0 + resolution: + longitude: 0.00025 + latitude: -0.00025 + chunking: + longitude: 500 + latitude: 500 + time: 1 + dimension_order: ['time', 'latitude', 'longitude'] + +measurements: + 'dems': + dtype: float32 + resampling_method: cubic + varname: dems + nodata: -999 +--- + +name: DEM-H +description: DEM-H 25 metre, 1 degree tile + +match: + metadata: + platform: + code: ENDEAVOUR + instrument: + name: SRTM + product_type: DEM-H + + +location_name: eotiles + +file_path_template: '{platform[code]}_{instrument[name]}_{tile_index[0]}_{tile_index[1]}_DEMH_{start_time:%Y-%m-%dT%H-%M-%S.%f}.nc' + +global_attributes: + title: Experimental Data files From the Australian Geoscience Data Cube - DO NOT USE + summary: These files are experimental, short lived, and the format will change. + source: This data is a reprojection and retile of Landsat surface reflectance scene data available from /g/data/rs0/scenes/ + product_version: '0.0.0' + license: Creative Commons Attribution 4.0 International CC BY 4.0 + +storage: + driver: NetCDF CF + crs: | + GEOGCS["WGS 84", + DATUM["WGS_1984", + SPHEROID["WGS 84",6378137,298.257223563, + AUTHORITY["EPSG","7030"]], + AUTHORITY["EPSG","6326"]], + PRIMEM["Greenwich",0, + AUTHORITY["EPSG","8901"]], + UNIT["degree",0.0174532925199433, + AUTHORITY["EPSG","9122"]], + AUTHORITY["EPSG","4326"]] + tile_size: + longitude: 1.0 + latitude: 1.0 + resolution: + longitude: 0.00025 + latitude: -0.00025 + chunking: + longitude: 500 + latitude: 500 + time: 1 + dimension_order: ['time', 'latitude', 'longitude'] + +measurements: + 'demh': + dtype: float32 + resampling_method: cubic + varname: dems + nodata: -999 diff --git a/docs/config_samples/dem/dems_mapping.yaml b/docs/config_samples/dem/dems_mapping.yaml deleted file mode 100644 index 37000b8a43..0000000000 --- a/docs/config_samples/dem/dems_mapping.yaml +++ /dev/null @@ -1,56 +0,0 @@ - -name: DEM-S -description: DEM-S 25 metre, 1 degree tile - -match: - metadata: - platform: - code: SRTM_PLATFORM - instrument: - name: SRTM - product_type: DEM-S - - -location_name: eotiles - -file_path_template: '{platform[code]}_{instrument[name]}_{tile_index[0]}_{tile_index[1]}_DEMS_{start_time:%Y-%m-%dT%H-%M-%S.%f}.nc' - -global_attributes: - title: Experimental Data files From the Australian Geoscience Data Cube - DO NOT USE - summary: These files are experimental, short lived, and the format will change. - source: This data is a reprojection and retile of Landsat surface reflectance scene data available from /g/data/rs0/scenes/ - product_version: '0.0.0' - license: Creative Commons Attribution 4.0 International CC BY 4.0 - -storage: - driver: NetCDF CF - crs: | - GEOGCS["WGS 84", - DATUM["WGS_1984", - SPHEROID["WGS 84",6378137,298.257223563, - AUTHORITY["EPSG","7030"]], - AUTHORITY["EPSG","6326"]], - PRIMEM["Greenwich",0, - AUTHORITY["EPSG","8901"]], - UNIT["degree",0.0174532925199433, - AUTHORITY["EPSG","9122"]], - AUTHORITY["EPSG","4326"]] - tile_size: - longitude: 1.0 - latitude: 1.0 - resolution: - longitude: 0.00025 - latitude: -0.00025 - chunking: - longitude: 500 - latitude: 500 - time: 1 - dimension_order: ['time', 'latitude', 'longitude'] - -measurements: - 'dems': - dtype: float32 - resampling_method: cubic - varname: dems - nodata: -999 - diff --git a/utils/demprepare.py b/utils/demprepare.py index 3c62bffa78..f080e08a5f 100644 --- a/utils/demprepare.py +++ b/utils/demprepare.py @@ -28,16 +28,18 @@ def get_projection(img): def prepare_dataset(path): documents = [] - for dsfilename in list(path.glob('[ew][01][0-9][0-9][sn][0-9][0-9]dems'))[:10]: - format_ = None - creation_dt = datetime.fromtimestamp(dsfilename.stat().st_ctime).isoformat() - im = rasterio.open(str(dsfilename)) + for dspath in path.glob('[ew][01][0-9][0-9][sn][0-9][0-9]dem[sh]'): + dspath_str = str(dspath) + product_type = 'DEM-' + dspath_str[-1].upper() + band_name = 'dem' + dspath_str[-1].lower() + creation_dt = datetime.fromtimestamp(dspath.stat().st_ctime).isoformat() + im = rasterio.open(dspath_str) documents.append({ 'id': str(uuid.uuid4()), #'processing_level': level.replace('Level-', 'L'), - 'product_type': 'DEM-S', + 'product_type': product_type, 'creation_dt': creation_dt, - 'platform': {'code': 'SRTM_PLATFORM'}, + 'platform': {'code': 'ENDEAVOUR'}, 'instrument': {'name': 'SRTM'}, #'acquisition': {'groundstation': {'code': station}}, 'extent': { @@ -52,11 +54,12 @@ def prepare_dataset(path): }, 'image': { 'bands': { - 'dems': { - 'path': str(dsfilename) + band_name: { + 'path': dspath_str } } }, + # TODO: provenance chain is DSM -> DEM -> DEM-S -> DEM-H 'lineage': {'source_datasets': {}}, }) return documents