cleanup rainfall prepare script

opendatacube · Aug 18, 2016 · 301696c · 301696c
1 parent 85ac9c0
commit 301696c
Showing 1 changed file with 56 additions and 75 deletions.
diff --git a/utils/bom_rainfall_prepare.py b/utils/bom_rainfall_prepare.py
@@ -1,97 +1,78 @@
 # coding=utf-8
 """
 Ingest data from the command-line.
+
+python utils/bom_rainfall_prepare.py --output rainfall.yaml /g/data/rr5/agcd/0_05/rainfall/daily/*/*
 """
 from __future__ import absolute_import
 
 import uuid
-import logging
-from pathlib import Path
+from dateutil.parser import parse
 import yaml
 import click
-from osgeo import osr
-import os
 import netCDF4
 
-def prepare_dataset(path):
-    images = {}
-    documents = []
-    for image in path.glob('*.nc'):
-        fromtime = image.name[3:11]
-        totime = image.name[11:19]
-        image = netCDF4.Dataset(image)
-        times = image['time']
-        sensing_time = str(netCDF4.num2date(times[0], units=times.units, calendar=times.calendar))
-        projection = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'
-        left = float('111.975')
-        bottom = float('-44.525')
-        right = float('156.275')
-        top = float('-9.9750')
 
-        documents.append({
-            'id': str(uuid.uuid4()),
-            'processing_level': 'modelled_rainfall',
-            'product_type': str(image.cdm_data_type),
-            'creation_dt': str(image.date_created),
-            'platform': {'code': 'BoM'},
-            'instrument': {'name': 'rain gauge'},
-            'extent': {
-                'coord': {
-                    'ul': {'lon': left, 'lat': top},
-                    'ur': {'lon': right, 'lat': top},
-                    'll': {'lon': left, 'lat': bottom},
-                    'lr': {'lon': right, 'lat': bottom},
-                },
-                'from_dt': sensing_time,
-                'to_dt': sensing_time,
-                'center_dt': sensing_time
+def prepare_dataset(image):
+    image = netCDF4.Dataset(image)
+    times = image['time']
+    sensing_time = netCDF4.num2date(times[0], units=times.units, calendar=times.calendar).isoformat()
+
+    projection = 'EPSG:4326'
+    left, right = 111.975, 156.275
+    bottom, top = -44.525, -9.975
+
+    return {
+        'id': str(uuid.uuid4()),
+        'processing_level': 'modelled',
+        'product_type': 'rainfall',
+        'creation_dt': parse(image.date_created).isoformat(),
+        'platform': {'code': 'BoM'},
+        'instrument': {'name': 'rain gauge'},
+        'extent': {
+            'coord': {
+                'ul': {'lon': left, 'lat': top},
+                'ur': {'lon': right, 'lat': top},
+                'll': {'lon': left, 'lat': bottom},
+                'lr': {'lon': right, 'lat': bottom},
             },
-            'format': {'name': 'NETCDF'},
-            'grid_spatial': {
-                'projection': {
-                    'spatial_reference': projection,
-                    'geo_ref_points': {
-                        'ul': {'x': left, 'y': top},
-                        'ur': {'x': right, 'y': top},
-                        'll': {'x': left, 'y': bottom},
-                        'lr': {'x': right, 'y': bottom},
-                    }
+            'from_dt': sensing_time,
+            'to_dt': sensing_time,
+            'center_dt': sensing_time
+        },
+        'format': {'name': 'NETCDF'},
+        'grid_spatial': {
+            'projection': {
+                'spatial_reference': projection,
+                'geo_ref_points': {
+                    'ul': {'x': left, 'y': top},
+                    'ur': {'x': right, 'y': top},
+                    'll': {'x': left, 'y': bottom},
+                    'lr': {'x': right, 'y': bottom},
                 }
-            },
-            'image': {
-                'bands': {
-                    'rainfall': {
-                        'path': str(image.filepath()),
-                        'layer': 'rain_day',
-                    }
+            }
+        },
+        'image': {
+            'bands': {
+                'rainfall': {
+                    'path': str(image.filepath()),
+                    'layer': 'rain_day',
                 }
-            },
-            'lineage': {'source_datasets': {}},
-        })
-    return documents
+            }
+        },
+        'lineage': {'source_datasets': {}},
+    }
+
 
 @click.command(help="Prepare BoM interpolated rainfall grid dataset for ingestion into the Data Cube.")
 @click.argument('datasets',
-                type=click.Path(exists=True, readable=True, writable=True),
+                type=click.Path(exists=True, readable=True),
                 nargs=-1)
-@click.argument('output_path',
-                type=click.Path(exists=True, readable=True, writable=True),
-                nargs=1)
-def main(datasets, output_path):
-    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
-
-    for dataset in datasets:
-        path = Path(dataset)
-
-        logging.info("Processing %s", path)
-        documents = prepare_dataset(path)
-        if documents:
-            yaml_path = os.path.join(output_path, 'agdc_metadata.yaml')
-            logging.info("Writing %s dataset(s) into %s", len(documents), yaml_path)
-            with open(yaml_path, 'w') as stream:
-                yaml.dump_all(documents, stream)
-        else:
-            logging.info("No datasets discovered. Bye!")
+@click.option('--output', help="Write datasets into this file",
+              type=click.Path(exists=False, writable=True))
+def main(datasets, output):
+    with open(output, 'w') as stream:
+        yaml.dump_all((prepare_dataset(path) for path in datasets), stream)
 
 
 if __name__ == "__main__":