Update for new GQA outputs.

eodatasets/metadata/gqa.py

@@ -5,7 +5,6 @@
 
 import datetime
 import logging
-import re
 
 import yaml
 
@@ -57,32 +56,18 @@ def populate_from_gqa(md, gqa_file):
     :type gqa_file: pathlib.Path
     :rtype eodatasets.type.DatasetMetadata
     """
-    # Example: 20141201_20010425_B6_gqa_results.yaml
-    fields = re.match(
-        (
-            r"(?P<acq_day>[0-9]{8})"
-            r"_(?P<ref_day>[0-9]{8})"
-            r"_B(?P<band>[0-9a-zA-Z]+)"
-            r"_gqa_results"
-            "$"
-        ), gqa_file.stem).groupdict()
-
-    # Parse from filename: date, reference date, band
-    md.gqa = {
-        'acq_day': _parse_day('acq_day', fields),
-        'ref_day': _parse_day('ref_day', fields),
-        'band': fields['band']
-    }
-
     with gqa_file.open('r') as f:
         gqa_values = yaml.safe_load(f)
 
     # "Scene id" is just the name of the parent folder. Often wrong and misleading.
     del gqa_values['scene_id']
 
-    md.gqa.update(
-        gqa_values
-    )
+    version = gqa_values.pop('software_version')
+    repo = gqa_values.pop('software_repository')
+
+    md.gqa = {}
+    md.gqa.update(gqa_values)
+    md.lineage.machine.note_software_version('gqa', version=version, repo_url=repo)
 
     return md
 

tests/integration/input/ls7-ortho/data/additional/20130818_20000119_B5_gqa_results.yaml

@@ -1,28 +1,33 @@
-abs_iterative_mean_residual:
-    x: 8.281189264069264
-    y: 4.081039145021645
-cep90: 440.00546272953613
+cep90: 0.41
 colors:
-    blue: 10.0
-    green: 0.0
-    red: 274.0
-    teal: 0.0
-    yellow: 25.0
-final_gcp_count: 308
-iterative_mean_residual:
-    x: 8.281189264069264
-    y: 0.9739366558441558
-iterative_stddev_residual:
-    x: 4.23604905517344
-    y: 4.524645392606803
-mean_residual:
-    x: 8.281189264069264
-    y: 0.9739366558441558
-ref_source: GLS_v1
+    blue: 600.0
+    green: 30164.0
+    red: 336.0
+    teal: 1340.0
+    yellow: 399.0
+final_gcp_count: 32582
+ref_date: 2000-09-04
+ref_source: GLS_v2
+ref_source_path: /g/data/v10/eoancillarydata/GCP/GQA_v2/wrs2/091/081/LE70910812000248ASA00_B5.TIF
 residual:
-    x: 8.266335
-    y: 4.070382
-scene_id: scenes
-stddev_residual:
-    x: 4.23604905517344
-    y: 4.524645392606803
+    abs:
+        x: 0.2
+        y: 0.23
+    abs_iterative_mean:
+        x: 0.15
+        y: 0.17
+    iterative_mean:
+        x: 0.02
+        y: 0.0
+    iterative_stddev:
+        x: 0.32
+        y: 0.52
+    mean:
+        x: 0.01
+        y: -0.03
+    stddev:
+        x: 1.27
+        y: 3.94
+scene_id: LS5_TM_OTH_P51_GALPGS01-002_091_081_20060812
+software_repository: https://github.com/GeoscienceAustralia/gqa.git
+software_version: 0.4+20.gb0d00dc

tests/integration/input/ls8-ortho/data/additional/20141201_20010425_B6_gqa_results.yaml

@@ -1,28 +1,33 @@
-abs_iterative_mean_residual:
-    x: 0.6114941044946384
-    y: 0.6841843509012092
-cep90: 79.42727444360659
+cep90: 0.41
 colors:
-    blue: 121.0
-    green: 843.0
-    red: 124.0
-    teal: 319.0
-    yellow: 104.0
-final_gcp_count: 1461
-iterative_mean_residual:
-    x: -0.14231046315309148
-    y: 0.05311033766826375
-iterative_stddev_residual:
-    x: 1.263801209338813
-    y: 1.382672269713929
-mean_residual:
-    x: -0.18180196558570483
-    y: 0.148098508713876
-ref_source: GLS_v1
+    blue: 600.0
+    green: 30164.0
+    red: 336.0
+    teal: 1340.0
+    yellow: 399.0
+final_gcp_count: 32582
+ref_date: 2000-09-04
+ref_source: GLS_v2
+ref_source_path: /g/data/v10/eoancillarydata/GCP/GQA_v2/wrs2/091/081/LE70910812000248ASA00_B5.TIF
 residual:
-    x: 0.8409989999999999
-    y: 0.9375190000000001
-scene_id: scenes
-stddev_residual:
-    x: 2.3751053554108696
-    y: 2.361825254310523
+    abs:
+        x: 0.2
+        y: 0.23
+    abs_iterative_mean:
+        x: 0.15
+        y: 0.17
+    iterative_mean:
+        x: 0.02
+        y: 0.0
+    iterative_stddev:
+        x: 0.32
+        y: 0.52
+    mean:
+        x: 0.01
+        y: -0.03
+    stddev:
+        x: 1.27
+        y: 3.94
+scene_id: LS5_TM_OTH_P51_GALPGS01-002_091_081_20060812
+software_repository: https://github.com/GeoscienceAustralia/gqa.git
+software_version: 0.4+20.gb0d00dc

tests/integration/test_ls7_ortho.py

@@ -180,25 +180,44 @@ def test_package():
     'product_level': 'L1G',
     'product_type': 'ortho',
     'gqa': {
-        'colors': {
-            'teal': 0.0, 'blue': 10.0, 'red': 274.0, 'green': 0.0, 'yellow': 25.0}, 'band': '5',
-        'acq_day': datetime.date(2013, 8, 18),
-        'abs_iterative_mean_residual': {
-            'y': 4.081039145021645,
-            'x': 8.281189264069264},
-        'ref_source': 'GLS_v1', 'cep90': 440.00546272953613,
-        'final_gcp_count': 308,
-        'stddev_residual': {
-            'y': 4.524645392606803, 'x': 4.23604905517344},
-        'mean_residual': {
-            'y': 0.9739366558441558, 'x': 8.281189264069264},
-        'iterative_stddev_residual': {
-            'y': 4.524645392606803, 'x': 4.23604905517344},
-        'ref_day': datetime.date(2000, 1, 19),
-        'residual': {
-            'y': 4.070382, 'x': 8.266335},
-        'iterative_mean_residual': {
-            'y': 0.9739366558441558, 'x': 8.281189264069264}
+        "cep90": 0.41,
+        "colors": {
+            "blue": 600,
+            "green": 30164,
+            "red": 336,
+            "teal": 1340,
+            "yellow": 399
+        },
+        "final_gcp_count": 32582,
+        "ref_date": datetime.date(2000, 9, 4),
+        "ref_source": "GLS_v2",
+        "ref_source_path": "/g/data/v10/eoancillarydata/GCP/GQA_v2/wrs2/091/081/LE70910812000248ASA00_B5.TIF",
+        "residual": {
+            "abs": {
+                "x": 0.2,
+                "y": 0.23
+            },
+            "abs_iterative_mean": {
+                "x": 0.15,
+                "y": 0.17
+            },
+            "iterative_mean": {
+                "x": 0.02,
+                "y": 0
+            },
+            "iterative_stddev": {
+                "x": 0.32,
+                "y": 0.52
+            },
+            "mean": {
+                "x": 0.01,
+                "y": -0.03
+            },
+            "stddev": {
+                "x": 1.27,
+                "y": 3.94
+            }
+        }
     },
     'size_bytes': 21280,
     'grid_spatial': {
@@ -274,7 +293,11 @@ def test_package():
         'machine': {
             'runtime_id': 'fa9f288a-dcf3-11e5-a095-0023dfa0db82',
             'software_versions': {
-                'pinkmatter': '4.1.4009'
+                'pinkmatter': '4.1.4009',
+                'gqa': {
+                    'repo_url': 'https://github.com/GeoscienceAustralia/gqa.git',
+                    'version': '0.4+20.gb0d00dc'
+                }
             },
             'hostname': 'witzo.local'
         },

tests/integration/test_ls8_ortho.py

@@ -8,8 +8,8 @@
 
 import yaml
 from pathlib import Path
-
 from tests import temp_dir, assert_file_structure, assert_same, integration_test, run_packaging_cli
+
 from tests.integration import load_checksum_filenames, hardlink_arg, prepare_datasets_for_comparison, FakeAncilFile, \
     prepare_work_order
 
@@ -291,40 +291,44 @@ def test_package():
         }
     },
     'gqa': {
-        'ref_source': 'GLS_v1',
-        'mean_residual': {
-            'y': 0.148098508713876,
-            'x': -0.18180196558570483
-        },
-        'acq_day': datetime.date(2014, 12, 1),
-        'ref_day': datetime.date(2001, 4, 25),
-        # Bands are always strings as they can have letters.
-        'band': '6',
-        'final_gcp_count': 1461,
-        'iterative_mean_residual': {
-            'y': 0.05311033766826375,
-            'x': -0.14231046315309148
-        },
-        'abs_iterative_mean_residual': {
-            'y': 0.6841843509012092,
-            'x': 0.6114941044946384
-        },
-        'residual': {
-            'y': 0.9375190000000001,
-            'x': 0.8409989999999999
-        },
-        'colors': {
-            'blue': 121.0, 'green': 843.0, 'teal': 319.0, 'yellow': 104.0, 'red': 124.0
-        },
-        'iterative_stddev_residual': {
-            'y': 1.382672269713929,
-            'x': 1.263801209338813
-        },
-        'stddev_residual': {
-            'y': 2.361825254310523,
-            'x': 2.3751053554108696
+        "cep90": 0.41,
+        "colors": {
+            "blue": 600,
+            "green": 30164,
+            "red": 336,
+            "teal": 1340,
+            "yellow": 399
         },
-        'cep90': 79.42727444360659
+        "final_gcp_count": 32582,
+        "ref_date": datetime.date(2000, 9, 4),
+        "ref_source": "GLS_v2",
+        "ref_source_path": "/g/data/v10/eoancillarydata/GCP/GQA_v2/wrs2/091/081/LE70910812000248ASA00_B5.TIF",
+        "residual": {
+            "abs": {
+                "x": 0.2,
+                "y": 0.23
+            },
+            "abs_iterative_mean": {
+                "x": 0.15,
+                "y": 0.17
+            },
+            "iterative_mean": {
+                "x": 0.02,
+                "y": 0
+            },
+            "iterative_stddev": {
+                "x": 0.32,
+                "y": 0.52
+            },
+            "mean": {
+                "x": 0.01,
+                "y": -0.03
+            },
+            "stddev": {
+                "x": 1.27,
+                "y": 3.94
+            }
+        }
     },
     'acquisition':
         {
@@ -358,7 +362,11 @@ def test_package():
         {
             'machine': {
                 'software_versions': {
-                    'pinkmatter': '4.0.3616'
+                    'pinkmatter': '4.0.3616',
+                    'gqa': {
+                        'repo_url': 'https://github.com/GeoscienceAustralia/gqa.git',
+                        'version': '0.4+20.gb0d00dc'
+                    }
                 }
             },
             'ancillary': {