Fix missing Ortho CPF information.

eodatasets/metadata/ortho.py

@@ -21,9 +21,9 @@ def populate_ortho(md, base_folder):
     :type base_folder: pathlib.Path
     :rtype: eodatasets.type.DatasetMetadata
     """
-    mtl_path = _get_file(base_folder, '*_MTL.txt')
-    work_order = _find_parent_file(base_folder, 'work_order.xml')
-    lpgs_out = _find_parent_file(base_folder, 'lpgs_out.xml')
+    mtl_path = _get_mtl(base_folder)
+    work_order = _get_work_order(base_folder)
+    lpgs_out = _get_lpgs_out(base_folder)
 
     return _populate_ortho_from_files(
         base_folder, md,
@@ -33,6 +33,22 @@ def populate_ortho(md, base_folder):
     )
 
 
+def _get_lpgs_out(base_folder):
+    lpgs_out = _find_parent_file(base_folder, 'lpgs_out.xml')
+    return lpgs_out
+
+
+def _get_work_order(base_folder):
+    """
+    :rtype: pathlib.Path
+    """
+    return _find_parent_file(base_folder, 'work_order.xml')
+
+
+def _get_mtl(base_folder):
+    return _get_file(base_folder, '*_MTL.txt')
+
+
 def _parse_type(s):
     """Parse the string `s` and return a native python object.
 
@@ -107,7 +123,7 @@ def parse(lines, tree, level=0):
                     tree[key.lower()] = _parse_type(value)
 
     tree = {}
-    with open(filename, 'r') as fo:
+    with open(str(filename), 'r') as fo:
         parse(fo.readlines(), tree)
 
     return tree[root]
@@ -253,7 +269,7 @@ def _get_ancillary_metadata(mtl_doc, wo_doc, mtl_name_offset, order_dir_offset):
 
 def _get_node_text(offset, parsed_doc):
     xml_node = parsed_doc.findall(offset)
-    file_search_directory = Path(xml_node[0].text)
+    file_search_directory = Path(str(xml_node[0].text).strip())
     return file_search_directory
 
 
@@ -264,42 +280,19 @@ def _populate_ortho_from_files(base_folder, md, mtl_path, work_order_path, lpgs_
         base_folder = mtl_path.parent
 
     _LOG.info('Reading MTL %r', mtl_path)
-    mtl_doc = _load_mtl(str(mtl_path.absolute()))
+    mtl_doc = _load_mtl(mtl_path.absolute())
 
-    work_order_doc = None
-    if work_order_path:
-        _LOG.info('Reading work order %r', work_order_path)
-        work_order_doc = etree.parse(str(work_order_path))
+    _LOG.info('Reading work order %r', work_order_path)
+    work_order_doc = _load_xml(work_order_path) if work_order_path else None
 
     md = _populate_from_mtl_dict(md, mtl_doc, base_folder)
-    md.lineage.ancillary.update(
-        _remove_missing({
-            'cpf': _get_ancillary_metadata(
-                mtl_doc, work_order_doc,
-                mtl_name_offset=('PRODUCT_METADATA', 'cpf_name'),
-                order_dir_offset='./L0RpProcessing/CalibrationFile'
-            ),
-            'bpf_oli': _get_ancillary_metadata(
-                mtl_doc, work_order_doc,
-                mtl_name_offset=('PRODUCT_METADATA', 'bpf_name_oli'),
-                order_dir_offset='./L1Processing/BPFOliFile'
-            ),
-            'bpf_tirs': _get_ancillary_metadata(
-                mtl_doc, work_order_doc,
-                mtl_name_offset=('PRODUCT_METADATA', 'bpf_name_tirs'),
-                order_dir_offset='./L1Processing/BPFTirsFile'
-            ),
-            'rlut': _get_ancillary_metadata(
-                mtl_doc, work_order_doc,
-                mtl_name_offset=('PRODUCT_METADATA', 'rlut_file_name'),
-                order_dir_offset='./L1Processing/RlutFile'
-            )
-        })
-    )
+
+    ancil_files = _get_ancil_files(mtl_doc, work_order_doc)
+    md.lineage.ancillary.update(ancil_files)
 
     if lpgs_out_path:
         _LOG.info('Reading lpgs_out: %r', lpgs_out_path)
-        lpgs_out_doc = etree.parse(str(lpgs_out_path))
+        lpgs_out_doc = _load_xml(lpgs_out_path)
         pinkmatter_version = lpgs_out_doc.findall('./Version')[0].text
 
         md.lineage.machine.note_software_version('pinkmatter', str(pinkmatter_version))
@@ -308,6 +301,36 @@ def _populate_ortho_from_files(base_folder, md, mtl_path, work_order_path, lpgs_
     return md
 
 
+def _load_xml(path):
+    return etree.parse(str(path))
+
+
+def _get_ancil_files(mtl_doc, work_order_doc):
+    ancil_files = _remove_missing({
+        'cpf': _get_ancillary_metadata(
+            mtl_doc, work_order_doc,
+            mtl_name_offset=('PRODUCT_METADATA', 'cpf_name'),
+            order_dir_offset='./L0RpProcessing/CalibrationFile'
+        ),
+        'bpf_oli': _get_ancillary_metadata(
+            mtl_doc, work_order_doc,
+            mtl_name_offset=('PRODUCT_METADATA', 'bpf_name_oli'),
+            order_dir_offset='./L1Processing/BPFOliFile'
+        ),
+        'bpf_tirs': _get_ancillary_metadata(
+            mtl_doc, work_order_doc,
+            mtl_name_offset=('PRODUCT_METADATA', 'bpf_name_tirs'),
+            order_dir_offset='./L1Processing/BPFTirsFile'
+        ),
+        'rlut': _get_ancillary_metadata(
+            mtl_doc, work_order_doc,
+            mtl_name_offset=('PRODUCT_METADATA', 'rlut_file_name'),
+            order_dir_offset='./L1Processing/RlutFile'
+        )
+    })
+    return ancil_files
+
+
 def _populate_extent(md, product_md):
     # Extent
     if not md.extent:

tests/integration/input/ls8-ortho/data/work_order.xml

@@ -12,14 +12,14 @@
 
     <L0RaProcessing>
         <CalibrationFile useDownloader="false">
-            /g/data/v10/eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile
+            {cpf_path}
         </CalibrationFile>
         <Parameters mayFallBackSensor="true"/>
     </L0RaProcessing>
 
     <L0RpProcessing maxConcurrentScenes="13">
         <CalibrationFile useDownloader="false">
-            /g/data/v10/eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile
+            {cpf_path}
         </CalibrationFile>
     </L0RpProcessing>
 
@@ -51,13 +51,11 @@
             <DEM format="srtm90">/g/data/v10/eoancillarydata/elevation/2.0</DEM>
             <GCP format="chips">/g/data/v10/eoancillarydata/GCP/GLS2005_GCP_CHIP</GCP>
         </L1T>
-        <RlutFile useDownloader="false">/g/data/v10/eoancillarydata/sensor-specific/LANDSAT8/RLUT</RlutFile>
+        <RlutFile useDownloader="false">{rlut_path}</RlutFile>
         <BPFTirsFile
-                useDownloader="false">/g/data/v10/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/12
-        </BPFTirsFile>
+                useDownloader="false">{bpf_tirs_path}</BPFTirsFile>
         <BPFOliFile
-                useDownloader="false">/g/data/v10/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/12
-        </BPFOliFile>
+                useDownloader="false">{bpf_oli_path}</BPFOliFile>
         <OutputPath additionalMetadata="false">/g/data/v10/agdc/jez/galpgs/test-out/tmp/scenes</OutputPath>
     </L1Processing>
 </LandsatProcessingRequest>

tests/integration/test_ls8_ortho.py

@@ -5,12 +5,13 @@
 from __future__ import absolute_import
 
 import datetime
+import shutil
 import socket
 
 import yaml
 from pathlib import Path
 
-import eodatasets.metadata.ortho
+from eodatasets.metadata.ortho import _get_lpgs_out, _get_work_order
 from eodatasets.package import _RUNTIME_ID
 from tests import temp_dir, assert_file_structure, assert_same, integration_test, run_packaging_cli
 from tests.integration import load_checksum_filenames, hardlink_arg, directory_size, add_default_software_versions
@@ -27,56 +28,63 @@
 
 
 @integration_test
-def test_package(monkeypatch):
-    output_path = temp_dir()
-    ancil_path = temp_dir()
+def test_package():
+    work_path = temp_dir()
+    output_path = work_path.joinpath('out')
+    output_path.mkdir(parents=True)
+    ancil_path = work_path.joinpath('ancil')
+    input_product_path = work_path.joinpath('product')
 
     # We have to override the ancillary directory lookup as they won't exist on test systems.
     ANCIL_FILES = {
         'cpf': (
-            './L0RpProcessing/CalibrationFile',
             ancil_path.joinpath('cpf'),
             ('L8CPF20140101_20140331.05',),
             'da39a3ee5e6b4b0d3255bfef95601890afd80709'
         ),
         'bpf_oli': (
-            './L1Processing/BPFOliFile',
             ancil_path.joinpath('bpf-oli'),
             ('LO8BPF20140127130115_20140127144056.01',),
             'da39a3ee5e6b4b0d3255bfef95601890afd80709'
         ),
         'bpf_tirs': (
-            './L1Processing/BPFTirsFile',
             ancil_path.joinpath('bpf-tirs'),
             ('LT8BPF20140116023714_20140116032836.02',),
             'da39a3ee5e6b4b0d3255bfef95601890afd80709'
         ),
         'rlut': (
-            './L1Processing/RlutFile',
             ancil_path.joinpath('rlut'),
             # It should search subdirectories too.
             ('2013', 'L8RLUT20130211_20431231v09.h5'),
             'da39a3ee5e6b4b0d3255bfef95601890afd80709'
         )
     }
-    for name, (_, dir, file_offset, _) in ANCIL_FILES.items():
+    for name, (dir, file_offset, _) in ANCIL_FILES.items():
         # Create directories
         dir.joinpath(*file_offset[:-1]).mkdir(parents=True)
         # Create blank ancil file.
         dir.joinpath(*file_offset).open('w').close()
 
-    def mock_get_node_text(offset, parsed_doc):
-        for name, (o, dir, _, _) in ANCIL_FILES.items():
-            if offset == o:
-                return dir
-        raise AssertionError("Unknown work_order.xml offset requested: " + offset)
+    # Write all our input data to a temp directory (so that we can use a custom work order)
+    shutil.copytree(str(source_dataset), str(input_product_path))
+    shutil.copy(
+        str(_get_lpgs_out(source_dataset)),
+        str(work_path.joinpath('lpgs_out.xml'))
+    )
+    # Write a work order with ancillary locations replaced.
+    with _get_work_order(source_dataset).open('rb') as wo:
+        wo_text = wo.read().decode('utf-8').format(
+            **{k + '_path': v[0] for k, v in ANCIL_FILES.items()}
+        )
+        with work_path.joinpath('work_order.xml').open('w') as out_wo:
+            out_wo.write(wo_text)
 
-    monkeypatch.setattr(eodatasets.metadata.ortho, '_get_node_text', mock_get_node_text)
+    # Run!
     run_packaging_cli([
         hardlink_arg(output_path, source_dataset),
         'ortho', '--newly-processed',
         '--parent', str(parent_dataset),
-        str(source_dataset), str(output_path)
+        str(input_product_path), str(output_path)
     ])
 
     output_dataset = output_path.joinpath('LS8_OLITIRS_OTH_P51_GALPGS01-002_112_079_20140126')
@@ -125,9 +133,10 @@ def mock_get_node_text(offset, parsed_doc):
     EXPECTED_METADATA['lineage']['machine']['runtime_id'] = str(_RUNTIME_ID)
     EXPECTED_METADATA['lineage']['machine']['hostname'] = socket.getfqdn()
 
+    # Create the expected ancillary information.
     for ancil_name, ancil_d in EXPECTED_METADATA['lineage']['ancillary'].items():
         assert ancil_name in ANCIL_FILES, 'Unexpected ancil type: ' + ancil_name
-        _, dir_, file_offset, chk = ANCIL_FILES[ancil_name]
+        dir_, file_offset, chk = ANCIL_FILES[ancil_name]
         #: :type: pathlib.Path
         ancil_path = dir_.joinpath(*file_offset)
         ancil_d['uri'] = str(ancil_path)

tests/metadata/work_orders.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+"""
+
+import logging
+
+_LOG = logging.getLogger(__name__)