Read Pinkmatter version from output files.

eodatasets/drivers.py

@@ -608,7 +608,10 @@ def fill_metadata(self, dataset, path):
 
         if not dataset.lineage.machine:
             dataset.lineage.machine = ptype.MachineMetadata()
-        dataset.lineage.machine.software = str(nbar_metadata['algorithm_information']['software_version'])
+        nbar_version = nbar_metadata['algorithm_information']['software_version']
+        if dataset.lineage.machine.software_versions is None:
+            dataset.lineage.machine.software_versions = {}
+        dataset.lineage.machine.software_versions['nbar'] = str(nbar_version)
 
         dataset.product_doi = nbar_metadata['algorithm_information']['arg25_doi']
 

eodatasets/metadata/ortho.py

@@ -23,8 +23,14 @@ def populate_ortho(md, base_folder):
     """
     mtl_path = _get_file(base_folder, '*_MTL.txt')
     work_order = _find_parent_file(base_folder, 'work_order.xml')
+    lpgs_out = _find_parent_file(base_folder, 'lpgs_out.xml')
 
-    return _populate_ortho_from_files(base_folder, md, mtl_path, work_order)
+    return _populate_ortho_from_files(
+        base_folder, md,
+        mtl_path=mtl_path,
+        work_order_path=work_order,
+        lpgs_out_path=lpgs_out
+    )
 
 
 def _parse_type(s):
@@ -251,7 +257,7 @@ def _get_node_text(offset, parsed_doc):
     return file_search_directory
 
 
-def _populate_ortho_from_files(base_folder, md, mtl_path, work_order):
+def _populate_ortho_from_files(base_folder, md, mtl_path, work_order_path, lpgs_out_path):
     if not md:
         md = ptype.DatasetMetadata()
     if not base_folder:
@@ -261,9 +267,9 @@ def _populate_ortho_from_files(base_folder, md, mtl_path, work_order):
     mtl_doc = _load_mtl(str(mtl_path.absolute()))
 
     work_order_doc = None
-    if work_order:
-        _LOG.info('Reading work order %r', work_order)
-        work_order_doc = etree.parse(str(work_order))
+    if work_order_path:
+        _LOG.info('Reading work order %r', work_order_path)
+        work_order_doc = etree.parse(str(work_order_path))
 
     md = _populate_from_mtl_dict(md, mtl_doc, base_folder)
     md.lineage.ancillary.update(
@@ -291,6 +297,16 @@ def _populate_ortho_from_files(base_folder, md, mtl_path, work_order):
         })
     )
 
+    if lpgs_out_path:
+        _LOG.info('Reading lpgs_out: %r', lpgs_out_path)
+        lpgs_out_doc = etree.parse(str(lpgs_out_path))
+        pinkmatter_version = lpgs_out_doc.findall('./Version')[0].text
+
+        if md.lineage.machine.software_versions is None:
+            md.lineage.machine.software_versions = {}
+        md.lineage.machine.software_versions['pinkmatter'] = str(pinkmatter_version)
+        # We could read processing hostname, start & stop times. Do we care? We currently get that info elsewhere.
+
     return md
 
 

eodatasets/package.py

@@ -24,11 +24,11 @@
 
 
 def init_locally_processed_dataset(directory, dataset_driver, source_datasets,
-                                   software_provenance=None, uuid_=None):
+                                   software_versions=None, uuid_=None):
     """
     Create a blank dataset for a newly created dataset on this machine.
 
-    :type software_provenance: eodatasets.provenance.SoftwareProvenance
+    :type software_versions: dict[str, str]
     :param uuid_: The existing dataset_id, if any.
     :rtype: ptype.DatasetMetadata
     """
@@ -40,7 +40,7 @@ def init_locally_processed_dataset(directory, dataset_driver, source_datasets,
             machine=ptype.MachineMetadata(
                 hostname=socket.getfqdn(),
                 runtime_id=_RUNTIME_ID,
-                software=software_provenance,
+                software_versions=software_versions,
                 uname=' '.join(os.uname())
             ),
             source_datasets=source_datasets
@@ -50,7 +50,7 @@ def init_locally_processed_dataset(directory, dataset_driver, source_datasets,
 
 
 def init_existing_dataset(directory, dataset_driver, source_datasets,
-                          software_provenance=None, uuid_=None, source_hostname=None):
+                          software_versions=None, uuid_=None, source_hostname=None):
     """
     Package an existing dataset folder (with mostly unknown provenance).
 
@@ -60,6 +60,7 @@ def init_existing_dataset(directory, dataset_driver, source_datasets,
     local machine information.
 
     :param uuid_: The existing dataset_id, if any.
+    :type software_versions: dict[str, str]
     :param source_hostname: Hostname where processed, if known.
     :rtype: ptype.DatasetMetadata
     """
@@ -70,7 +71,7 @@ def init_existing_dataset(directory, dataset_driver, source_datasets,
         lineage=ptype.LineageMetadata(
             machine=ptype.MachineMetadata(
                 hostname=source_hostname,
-                software=software_provenance
+                software_versions=software_versions
             ),
             source_datasets=source_datasets
 

eodatasets/type.py

@@ -564,7 +564,7 @@ class MachineMetadata(SimpleObject):
         'runtime_id': uuid.UUID
     }
 
-    def __init__(self, hostname=None, runtime_id=None, type_id=None, version=None, software=None, uname=None):
+    def __init__(self, hostname=None, runtime_id=None, type_id=None, version=None, software_versions=None, uname=None):
         # Hostname the dataset was processed on.
         self.hostname = hostname
         # A uuid generated each time a program runs.
@@ -577,9 +577,9 @@ def __init__(self, hostname=None, runtime_id=None, type_id=None, version=None, s
         # Version of machine type/class (eg. puppet file VCS version)
         self.version = version
 
-        # Software provenance on the machine at time of processing.
-        #: :type: eodatasets.provenance.SoftwareProvenance
-        self.software = software
+        # TODO: This is rather informal. names of software used and their versions.
+        #: :type: dict[str, str]
+        self.software_versions = software_versions
 
         # Uname information of machine.
         #    - from python: ' '.join(os.uname()))

tests/integration/test_ls8_nbar.py

@@ -224,7 +224,9 @@ def test_package():
             },
         },
         'machine': {
-            'software': '4.0'
+            'software_versions': {
+                'nbar': '4.0'
+            }
         },
         'ancillary': {
             'band_3_brdf_geo': {

tests/integration/test_ls8_ortho.py

@@ -344,7 +344,11 @@ def mock_get_node_text(offset, parsed_doc):
     },
     'lineage':
         {
-            'machine': {},
+            'machine': {
+                'software_versions': {
+                    'pinkmatter': '4.0.3616'
+                }
+            },
             'ancillary': {
                 'cpf': {
                     'name': 'L8CPF20140101_20140331.05'
@@ -404,7 +408,8 @@ def mock_get_node_text(offset, parsed_doc):
                                             'uname': 'Darwin niggle.local 14.3.0 Darwin Kernel Version 14.3.0: '
                                                      'Mon Mar 23 11:59:05 PDT 2015; '
                                                      'root:xnu-2782.20.48~5/RELEASE_X86_64 x86_64',
-                                            'runtime_id': '4bc6225c-e8b9-11e4-8b66-1040f381a756'
+                                            'runtime_id': '4bc6225c-e8b9-11e4-8b66-1040f381a756',
+
                                         },
                                     'source_datasets': {}
                                 },

tests/metadata/mtl/__init__.py

@@ -21,7 +21,7 @@ def assert_expected_mtl(mtl_file, expected_ds, base_folder=Path('/tmp/fake-folde
     assigned_id = uuid.UUID('3ff71eb0-d5c5-11e4-aebb-1040f381a756')
 
     ds = ptype.DatasetMetadata(id_=assigned_id)
-    ds = ortho._populate_ortho_from_files(base_folder, ds, mtl_file, None)
+    ds = ortho._populate_ortho_from_files(base_folder, ds, mtl_file, None, None)
 
     expected_ds.id_ = assigned_id
 

tests/test_drivers.py

@@ -2,8 +2,8 @@
 from __future__ import absolute_import
 
 import datetime
-from uuid import UUID
 from textwrap import dedent
+from uuid import UUID
 
 from pathlib import Path
 
@@ -99,7 +99,7 @@
     lineage=ptype.LineageMetadata(
         source_datasets={'ortho': test_ls8.EXPECTED_OUT},
         algorithm=ptype.AlgorithmMetadata(name='terrain', version='1.0'),
-        machine=ptype.MachineMetadata(software='1.0'),
+        machine=ptype.MachineMetadata(software_versions={'nbar': '1.0'}),
     )
 )