Fix bug and improve dataio integration

docs/usage.rst

@@ -94,15 +94,34 @@ Note that dates and diffdates are two separate lists
 Using fmu-dataio for SUMO output
 --------------------------------
 
-From version 1.3, a ``fmu-dataio`` pipeline is supported and encouraged. This will make results
-compatible with upload to SUMO. This involves some changes:
+From version 1.3, a ``fmu-dataio`` pipeline is supported and encouraged. This will generate metadata
+and make results compatible with upload to SUMO. This involves some changes:
 
-  * The environment variable ``FMU_GLOBAL_CONFIG`` shall be set in advance and this shall
-    point to a fmu-dataio compatible global variables file (on YAML format), for example:
+  * The location of the FMU ``global_variables`` file must be set, which can be done either by:
 
-  .. code-block:: bash
+    .. code-block:: yaml
 
-      export FMU_GLOBAL_CONFIG=fmuconfig/output/global_variables.yml
+        input:
+          fmu_global_config: some_path/global_variables.yml  # relative to RUNPATH in ERT
+
+    or by setting one of the following environment variables either in shell or inside ERT
+    (see e.g. ``setenv`` in ERT)
+
+    .. code-block:: bash
+
+        # bash
+        export FMU_GLOBAL_CONFIG_GRD3DMAPS=fmuconfig/output/global_variables.yml
+        export FMU_GLOBAL_CONFIG=fmuconfig/output/global_variables.yml
+
+    .. code-block:: csh
+
+        # csh
+        setenv FMU_GLOBAL_CONFIG_GRD3DMAPS fmuconfig/output/global_variables.yml
+        setenv FMU_GLOBAL_CONFIG fmuconfig/output/global_variables.yml
+
+    The first environment variable will be dedicated to the AVG and HC scripts, while the last one
+    is a generic for ``fmu.dataio``. The priority is: first use the setting under ``input:`` in
+    the script, then ``FMU_GLOBAL_CONFIG_GRD3DMAPS`` and finally ``FMU_GLOBAL_CONFIG``
 
   * The output.mapfolder must either:
 
@@ -116,6 +135,7 @@ compatible with upload to SUMO. This involves some changes:
   .. code-block:: yaml
 
       input:
+        fmu_global_config: fmuconfig/output/global_variables.yml  # relative to RUNPATH in ERT
         eclroot: tests/data/reek/REEK
         grid: $eclroot.EGRID
 
@@ -127,3 +147,8 @@ compatible with upload to SUMO. This involves some changes:
             metadata:
               attribute: saturation
               unit: fraction
+
+        ...
+
+        output:
+          mapfolder: fmu-dataio

pyproject.toml

@@ -26,10 +26,10 @@ exclude = '''
 
 [build-system]
 requires = [
-    "pip>=19.1.1",
-    "setuptools>=43",
-    "wheel",
-    "setuptools_scm",
-    "Sphinx",
-    "sphinx-rtd-theme",
+  "pip>=19.1.1",
+  "setuptools>=43",
+  "wheel",
+  "setuptools_scm",
+  "Sphinx",
+  "sphinx-rtd-theme",
 ]

requirements/requirements.txt

@@ -3,3 +3,4 @@ pyyaml
 xtgeo>=2.20.7
 fmu-dataio>=1.3.0
 xtgeoviz
+fmu-config

setup.cfg

@@ -11,3 +11,6 @@ test = pytest
 
 [tool:pytest]
 addopts = --verbose -x
+
+[mypy]
+ignore_missing_imports = True

src/xtgeoapp_grd3dmaps/avghc/_export_via_fmudataio.py

@@ -1,42 +1,90 @@
 """General functions that exports maps / plots using fmu-dataio."""
 import json
 import os
+from pathlib import Path
 
 import fmu.dataio as dataio
+from fmu.config import utilities as ut
 from xtgeo.common import XTGeoDialog
 
 xtg = XTGeoDialog()
 
 logger = xtg.functionlogger(__name__)
 
 
+def _get_global_config(thisconfig):
+    """Get the global config in different manners. Priority:
+
+    (1) A setting inside the setup file: input: fmu_global_config: will win if present
+    (2) FMU_GLOBAL_CONFIG_GRD3DMAPS exists as env variable. Will be second
+    (3) FMU_GLOBAL_CONFIG as env variabel!
+    """
+    alternatives = [
+        "[input][fmu_global_config] in setup file",
+        "FMU_GLOBAL_CONFIG_GRD3DMAPS",
+        "FMU_GLOBAL_CONFIG",
+    ]
+
+    alt = []
+    if "input" in thisconfig:
+        alt.append(thisconfig["input"].get("fmu_global_config"))
+    else:
+        alt.append(None)
+
+    alt.append(os.environ.get("FMU_GLOBAL_CONFIG_GRD3DMAPS"))
+    alt.append(os.environ.get("FMU_GLOBAL_CONFIG"))
+
+    cfg = None
+
+    for altno, description in enumerate(alternatives):
+        alternative = alt[altno]
+        logger.info("Global %s", alternative)
+        if not alternative:
+            continue
+
+        if Path(alternative).is_file():
+            cfg = ut.yaml_load(alternative)
+            logger.info("Global no %s config from %s", alternative, description)
+            break
+        else:
+            raise IOError(
+                f"Config file does not exist: {alternative}, source is {description}"
+            )
+
+    if not cfg:
+        raise RuntimeError("Not able lo load the global config!")
+
+    return cfg
+
+
 def export_avg_map_dataio(surf, nametuple, config):
     """Export avererage maps using dataio.
 
     Args:
         surf: XTGeo RegularSurface object
         nametuple: On form ('myzone1', 'PRESSURE--19991201') where the last
             is an identifier (nameid) for the metadata config
-        config: The processed config setup
+        config: The processed config setup for this script (i.e. not global_config)
     """
 
     zoneinfo, nameid = nametuple
     logger.debug("Processed config: \n%s", json.dumps(config, indent=4))
 
-    # this routine is dependent that the env variable FMU_GLOBAL_CONFIG is active
-    if "FMU_GLOBAL_CONFIG" not in os.environ:
-        raise RuntimeError("The env variable FMU_GLOBAL_CONFIG is not set.")
+    fmu_global_config = _get_global_config(config)
 
     metadata = config["metadata"]
     if nameid not in metadata:
-        raise ValueError(f"Seems that medata for {nameid} is missing!")
+        raise ValueError(
+            f"Seems that 'metadata' for {nameid} is missing! Cf. documentation"
+        )
 
     mdata = metadata[nameid]
     name = mdata.get("name", "unknown_name")
     attribute = mdata.get("attribute", "unknown_attribute")
     unit = mdata.get("unit", None)
     tt1 = mdata.get("t1", None)
     tt2 = mdata.get("t2", None)
+
     if tt1 and tt1 not in nameid:
         tt1 = None
     if tt2 and tt2 not in nameid:
@@ -55,6 +103,7 @@ def export_avg_map_dataio(surf, nametuple, config):
             tdata = [[tt2, "base"]]
 
     edata = dataio.ExportData(
+        config=fmu_global_config,
         name=zoneinfo,
         unit=unit,
         content={"property": {"attribute": attribute, "is_discrete": False}},
@@ -66,7 +115,7 @@ def export_avg_map_dataio(surf, nametuple, config):
         verbosity="WARNING",
         workflow="xtgeoapp-grd3dmaps script average maps",
     )
-    fname = edata.export(surf, unit=unit)
+    fname = edata.export(surf)
     xtg.say(f"Output as fmu-dataio: {fname}")
     return fname
 
@@ -76,17 +125,15 @@ def export_hc_map_dataio(surf, zname, date, hcmode, config):
 
     Args:
         surf: XTGeo RegularSurface object
-        nametuple: The zone name
+        nametuple: The zone name.
         date: The date tag
         config: The processed config setup
         hcmode: e.g. "oil", "gas"
     """
 
     logger.debug("Processed config: \n%s", json.dumps(config, indent=4))
 
-    # this routine is dependent that the env variable FMU_GLOBAL_CONFIG is active
-    if "FMU_GLOBAL_CONFIG" not in os.environ:
-        raise RuntimeError("The env variable FMU_GLOBAL_CONFIG is not set.")
+    fmu_global_config = _get_global_config(config)
 
     mdata = config["metadata"]
 
@@ -100,7 +147,7 @@ def export_hc_map_dataio(surf, zname, date, hcmode, config):
         tt1 = date[0:8]
 
     if len(date) > 8:
-        tt2 = date[9:16]
+        tt2 = date[9:17]
 
     if tt1:
         tdata = [[tt1, "monitor"]]
@@ -111,17 +158,18 @@ def export_hc_map_dataio(surf, zname, date, hcmode, config):
     globaltag = globaltag + "_" if globaltag else ""
 
     edata = dataio.ExportData(
+        config=fmu_global_config,
         name=zname,
-        unit=unit,
         content={"property": {"attribute": attribute, "is_discrete": False}},
         vertical_domain={"depth": "msl"},
         timedata=tdata,
         is_prediction=True,
         is_observation=False,
+        unit=unit,
         tagname=globaltag + name,
         verbosity="WARNING",
         workflow="xtgeoapp-grd3dmaps script hc thickness maps",
     )
-    fname = edata.export(surf, unit=unit)
-    xtg.say(f"Outout as fmu-dataio: {fname}")
+    fname = edata.export(surf)
+    xtg.say(f"Output as fmu-dataio: {fname}")
     return fname

tests/test_scripts/test_grid3d_average_map_dataio1a.py

@@ -7,6 +7,7 @@
 import pytest
 import xtgeo
 import yaml
+
 from xtgeoapp_grd3dmaps.avghc import grid3d_average_map
 
 YAMLCONTENT = """
@@ -117,7 +118,7 @@ def test_average_map_1a_legacy(datatree):
 def test_average_map_dataio1a(datatree, avgdataio1aconfig):
     """Test AVG with YAML config example 3a piped through dataio"""
 
-    os.environ["FMU_GLOBAL_CONFIG"] = str(
+    os.environ["FMU_GLOBAL_CONFIG_GRD3DMAPS"] = str(
         datatree / "tests" / "data" / "reek" / "global_variables.yml"
     )
     grid3d_average_map.main(

tests/test_scripts/test_grid3d_hc_thickness_dataio1a.py

@@ -6,9 +6,10 @@
 
 import pytest
 import xtgeo
-import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness
 import yaml
 
+import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness
+
 YAMLCONTENT = """
 title: Reek
 #
@@ -69,7 +70,7 @@ def test_hc_thickness_1a_add2docs(hcdataio1aconfig):
 def test_hc_thickness_1a(datatree, hcdataio1aconfig):
     """Test HC thickness map piped through dataio"""
 
-    os.environ["FMU_GLOBAL_CONFIG"] = str(
+    os.environ["FMU_GLOBAL_CONFIG_GRD3DMAPS"] = str(
         datatree / "tests" / "data" / "reek" / "global_variables.yml"
     )
     grid3d_hc_thickness.main(

tests/test_scripts/test_grid3d_hc_thickness_dataio1b.py

@@ -6,9 +6,10 @@
 
 import pytest
 import xtgeo
-import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness
 import yaml
 
+import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness
+
 YAMLCONTENT = """
 title: Reek
 
@@ -68,7 +69,7 @@ def test_hc_thickness_1b_add2docs(hcdataio1bconfig):
 def test_hc_thickness_1b(datatree, hcdataio1bconfig):
     """Test HC thickness map piped through dataio, using 'both' mode"""
 
-    os.environ["FMU_GLOBAL_CONFIG"] = str(
+    os.environ["FMU_GLOBAL_CONFIG_GRD3DMAPS"] = str(
         datatree / "tests" / "data" / "reek" / "global_variables.yml"
     )
     grid3d_hc_thickness.main(

tests/test_scripts/test_grid3d_hc_thickness_dataio1c.py

@@ -6,8 +6,10 @@
 
 import pytest
 import xtgeo
-import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness
 import yaml
+from fmu.config import utilities as ut
+
+import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness
 
 YAMLCONTENT = """
 title: Reek
@@ -64,12 +66,24 @@ def test_hc_thickness_1c_add2docs(hcdataio1cconfig):
     assert hcdataio1cconfig is not None
 
 
-def test_hc_thickness_1c(datatree, hcdataio1cconfig):
+@pytest.mark.parametrize(
+    "variant",
+    ["inputconfig", "FMU_GLOBAL_CONFIG_GRD3DMAPS", "FMU_GLOBAL_CONFIG"],
+)
+def test_hc_thickness_1c(datatree, hcdataio1cconfig, variant):
     """Test HC thickness map piped through dataio, see former yaml hc_thickness2a."""
 
-    os.environ["FMU_GLOBAL_CONFIG"] = str(
-        datatree / "tests" / "data" / "reek" / "global_variables.yml"
-    )
+    if "FMU" in variant:
+        os.environ[variant] = str(
+            datatree / "tests" / "data" / "reek" / "global_variables.yml"
+        )
+    else:
+        cfg = ut.yaml_load(hcdataio1cconfig)
+        cfg["input"]["fmu_global_config"] = "tests/data/reek/global_variables.yml"
+
+        with open(hcdataio1cconfig, "w", encoding="utf-8") as outfile:
+            yaml.dump(cfg, outfile)
+
     grid3d_hc_thickness.main(
         ["--config", hcdataio1cconfig, "--dump", "dump_config.yml"]
     )