From cb95e14f2e4f237cceadc8ddfd9f363e61fdeba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20C=2E=20Riven=C3=A6s?= Date: Fri, 27 Jan 2023 08:57:48 +0100 Subject: [PATCH] Fix bug and improve dataio integration --- docs/usage.rst | 37 +++++++-- pyproject.toml | 12 +-- requirements/requirements.txt | 1 + setup.cfg | 3 + .../avghc/_export_via_fmudataio.py | 76 +++++++++++++++---- .../test_grid3d_average_map_dataio1a.py | 3 +- .../test_grid3d_hc_thickness_dataio1a.py | 5 +- .../test_grid3d_hc_thickness_dataio1b.py | 5 +- .../test_grid3d_hc_thickness_dataio1c.py | 24 ++++-- 9 files changed, 130 insertions(+), 36 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 1bd00f1..ef3c5ad 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -94,15 +94,34 @@ Note that dates and diffdates are two separate lists Using fmu-dataio for SUMO output -------------------------------- -From version 1.3, a ``fmu-dataio`` pipeline is supported and encouraged. This will make results -compatible with upload to SUMO. This involves some changes: +From version 1.3, a ``fmu-dataio`` pipeline is supported and encouraged. This will generate metadata +and make results compatible with upload to SUMO. This involves some changes: - * The environment variable ``FMU_GLOBAL_CONFIG`` shall be set in advance and this shall - point to a fmu-dataio compatible global variables file (on YAML format), for example: + * The location of the FMU ``global_variables`` file must be set, which can be done either by: - .. code-block:: bash + .. code-block:: yaml - export FMU_GLOBAL_CONFIG=fmuconfig/output/global_variables.yml + input: + fmu_global_config: some_path/global_variables.yml # relative to RUNPATH in ERT + + or by setting one of the following environment variables either in shell or inside ERT + (see e.g. ``setenv`` in ERT) + + .. code-block:: bash + + # bash + export FMU_GLOBAL_CONFIG_GRD3DMAPS=fmuconfig/output/global_variables.yml + export FMU_GLOBAL_CONFIG=fmuconfig/output/global_variables.yml + + .. code-block:: csh + + # csh + setenv FMU_GLOBAL_CONFIG_GRD3DMAPS fmuconfig/output/global_variables.yml + setenv FMU_GLOBAL_CONFIG fmuconfig/output/global_variables.yml + + The first environment variable will be dedicated to the AVG and HC scripts, while the last one + is a generic for ``fmu.dataio``. The priority is: first use the setting under ``input:`` in + the script, then ``FMU_GLOBAL_CONFIG_GRD3DMAPS`` and finally ``FMU_GLOBAL_CONFIG`` * The output.mapfolder must either: @@ -116,6 +135,7 @@ compatible with upload to SUMO. This involves some changes: .. code-block:: yaml input: + fmu_global_config: fmuconfig/output/global_variables.yml # relative to RUNPATH in ERT eclroot: tests/data/reek/REEK grid: $eclroot.EGRID @@ -127,3 +147,8 @@ compatible with upload to SUMO. This involves some changes: metadata: attribute: saturation unit: fraction + + ... + + output: + mapfolder: fmu-dataio diff --git a/pyproject.toml b/pyproject.toml index 597d289..db30b95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,10 +26,10 @@ exclude = ''' [build-system] requires = [ - "pip>=19.1.1", - "setuptools>=43", - "wheel", - "setuptools_scm", - "Sphinx", - "sphinx-rtd-theme", + "pip>=19.1.1", + "setuptools>=43", + "wheel", + "setuptools_scm", + "Sphinx", + "sphinx-rtd-theme", ] diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0d6a3e9..f8ca727 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,3 +3,4 @@ pyyaml xtgeo>=2.20.7 fmu-dataio>=1.3.0 xtgeoviz +fmu-config diff --git a/setup.cfg b/setup.cfg index aa4885c..0cdb8dd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,3 +11,6 @@ test = pytest [tool:pytest] addopts = --verbose -x + +[mypy] +ignore_missing_imports = True diff --git a/src/xtgeoapp_grd3dmaps/avghc/_export_via_fmudataio.py b/src/xtgeoapp_grd3dmaps/avghc/_export_via_fmudataio.py index 866567b..9497830 100644 --- a/src/xtgeoapp_grd3dmaps/avghc/_export_via_fmudataio.py +++ b/src/xtgeoapp_grd3dmaps/avghc/_export_via_fmudataio.py @@ -1,8 +1,10 @@ """General functions that exports maps / plots using fmu-dataio.""" import json import os +from pathlib import Path import fmu.dataio as dataio +from fmu.config import utilities as ut from xtgeo.common import XTGeoDialog xtg = XTGeoDialog() @@ -10,6 +12,51 @@ logger = xtg.functionlogger(__name__) +def _get_global_config(thisconfig): + """Get the global config in different manners. Priority: + + (1) A setting inside the setup file: input: fmu_global_config: will win if present + (2) FMU_GLOBAL_CONFIG_GRD3DMAPS exists as env variable. Will be second + (3) FMU_GLOBAL_CONFIG as env variabel! + """ + alternatives = [ + "[input][fmu_global_config] in setup file", + "FMU_GLOBAL_CONFIG_GRD3DMAPS", + "FMU_GLOBAL_CONFIG", + ] + + alt = [] + if "input" in thisconfig: + alt.append(thisconfig["input"].get("fmu_global_config")) + else: + alt.append(None) + + alt.append(os.environ.get("FMU_GLOBAL_CONFIG_GRD3DMAPS")) + alt.append(os.environ.get("FMU_GLOBAL_CONFIG")) + + cfg = None + + for altno, description in enumerate(alternatives): + alternative = alt[altno] + logger.info("Global %s", alternative) + if not alternative: + continue + + if Path(alternative).is_file(): + cfg = ut.yaml_load(alternative) + logger.info("Global no %s config from %s", alternative, description) + break + else: + raise IOError( + f"Config file does not exist: {alternative}, source is {description}" + ) + + if not cfg: + raise RuntimeError("Not able lo load the global config!") + + return cfg + + def export_avg_map_dataio(surf, nametuple, config): """Export avererage maps using dataio. @@ -17,19 +64,19 @@ def export_avg_map_dataio(surf, nametuple, config): surf: XTGeo RegularSurface object nametuple: On form ('myzone1', 'PRESSURE--19991201') where the last is an identifier (nameid) for the metadata config - config: The processed config setup + config: The processed config setup for this script (i.e. not global_config) """ zoneinfo, nameid = nametuple logger.debug("Processed config: \n%s", json.dumps(config, indent=4)) - # this routine is dependent that the env variable FMU_GLOBAL_CONFIG is active - if "FMU_GLOBAL_CONFIG" not in os.environ: - raise RuntimeError("The env variable FMU_GLOBAL_CONFIG is not set.") + fmu_global_config = _get_global_config(config) metadata = config["metadata"] if nameid not in metadata: - raise ValueError(f"Seems that medata for {nameid} is missing!") + raise ValueError( + f"Seems that 'metadata' for {nameid} is missing! Cf. documentation" + ) mdata = metadata[nameid] name = mdata.get("name", "unknown_name") @@ -37,6 +84,7 @@ def export_avg_map_dataio(surf, nametuple, config): unit = mdata.get("unit", None) tt1 = mdata.get("t1", None) tt2 = mdata.get("t2", None) + if tt1 and tt1 not in nameid: tt1 = None if tt2 and tt2 not in nameid: @@ -55,6 +103,7 @@ def export_avg_map_dataio(surf, nametuple, config): tdata = [[tt2, "base"]] edata = dataio.ExportData( + config=fmu_global_config, name=zoneinfo, unit=unit, content={"property": {"attribute": attribute, "is_discrete": False}}, @@ -66,7 +115,7 @@ def export_avg_map_dataio(surf, nametuple, config): verbosity="WARNING", workflow="xtgeoapp-grd3dmaps script average maps", ) - fname = edata.export(surf, unit=unit) + fname = edata.export(surf) xtg.say(f"Output as fmu-dataio: {fname}") return fname @@ -76,7 +125,7 @@ def export_hc_map_dataio(surf, zname, date, hcmode, config): Args: surf: XTGeo RegularSurface object - nametuple: The zone name + nametuple: The zone name. date: The date tag config: The processed config setup hcmode: e.g. "oil", "gas" @@ -84,9 +133,7 @@ def export_hc_map_dataio(surf, zname, date, hcmode, config): logger.debug("Processed config: \n%s", json.dumps(config, indent=4)) - # this routine is dependent that the env variable FMU_GLOBAL_CONFIG is active - if "FMU_GLOBAL_CONFIG" not in os.environ: - raise RuntimeError("The env variable FMU_GLOBAL_CONFIG is not set.") + fmu_global_config = _get_global_config(config) mdata = config["metadata"] @@ -100,7 +147,7 @@ def export_hc_map_dataio(surf, zname, date, hcmode, config): tt1 = date[0:8] if len(date) > 8: - tt2 = date[9:16] + tt2 = date[9:17] if tt1: tdata = [[tt1, "monitor"]] @@ -111,17 +158,18 @@ def export_hc_map_dataio(surf, zname, date, hcmode, config): globaltag = globaltag + "_" if globaltag else "" edata = dataio.ExportData( + config=fmu_global_config, name=zname, - unit=unit, content={"property": {"attribute": attribute, "is_discrete": False}}, vertical_domain={"depth": "msl"}, timedata=tdata, is_prediction=True, is_observation=False, + unit=unit, tagname=globaltag + name, verbosity="WARNING", workflow="xtgeoapp-grd3dmaps script hc thickness maps", ) - fname = edata.export(surf, unit=unit) - xtg.say(f"Outout as fmu-dataio: {fname}") + fname = edata.export(surf) + xtg.say(f"Output as fmu-dataio: {fname}") return fname diff --git a/tests/test_scripts/test_grid3d_average_map_dataio1a.py b/tests/test_scripts/test_grid3d_average_map_dataio1a.py index 8dc0f38..5391ce9 100644 --- a/tests/test_scripts/test_grid3d_average_map_dataio1a.py +++ b/tests/test_scripts/test_grid3d_average_map_dataio1a.py @@ -7,6 +7,7 @@ import pytest import xtgeo import yaml + from xtgeoapp_grd3dmaps.avghc import grid3d_average_map YAMLCONTENT = """ @@ -117,7 +118,7 @@ def test_average_map_1a_legacy(datatree): def test_average_map_dataio1a(datatree, avgdataio1aconfig): """Test AVG with YAML config example 3a piped through dataio""" - os.environ["FMU_GLOBAL_CONFIG"] = str( + os.environ["FMU_GLOBAL_CONFIG_GRD3DMAPS"] = str( datatree / "tests" / "data" / "reek" / "global_variables.yml" ) grid3d_average_map.main( diff --git a/tests/test_scripts/test_grid3d_hc_thickness_dataio1a.py b/tests/test_scripts/test_grid3d_hc_thickness_dataio1a.py index 4a82b08..2194b11 100644 --- a/tests/test_scripts/test_grid3d_hc_thickness_dataio1a.py +++ b/tests/test_scripts/test_grid3d_hc_thickness_dataio1a.py @@ -6,9 +6,10 @@ import pytest import xtgeo -import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness import yaml +import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness + YAMLCONTENT = """ title: Reek # @@ -69,7 +70,7 @@ def test_hc_thickness_1a_add2docs(hcdataio1aconfig): def test_hc_thickness_1a(datatree, hcdataio1aconfig): """Test HC thickness map piped through dataio""" - os.environ["FMU_GLOBAL_CONFIG"] = str( + os.environ["FMU_GLOBAL_CONFIG_GRD3DMAPS"] = str( datatree / "tests" / "data" / "reek" / "global_variables.yml" ) grid3d_hc_thickness.main( diff --git a/tests/test_scripts/test_grid3d_hc_thickness_dataio1b.py b/tests/test_scripts/test_grid3d_hc_thickness_dataio1b.py index 70027cc..3e77671 100644 --- a/tests/test_scripts/test_grid3d_hc_thickness_dataio1b.py +++ b/tests/test_scripts/test_grid3d_hc_thickness_dataio1b.py @@ -6,9 +6,10 @@ import pytest import xtgeo -import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness import yaml +import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness + YAMLCONTENT = """ title: Reek @@ -68,7 +69,7 @@ def test_hc_thickness_1b_add2docs(hcdataio1bconfig): def test_hc_thickness_1b(datatree, hcdataio1bconfig): """Test HC thickness map piped through dataio, using 'both' mode""" - os.environ["FMU_GLOBAL_CONFIG"] = str( + os.environ["FMU_GLOBAL_CONFIG_GRD3DMAPS"] = str( datatree / "tests" / "data" / "reek" / "global_variables.yml" ) grid3d_hc_thickness.main( diff --git a/tests/test_scripts/test_grid3d_hc_thickness_dataio1c.py b/tests/test_scripts/test_grid3d_hc_thickness_dataio1c.py index 912ce1f..be8da38 100644 --- a/tests/test_scripts/test_grid3d_hc_thickness_dataio1c.py +++ b/tests/test_scripts/test_grid3d_hc_thickness_dataio1c.py @@ -6,8 +6,10 @@ import pytest import xtgeo -import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness import yaml +from fmu.config import utilities as ut + +import xtgeoapp_grd3dmaps.avghc.grid3d_hc_thickness as grid3d_hc_thickness YAMLCONTENT = """ title: Reek @@ -64,12 +66,24 @@ def test_hc_thickness_1c_add2docs(hcdataio1cconfig): assert hcdataio1cconfig is not None -def test_hc_thickness_1c(datatree, hcdataio1cconfig): +@pytest.mark.parametrize( + "variant", + ["inputconfig", "FMU_GLOBAL_CONFIG_GRD3DMAPS", "FMU_GLOBAL_CONFIG"], +) +def test_hc_thickness_1c(datatree, hcdataio1cconfig, variant): """Test HC thickness map piped through dataio, see former yaml hc_thickness2a.""" - os.environ["FMU_GLOBAL_CONFIG"] = str( - datatree / "tests" / "data" / "reek" / "global_variables.yml" - ) + if "FMU" in variant: + os.environ[variant] = str( + datatree / "tests" / "data" / "reek" / "global_variables.yml" + ) + else: + cfg = ut.yaml_load(hcdataio1cconfig) + cfg["input"]["fmu_global_config"] = "tests/data/reek/global_variables.yml" + + with open(hcdataio1cconfig, "w", encoding="utf-8") as outfile: + yaml.dump(cfg, outfile) + grid3d_hc_thickness.main( ["--config", hcdataio1cconfig, "--dump", "dump_config.yml"] )