diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 45718f9c..e5e9b9e2 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -69,25 +69,20 @@ repos:
files: ^backend/
# Frontend: Biome format + lint
+ # Both hooks use the versions pinned in frontend/package.json, not whatever npx resolves.
- repo: local
hooks:
- - id: biome-format
- name: Biome format (write)
- entry: bash -lc "cd frontend && npx --yes @biomejs/biome check --write"
+ - id: biome-check
+ name: Biome check (write)
+ entry: bash -c "cd frontend && exec ./node_modules/.bin/biome check --write"
language: system
- files: ^frontend/.*\.(ts|tsx|js|jsx|json|md|css)$
- pass_filenames: true
- - id: biome-lint
- name: Biome lint
- entry: bash -lc "cd frontend && npx --yes @biomejs/biome check"
- language: system
- files: ^frontend/.*\.(ts|tsx|js|jsx)$
- pass_filenames: true
+ files: ^frontend/.*\.(ts|tsx|js|jsx|json|jsonc|md|css)$
+ pass_filenames: false
# Frontend: TypeScript typecheck
- id: typescript-typecheck
name: TypeScript typecheck
- entry: bash -lc "cd frontend && npx --yes tsc -b --noEmit"
+ entry: bash -c "cd frontend && exec ./node_modules/.bin/tsc -b --noEmit"
language: system
pass_filenames: false
files: ^frontend/
diff --git a/backend/src/ref_backend/core/diagnostic_metadata.py b/backend/src/ref_backend/core/diagnostic_metadata.py
index d197bc88..1e881ae8 100644
--- a/backend/src/ref_backend/core/diagnostic_metadata.py
+++ b/backend/src/ref_backend/core/diagnostic_metadata.py
@@ -17,6 +17,21 @@
from loguru import logger
from pydantic import BaseModel, Field
+ReferenceDatasetSourceType = Literal[
+ "obs4mips",
+ "obs4ref",
+ "pmp-climatology",
+ "esmvaltool-reference",
+ "ilamb",
+ "recipe",
+]
+"""
+The source type of the data actually supplied for a reference dataset.
+
+A diagnostic can require ``obs4mips`` and be served the REF's own ``obs4ref`` copy, so what is
+supplied has to be recorded separately from what the requirement asks for.
+"""
+
class ReferenceDatasetLink(BaseModel):
"""
@@ -26,10 +41,30 @@ class ReferenceDatasetLink(BaseModel):
compare model outputs against. They can be classified by their role in the analysis.
"""
- slug: str = Field(..., description="Unique identifier for the dataset(e.g., 'obs4mips.CERES-EBAF.v4.2')")
+ slug: str = Field(
+ ...,
+ description=(
+ "Unique identifier for the dataset, `.`. The prefix names who "
+ "supplies the data rather than the source type the requirement asks for, so a dataset "
+ "required as obs4mips but supplied from obs4REF reads 'obs4ref.HadISST-1-1'"
+ ),
+ )
description: str | None = Field(
None, description="Description of how this dataset is used in the diagnostic"
)
+ source_type: ReferenceDatasetSourceType | None = Field(
+ None,
+ description=(
+ "The source type of the data actually supplied, which is not always the one the "
+ "requirement asks for:\n"
+ "- 'obs4mips': published on ESGF obs4MIPs\n"
+ "- 'obs4ref': reference data served by the REF, not yet published on obs4MIPs\n"
+ "- 'pmp-climatology': the PMP climatology registry\n"
+ "- 'esmvaltool-reference': the ESMValTool reference registry\n"
+ "- 'ilamb': the ILAMB registry\n"
+ "- 'recipe': named inside the provider's recipe rather than ingested by the REF"
+ ),
+ )
type: Literal["primary", "secondary", "comparison"] = Field(
...,
description=(
@@ -121,7 +156,8 @@ def load_diagnostic_metadata(path: Path) -> dict[str, DiagnosticMetadata]:
pmp/annual-cycle:
reference_datasets:
- - slug: "obs4mips.CERES-EBAF.v4.2"
+ - slug: "obs4mips.CERES-EBAF-4-2-1"
+ source_type: "obs4mips"
description: "CERES Energy Balanced and Filled"
type: "primary"
display_name: "Annual Cycle Analysis"
diff --git a/backend/static/diagnostics/esmvaltool.yaml b/backend/static/diagnostics/esmvaltool.yaml
index 24841cc6..edbec4b0 100644
--- a/backend/static/diagnostics/esmvaltool.yaml
+++ b/backend/static/diagnostics/esmvaltool.yaml
@@ -1,18 +1,17 @@
# ESMValTool Diagnostic Metadata
#
# Reference datasets and display metadata for ESMValTool diagnostics.
+#
+# `slug` is `.`, and `source_type` says the same thing in a structured field.
+# The prefix names who supplies the data, not the source type the requirement asks for, so a
+# dataset required as `obs4mips` but supplied from obs4REF reads `obs4ref.HadISST-1-1`.
esmvaltool/cloud-radiative-effects:
reference_datasets:
- - slug: "esmvaltool.CERES-EBAF"
- description: "CERES Energy Balanced and Filled - TOA radiation fluxes (rlut, rlutcs, rsut, rsutcs) (planned to move to obs4MIPs)"
+ - slug: "obs4mips.CERES-EBAF-4-2-1"
+ source_type: "obs4mips"
+ description: "CERES Energy Balanced and Filled v4.2.1. Supplies rlut, rlutcs, rsut and rsutcs."
type: "primary"
- - slug: "esmvaltool.ESACCI-CLOUD"
- description: "ESA CCI Cloud properties (planned to move to obs4MIPs)"
- type: "secondary"
- - slug: "esmvaltool.ISCCP-FH"
- description: "ISCCP cloud properties (planned to move to obs4MIPs)"
- type: "secondary"
display_name: "Cloud Radiative Effects"
description: >-
Plot climatologies and zonal mean profiles of cloud radiative effects
@@ -21,11 +20,9 @@ esmvaltool/cloud-radiative-effects:
esmvaltool/cloud-scatterplots-cli-ta:
reference_datasets:
- - slug: "obs4mips.ERA-5"
- description: "ERA5 - Air temperature (ta)"
- type: "primary"
- slug: "esmvaltool.CALIPSO-ICECLOUD"
- description: "CALIPSO - Ice cloud properties (cli), baked into recipe"
+ source_type: "recipe"
+ description: "CALIPSO ice cloud properties. Supplies cli, named in the recipe."
type: "secondary"
display_name: "Cloud-Temperature Scatterplots (cli vs ta)"
description: >-
@@ -37,10 +34,12 @@ esmvaltool/cloud-scatterplots-cli-ta:
esmvaltool/cloud-scatterplots-clivi-lwcre:
reference_datasets:
- slug: "esmvaltool.CERES-EBAF"
- description: "CERES Energy Balanced and Filled - Longwave radiation (rlut, rlutcs; lwcre derived), baked into recipe"
+ source_type: "recipe"
+ description: "CERES Energy Balanced and Filled. Supplies rlut and rlutcs (lwcre derived), named in the recipe."
type: "primary"
- slug: "esmvaltool.ESACCI-CLOUD"
- description: "ESA CCI Cloud properties - Ice water path (clivi), baked into recipe"
+ source_type: "recipe"
+ description: "ESA CCI cloud properties. Supplies clivi, named in the recipe."
type: "secondary"
display_name: "Cloud-Radiation Scatterplots (clivi vs lwcre)"
description: "Scatterplot of ice water path (clivi) vs longwave cloud radiative effect (lwcre)."
@@ -49,10 +48,12 @@ esmvaltool/cloud-scatterplots-clivi-lwcre:
esmvaltool/cloud-scatterplots-clt-swcre:
reference_datasets:
- slug: "esmvaltool.CERES-EBAF"
- description: "CERES Energy Balanced and Filled - Shortwave radiation (rsut, rsutcs; swcre derived), baked into recipe"
+ source_type: "recipe"
+ description: "CERES Energy Balanced and Filled. Supplies rsut and rsutcs (swcre derived), named in the recipe."
type: "primary"
- slug: "esmvaltool.ESACCI-CLOUD"
- description: "ESA CCI Cloud properties - Total cloud fraction (clt), baked into recipe"
+ source_type: "recipe"
+ description: "ESA CCI cloud properties. Supplies clt, named in the recipe."
type: "secondary"
display_name: "Cloud-Radiation Scatterplots (clt vs swcre)"
description: "Scatterplot of total cloud fraction (clt) vs shortwave cloud radiative effect (swcre)."
@@ -60,14 +61,17 @@ esmvaltool/cloud-scatterplots-clt-swcre:
esmvaltool/cloud-scatterplots-clwvi-pr:
reference_datasets:
- - slug: "obs4MIPs.GPCPv2.3"
- description: "GPCP v2.3 - Precipitation (pr)"
+ - slug: "esmvaltool.GPCP-V2.3"
+ source_type: "recipe"
+ description: "GPCP v2.3. Supplies pr, named in the recipe."
type: "secondary"
- slug: "esmvaltool.ESACCI-CLOUD"
- description: "ESA CCI Cloud properties (clwvi, planned to move to obs4MIPs)"
+ source_type: "recipe"
+ description: "ESA CCI cloud properties. Supplies clwvi, named in the recipe."
type: "secondary"
- slug: "esmvaltool.CERES-EBAF"
- description: "CERES Energy Balanced and Filled - Radiation, baked into recipe"
+ source_type: "recipe"
+ description: "CERES Energy Balanced and Filled. Supplies the radiation fields, named in the recipe."
type: "primary"
display_name: "Cloud-Precipitation Scatterplots (clwvi vs pr)"
description: "Scatterplot of condensed water path (clwvi) vs precipitation (pr)."
@@ -76,19 +80,24 @@ esmvaltool/cloud-scatterplots-clwvi-pr:
esmvaltool/cloud-scatterplots-reference:
reference_datasets:
- slug: "obs4mips.ERA-5"
- description: "ERA5 - Air temperature (ta), ingested via REF"
+ source_type: "obs4mips"
+ description: "ERA5 reanalysis. Supplies ta over 2007 to 2014."
type: "primary"
- slug: "esmvaltool.CERES-EBAF"
- description: "CERES Energy Balanced and Filled - Radiation, baked into recipe"
+ source_type: "recipe"
+ description: "CERES Energy Balanced and Filled. Supplies the radiation fields, named in the recipe."
type: "primary"
- slug: "esmvaltool.CALIPSO-ICECLOUD"
- description: "CALIPSO - Ice cloud properties (cli), baked into recipe"
+ source_type: "recipe"
+ description: "CALIPSO ice cloud properties. Supplies cli, named in the recipe."
type: "secondary"
- slug: "esmvaltool.ESACCI-CLOUD"
- description: "ESA CCI Cloud properties (clt, clwvi, clivi), baked into recipe"
+ source_type: "recipe"
+ description: "ESA CCI cloud properties. Supplies clt, clwvi and clivi, named in the recipe."
type: "secondary"
- slug: "esmvaltool.GPCP-V2.3"
- description: "GPCP v2.3 - Precipitation (pr), hardcoded in update_recipe"
+ source_type: "recipe"
+ description: "GPCP v2.3. Supplies pr, named in the recipe because it cannot be ingested (climate-ref issue 260)."
type: "secondary"
display_name: "Cloud Scatterplots for Reference dataset"
description: >-
@@ -99,10 +108,12 @@ esmvaltool/cloud-scatterplots-reference:
esmvaltool/enso-basic-climatology:
reference_datasets:
- slug: "esmvaltool.GPCP-V2.3"
- description: "GPCP v2.3 - Precipitation (pr), baked into recipe"
+ source_type: "recipe"
+ description: "GPCP v2.3. Supplies pr, named in the recipe."
type: "secondary"
- slug: "esmvaltool.TROPFLUX"
- description: "TROPFLUX (OBS6) - Sea surface temperature (tos) and wind stress (tauu), baked into recipe"
+ source_type: "recipe"
+ description: "TropFlux. Supplies tos and tauu, named in the recipe."
type: "secondary"
display_name: "ENSO Basic Climatology"
description: >-
@@ -113,7 +124,8 @@ esmvaltool/enso-basic-climatology:
esmvaltool/enso-characteristics:
reference_datasets:
- slug: "esmvaltool.TROPFLUX"
- description: "TROPFLUX (OBS6) - Sea surface temperature (tos), hardcoded in update_recipe"
+ source_type: "recipe"
+ description: "TropFlux. Supplies tos, named in the recipe."
type: "primary"
display_name: "ENSO Characteristics"
description: >-
@@ -121,6 +133,54 @@ esmvaltool/enso-characteristics:
evaluating the amplitude, frequency, and spatial structure of ENSO events.
tags: ["ocean", "enso", "variability"]
+esmvaltool/ozone-annual-cycle:
+ reference_datasets:
+ - slug: "obs4mips.C3S-GTO-ECV-9-0"
+ source_type: "obs4mips"
+ description: "C3S GTO-ECV total column ozone v9.0. Supplies toz."
+ type: "primary"
+ display_name: "Ozone Annual Cycle"
+ description: "Plot the zonal mean total column ozone (toz) against the annual cycle."
+ tags: ["atmosphere", "ozone", "seasonal-cycle"]
+
+esmvaltool/ozone-lat-time:
+ reference_datasets:
+ - slug: "obs4mips.C3S-GTO-ECV-9-0"
+ source_type: "obs4mips"
+ description: "C3S GTO-ECV total column ozone v9.0. Supplies toz."
+ type: "primary"
+ display_name: "Ozone Latitude vs Time"
+ description: "Plot the zonal mean total column ozone (toz) against time."
+ tags: ["atmosphere", "ozone", "timeseries"]
+
+esmvaltool/ozone-nh-mar:
+ reference_datasets:
+ - slug: "obs4mips.C3S-GTO-ECV-9-0"
+ source_type: "obs4mips"
+ description: "C3S GTO-ECV total column ozone v9.0. Supplies toz."
+ type: "primary"
+ display_name: "Ozone Northern Polar Cap (March)"
+ description: "Plot the March Northern Hemisphere polar mean (60N to 80N) total column ozone (toz) time series."
+ tags: ["atmosphere", "ozone", "polar"]
+
+esmvaltool/ozone-sh-oct:
+ reference_datasets:
+ - slug: "obs4mips.C3S-GTO-ECV-9-0"
+ source_type: "obs4mips"
+ description: "C3S GTO-ECV total column ozone v9.0. Supplies toz."
+ type: "primary"
+ display_name: "Ozone Southern Polar Cap (October)"
+ description: "Plot the October Southern Hemisphere polar mean (60S to 85S) total column ozone (toz) time series."
+ tags: ["atmosphere", "ozone", "polar"]
+
+esmvaltool/ozone-zonal:
+ reference_datasets: []
+ display_name: "Ozone Zonal Mean Profiles"
+ description: >-
+ Plot the stratospheric zonal mean ozone (o3) profiles. No reference dataset is used yet,
+ because ESACCI-OZONE (SAGE-OMPS) is not on obs4MIPs.
+ tags: ["atmosphere", "ozone", "stratosphere"]
+
esmvaltool/equilibrium-climate-sensitivity:
reference_datasets: []
display_name: "Equilibrium Climate Sensitivity (ECS)"
@@ -161,16 +221,20 @@ esmvaltool/zero-emission-commitment:
esmvaltool/regional-historical-annual-cycle:
reference_datasets:
- slug: "obs4mips.ERA-5"
- description: "ERA-5 - Sea level pressure (psl) and zonal wind (ua), ingested via REF"
+ source_type: "obs4mips"
+ description: "ERA5 reanalysis. Supplies psl and ua."
type: "primary"
- slug: "esmvaltool.HadCRUT5"
- description: "HadCRUT5 v5.0.1.0-analysis - Near-surface air temperature (tas), hardcoded in recipe"
+ source_type: "recipe"
+ description: "HadCRUT5 v5.0.1.0 analysis. Supplies tas, named in the recipe."
type: "secondary"
- slug: "esmvaltool.GPCP-V2.3"
- description: "GPCP v2.3 - Precipitation (pr), hardcoded in recipe (cannot be ingested, issue #260)"
+ source_type: "recipe"
+ description: "GPCP v2.3. Supplies pr, named in the recipe because it cannot be ingested (climate-ref issue 260)."
type: "secondary"
- slug: "esmvaltool.ERA5-native6"
- description: "ERA5 native6 - Specific humidity (hus), not yet on obs4MIPs"
+ source_type: "recipe"
+ description: "ERA5 native6. Supplies hus, which is not on obs4MIPs."
type: "secondary"
display_name: "Regional Historical Annual Cycle"
description: >-
@@ -183,16 +247,20 @@ esmvaltool/regional-historical-annual-cycle:
esmvaltool/regional-historical-timeseries:
reference_datasets:
- slug: "obs4mips.ERA-5"
- description: "ERA-5 - Sea level pressure (psl) and zonal wind (ua), ingested via REF"
+ source_type: "obs4mips"
+ description: "ERA5 reanalysis. Supplies psl and ua."
type: "primary"
- slug: "esmvaltool.HadCRUT5"
- description: "HadCRUT5 v5.0.1.0-analysis - Near-surface air temperature (tas), hardcoded in recipe"
+ source_type: "recipe"
+ description: "HadCRUT5 v5.0.1.0 analysis. Supplies tas, named in the recipe."
type: "secondary"
- slug: "esmvaltool.GPCP-V2.3"
- description: "GPCP v2.3 - Precipitation (pr), hardcoded in recipe (cannot be ingested, issue #260)"
+ source_type: "recipe"
+ description: "GPCP v2.3. Supplies pr, named in the recipe because it cannot be ingested (climate-ref issue 260)."
type: "secondary"
- slug: "esmvaltool.ERA5-native6"
- description: "ERA5 native6 - Specific humidity (hus), not yet on obs4MIPs"
+ source_type: "recipe"
+ description: "ERA5 native6. Supplies hus, which is not on obs4MIPs."
type: "secondary"
display_name: "Regional Historical Timeseries"
description: >-
@@ -205,16 +273,20 @@ esmvaltool/regional-historical-timeseries:
esmvaltool/regional-historical-trend:
reference_datasets:
- slug: "obs4mips.ERA-5"
- description: "ERA-5 - Sea level pressure (psl), near-surface air temperature (tas), and zonal wind (ua), ingested via REF"
+ source_type: "obs4mips"
+ description: "ERA5 reanalysis. Supplies psl, tas and ua."
type: "primary"
- slug: "esmvaltool.HadCRUT5"
- description: "HadCRUT5 v5.0.1.0-analysis - Near-surface air temperature (tas), hardcoded in recipe"
+ source_type: "recipe"
+ description: "HadCRUT5 v5.0.1.0 analysis. Supplies tas, named in the recipe."
type: "secondary"
- slug: "esmvaltool.GPCP-V2.3"
- description: "GPCP v2.3 - Precipitation (pr), hardcoded in recipe (cannot be ingested, issue #260)"
+ source_type: "recipe"
+ description: "GPCP v2.3. Supplies pr, named in the recipe because it cannot be ingested (climate-ref issue 260)."
type: "secondary"
- slug: "esmvaltool.ERA5-native6"
- description: "ERA5 native6 - Specific humidity (hus) and precipitation (pr), hardcoded in recipe"
+ source_type: "recipe"
+ description: "ERA5 native6. Supplies hus and pr, named in the recipe."
type: "secondary"
display_name: "Regional Historical Trends"
description: >-
@@ -226,11 +298,13 @@ esmvaltool/regional-historical-trend:
esmvaltool/sea-ice-area-basic:
reference_datasets:
- - slug: "esmvaltool.OSI-450-nh"
- description: "OSI-450 v3 - Sea ice concentration (siconc), Northern Hemisphere, hardcoded in recipe"
+ - slug: "esmvaltool-reference.OSI-450-nh"
+ source_type: "esmvaltool-reference"
+ description: "OSI-450 v3 Northern Hemisphere. Supplies sic and areacello."
type: "primary"
- - slug: "esmvaltool.OSI-450-sh"
- description: "OSI-450 v3 - Sea ice concentration (siconc), Southern Hemisphere, hardcoded in recipe"
+ - slug: "esmvaltool-reference.OSI-450-sh"
+ source_type: "esmvaltool-reference"
+ description: "OSI-450 v3 Southern Hemisphere. Supplies sic and areacello."
type: "primary"
display_name: "Arctic and Antarctic Sea Ice Area Seasonal Cycle"
description: >-
diff --git a/backend/static/diagnostics/ilamb.yaml b/backend/static/diagnostics/ilamb.yaml
index 3714288b..a65e65f7 100644
--- a/backend/static/diagnostics/ilamb.yaml
+++ b/backend/static/diagnostics/ilamb.yaml
@@ -2,19 +2,26 @@
#
# Reference datasets and display metadata for ILAMB (International Land Model
# Benchmarking) and IOMB (International Ocean Model Benchmarking) diagnostics.
+#
+# `slug` is `.`, and `source_type` says the same thing in a structured field.
+# The prefix names who supplies the data, not the source type the requirement asks for, so a
+# dataset required as `obs4mips` but supplied from obs4REF reads `obs4ref.HadISST-1-1`.
# Land diagnostics
ilamb/gpp-wecann:
reference_datasets:
- - slug: "obs4REF.WECANN"
- description: "WECANN GPP dataset - Gross primary productivity (gpp)"
+ - slug: "obs4ref.WECANN-1-0"
+ source_type: "obs4ref"
+ description: "WECANN v1.0. Supplies gpp."
type: "primary"
- - slug: "obs4REF.GPCPv2.3"
- description: "GPCP v2.3 - Precipitation (pr), relationship dataset"
+ - slug: "obs4ref.GPCP-2-3"
+ source_type: "obs4ref"
+ description: "GPCP v2.3. Supplies pr for the relationship analysis."
type: "secondary"
- slug: "ilamb.CRU4.02"
- description: "CRU TS v4.02 - Near-surface air temperature (tas), relationship dataset"
+ source_type: "ilamb"
+ description: "CRU TS v4.02. Supplies tas for the relationship analysis."
type: "secondary"
display_name: "Gross Primary Productivity (WECANN)"
description: "Apply the standard ILAMB analysis of gross primary productivity (gpp) with respect to the WECANN reference dataset."
@@ -22,8 +29,9 @@ ilamb/gpp-wecann:
ilamb/gpp-fluxnet2015:
reference_datasets:
- - slug: "obs4REF.FLUXNET2015"
- description: "FLUXNET2015 - Tower-based GPP measurements"
+ - slug: "obs4ref.FLUXNET2015-1-0"
+ source_type: "obs4ref"
+ description: "FLUXNET2015 v1.0 tower sites. Supplies gpp, referenced by registry key rather than ingested facets."
type: "primary"
display_name: "Gross Primary Productivity (FLUXNET2015)"
description: "Apply the standard ILAMB analysis of gross primary productivity (gpp) with respect to the FLUXNET2015 tower-based reference dataset."
@@ -31,8 +39,9 @@ ilamb/gpp-fluxnet2015:
ilamb/mrro-lora:
reference_datasets:
- - slug: "obs4REF.LORA"
- description: "LORA dataset - Runoff observations"
+ - slug: "obs4ref.LORA-1-0"
+ source_type: "obs4ref"
+ description: "LORA v1.0. Supplies mrro."
type: "primary"
display_name: "Runoff (LORA)"
description: "Apply the standard ILAMB analysis of total runoff (mrro) with respect to the LORA reference dataset."
@@ -40,8 +49,9 @@ ilamb/mrro-lora:
ilamb/mrsos-wangmao:
reference_datasets:
- - slug: "obs4REF.WangMao"
- description: "Wang-Mao dataset - Soil moisture (mrsos)"
+ - slug: "ilamb.WangMao"
+ source_type: "ilamb"
+ description: "Wang and Mao soil moisture. Supplies mrsol, converted to a surface volumetric fraction."
type: "primary"
display_name: "Surface Soil Moisture (WangMao)"
description: "Apply the standard ILAMB analysis of moisture in upper portion of soil column (mrsos) with respect to the WangMao reference dataset."
@@ -49,8 +59,9 @@ ilamb/mrsos-wangmao:
ilamb/csoil-hwsd2:
reference_datasets:
- - slug: "obs4REF.HWSD2"
- description: "Harmonized World Soil Database v2 - Soil carbon content"
+ - slug: "obs4ref.HWSD-2-0"
+ source_type: "obs4ref"
+ description: "Harmonized World Soil Database v2.0. Supplies cSoil."
type: "primary"
display_name: "Soil Carbon (HWSD2)"
description: "Apply the standard ILAMB analysis of carbon mass in soil pool (cSoil) with respect to the HWSD2 reference dataset."
@@ -58,8 +69,9 @@ ilamb/csoil-hwsd2:
ilamb/lai-avh15c1:
reference_datasets:
- - slug: "obs4REF.AVH15C1"
- description: "AVHRR - Leaf area index observations"
+ - slug: "obs4mips.NOAA-NCEI-LAI-AVHRR-5-0"
+ source_type: "obs4mips"
+ description: "NOAA NCEI AVHRR leaf area index v5.0. Supplies lai, coarsened to a common grid before comparison."
type: "primary"
display_name: "Leaf Area Index (AVH15C1)"
description: "Apply the standard ILAMB analysis of leaf area index (lai) with respect to the AVH15C1 (AVHRR) reference dataset."
@@ -67,8 +79,9 @@ ilamb/lai-avh15c1:
ilamb/nbp-hoffman:
reference_datasets:
- - slug: "obs4REF.Hoffman"
- description: "Hoffman dataset - Net biome productivity"
+ - slug: "obs4ref.Hoffman-1-0"
+ source_type: "obs4ref"
+ description: "Hoffman v1.0. Supplies nbp as a global mean."
type: "primary"
display_name: "Net Biome Productivity (Hoffman)"
description: "Apply the standard ILAMB analysis of net biome productivity (nbp) with respect to the Hoffman reference dataset."
@@ -76,8 +89,9 @@ ilamb/nbp-hoffman:
ilamb/snc-esacci:
reference_datasets:
- - slug: "obs4REF.CCI-CryoClim-FSC"
- description: "ESA CCI Snow - Snow cover fraction"
+ - slug: "obs4ref.CCI-CryoClim-FSC-1"
+ source_type: "obs4ref"
+ description: "ESA CCI CryoClim fractional snow cover v1. Supplies snc."
type: "primary"
display_name: "Snow Cover (ESA CCI)"
description: "Apply the standard ILAMB analysis of snow cover fraction (snc) with respect to the ESA CCI CryoClim fractional snow cover reference dataset."
@@ -85,20 +99,23 @@ ilamb/snc-esacci:
ilamb/burntfractionall-gfed:
reference_datasets:
- - slug: "obs4REF.GFED"
- description: "Global Fire Emissions Database - Burnt area fraction"
+ - slug: "obs4ref.GFED-5-0"
+ source_type: "obs4ref"
+ description: "Global Fire Emissions Database v5.0. Supplies burntFractionAll."
type: "primary"
display_name: "Burnt Fraction (GFED)"
description: "Apply the standard ILAMB analysis of burnt area fraction (burntFractionAll) with respect to the GFED reference dataset."
tags: ["land", "fire", "disturbance"]
-ilamb/emp-gleamgpcp:
+ilamb/emp-gleamgpcp2.3:
reference_datasets:
- - slug: "obs4REF.GLEAMv3.3a"
- description: "GLEAM v3.3a - Evapotranspiration (et)"
+ - slug: "ilamb.GLEAMv3.3a"
+ source_type: "ilamb"
+ description: "GLEAM v3.3a. Supplies evapotranspiration (et)."
type: "primary"
- - slug: "obs4REF.GPCP-2-3"
- description: "GPCP v2.3 (obs4REF) - Precipitation (pr)"
+ - slug: "obs4ref.GPCP-2-3"
+ source_type: "obs4ref"
+ description: "GPCP v2.3. Supplies pr."
type: "primary"
display_name: "Evaporation minus Precipitation (GLEAM/GPCP)"
description: "Apply the standard ILAMB analysis of evaporation minus precipitation (E-P) using GLEAMv3.3a evapotranspiration and GPCP v2.3 precipitation reference datasets."
@@ -108,8 +125,9 @@ ilamb/emp-gleamgpcp:
ilamb/thetao-woa2023-surface:
reference_datasets:
- - slug: "obs4REF.WOA2023"
- description: "World Ocean Atlas 2023 - Sea water potential temperature (thetao, surface, mapped to tos)"
+ - slug: "obs4ref.WOA-23"
+ source_type: "obs4ref"
+ description: "World Ocean Atlas 2023 monthly climatology (2005 to 2022). Supplies thetao, taken at the surface."
type: "primary"
display_name: "Sea Water Temperature (WOA2023 Surface)"
description: "Apply the standard ILAMB analysis of sea water potential temperature (thetao) at the surface with respect to the WOA2023 reference dataset."
@@ -117,8 +135,9 @@ ilamb/thetao-woa2023-surface:
ilamb/so-woa2023-surface:
reference_datasets:
- - slug: "obs4REF.WOA2023"
- description: "World Ocean Atlas 2023 - Sea water salinity (so, surface)"
+ - slug: "obs4ref.WOA-23"
+ source_type: "obs4ref"
+ description: "World Ocean Atlas 2023 monthly climatology (2005 to 2022). Supplies so, taken at the surface."
type: "primary"
display_name: "Sea Water Salinity (WOA2023 Surface)"
description: "Apply the standard ILAMB analysis of sea water salinity (so) at the surface with respect to the WOA2023 reference dataset."
@@ -126,8 +145,9 @@ ilamb/so-woa2023-surface:
ilamb/amoc-rapid:
reference_datasets:
- - slug: "obs4REF.RAPID"
- description: "RAPID array - Atlantic Meridional Overturning Circulation (amoc, from msftmz)"
+ - slug: "obs4ref.RAPID-2023-1a"
+ source_type: "obs4ref"
+ description: "RAPID array 2023 release 1a. Supplies msftmz, transformed into the RAPID AMOC index."
type: "primary"
display_name: "Atlantic Meridional Overturning Circulation (RAPID)"
description: "Apply the standard ILAMB analysis of the Atlantic Meridional Overturning Circulation (AMOC) with respect to the RAPID array reference dataset."
diff --git a/backend/static/diagnostics/pmp.yaml b/backend/static/diagnostics/pmp.yaml
index 0af94010..46ef9145 100644
--- a/backend/static/diagnostics/pmp.yaml
+++ b/backend/static/diagnostics/pmp.yaml
@@ -1,17 +1,24 @@
# PMP Diagnostic Metadata
#
# Reference datasets and display metadata for PMP (PCMDI Metrics Package) diagnostics.
+#
+# `slug` is `.`, and `source_type` says the same thing in a structured field.
+# The prefix names who supplies the data, not the source type the requirement asks for, so a
+# dataset required as `obs4mips` but supplied from obs4REF reads `obs4ref.HadISST-1-1`.
pmp/annual-cycle:
reference_datasets:
- - slug: "obs4MIPs-climatology.ERA-5"
- description: "ERA5 Reanalysis - Primary climatology for surface temperature (ts), winds (uas, vas), sea level pressure (psl), air temperature (ta), zonal/meridional wind (ua, va), and geopotential height (zg). Period: 1981-2004."
+ - slug: "pmp-climatology.ERA-5"
+ source_type: "pmp-climatology"
+ description: "ERA5 reanalysis climatology. Supplies ts, uas, vas, psl, ta, ua, va and zg."
type: "primary"
- - slug: "obs4MIPs-climatology.GPCP-Monthly-3-2"
- description: "Global Precipitation Climatology Project v3.2 - Precipitation reference (pr). Period: 1983-2004."
+ - slug: "pmp-climatology.GPCP-3-3"
+ source_type: "pmp-climatology"
+ description: "Global Precipitation Climatology Project v3.3 climatology. Supplies pr."
type: "primary"
- - slug: "obs4MIPs-climatology.CERES-EBAF-4-2"
- description: "CERES Energy Balanced and Filled v4.2 - Radiation fields (rlds, rlus, rlut, rsds, rsdt, rsus, rsut) and clear-sky variants (rldscs, rsdscs, rlutcs, rsutcs, rsuscs, rltcre, rstcre, rt). Period: 2001-2004."
+ - slug: "pmp-climatology.CERES-EBAF-4-2"
+ source_type: "pmp-climatology"
+ description: "CERES Energy Balanced and Filled v4.2 climatology. Supplies rlds, rlus, rlut, rsds, rsdt, rsus and rsut."
type: "primary"
display_name: "Annual Cycle Analysis"
description: >-
@@ -21,33 +28,19 @@ pmp/annual-cycle:
(rlds, rlus, rsds, rsdt, rsus, rlut, rsut) including clear-sky variants.
tags: ["atmosphere", "seasonal-cycle", "climatology"]
-pmp/enso_perf:
- reference_datasets:
- - slug: "obs4mips.GPCP-Monthly-3-2"
- description: "Global Precipitation Climatology Project v3.2 - Precipitation (pr)"
- type: "primary"
- - slug: "obs4mips.TropFlux-1-0"
- description: "TropFlux v1.0 - Wind stress (tauu)"
- type: "primary"
- - slug: "obs4mips.HadISST-1-1"
- description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1 - Sea surface temperature (ts)"
- type: "primary"
- display_name: "ENSO Performance Metrics"
- description: >-
- Calculate the ENSO performance metrics for a dataset. These metrics evaluate
- how well a model simulates key aspects of El Nino-Southern Oscillation variability.
- tags: ["ocean", "atmosphere", "enso", "variability"]
-
pmp/enso_tel:
reference_datasets:
- - slug: "obs4mips.GPCP-Monthly-3-2"
- description: "Global Precipitation Climatology Project v3.2 - Precipitation teleconnections (pr)"
+ - slug: "obs4ref.GPCP-Monthly-3-2"
+ source_type: "obs4ref"
+ description: "Global Precipitation Climatology Project v3.2. Supplies pr."
type: "primary"
- - slug: "obs4mips.TropFlux-1-0"
- description: "TropFlux v1.0 - Wind stress (tauu)"
+ - slug: "obs4ref.HadISST-1-1"
+ source_type: "obs4ref"
+ description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1. Supplies ts."
type: "primary"
- - slug: "obs4mips.HadISST-1-1"
- description: "Hadley Centre Sea Ice and SST v1.1 - Sea surface temperature teleconnections (ts)"
+ - slug: "obs4ref.TropFlux-1-0"
+ source_type: "obs4ref"
+ description: "TropFlux v1.0. Supplies air/sea flux and surface fields over the tropical oceans."
type: "primary"
display_name: "ENSO Teleconnections"
description: >-
@@ -58,17 +51,21 @@ pmp/enso_tel:
pmp/enso_proc:
reference_datasets:
- - slug: "obs4mips.GPCP-Monthly-3-2"
- description: "Global Precipitation Climatology Project v3.2 - Precipitation (pr)"
+ - slug: "obs4ref.GPCP-Monthly-3-2"
+ source_type: "obs4ref"
+ description: "Global Precipitation Climatology Project v3.2. Supplies pr."
type: "primary"
- - slug: "obs4mips.TropFlux-1-0"
- description: "TropFlux v1.0 - Wind stress (tauu)"
+ - slug: "obs4ref.HadISST-1-1"
+ source_type: "obs4ref"
+ description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1. Supplies ts."
type: "primary"
- - slug: "obs4mips.HadISST-1-1"
- description: "Hadley Centre Sea Ice and SST v1.1 - Sea surface temperature (ts)"
+ - slug: "obs4ref.TropFlux-1-0"
+ source_type: "obs4ref"
+ description: "TropFlux v1.0. Supplies zonal wind stress (tauu) and turbulent heat fluxes (hfls, hfss)."
type: "primary"
- - slug: "obs4mips.CERES-EBAF-4-2"
- description: "CERES Energy Balanced and Filled v4.2 - Surface and TOA radiation fluxes (hfls, hfss, rlds, rlus, rsds, rsus)"
+ - slug: "obs4ref.CERES-EBAF-4-2"
+ source_type: "obs4ref"
+ description: "CERES Energy Balanced and Filled v4.2. Supplies surface radiation (rlds, rlus, rsds, rsus)."
type: "primary"
display_name: "ENSO Processes"
description: >-
@@ -80,8 +77,9 @@ pmp/enso_proc:
# Extratropical Modes of Variability (Sea Surface Temperature modes)
pmp/extratropical-modes-of-variability-pdo:
reference_datasets:
- - slug: "obs4mips.HadISST-1-1"
- description: "Hadley Centre Sea Ice and SST v1.1 - SST for Pacific Decadal Oscillation"
+ - slug: "obs4ref.HadISST-1-1"
+ source_type: "obs4ref"
+ description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1. Supplies ts."
type: "primary"
display_name: "Pacific Decadal Oscillation (PDO)"
description: >-
@@ -91,8 +89,9 @@ pmp/extratropical-modes-of-variability-pdo:
pmp/extratropical-modes-of-variability-npgo:
reference_datasets:
- - slug: "obs4mips.HadISST-1-1"
- description: "Hadley Centre Sea Ice and SST v1.1 - SST for North Pacific Gyre Oscillation"
+ - slug: "obs4ref.HadISST-1-1"
+ source_type: "obs4ref"
+ description: "Hadley Centre Sea Ice and Sea Surface Temperature v1.1. Supplies ts."
type: "primary"
display_name: "North Pacific Gyre Oscillation (NPGO)"
description: >-
@@ -100,22 +99,12 @@ pmp/extratropical-modes-of-variability-npgo:
which tracks changes in the strength of the North Pacific subtropical and subpolar gyres.
tags: ["ocean", "variability", "pacific"]
-pmp/extratropical-modes-of-variability-amo:
- reference_datasets:
- - slug: "obs4mips.HadISST-1-1"
- description: "Hadley Centre Sea Ice and SST v1.1 - SST for Atlantic Multidecadal Oscillation"
- type: "primary"
- display_name: "Atlantic Multidecadal Oscillation (AMO)"
- description: >-
- Calculate the extratropical modes of variability for the Atlantic Multidecadal Oscillation (AMO),
- a coherent pattern of variability in North Atlantic SSTs with a period of 60-80 years.
- tags: ["ocean", "variability", "atlantic", "decadal"]
-
# Extratropical Modes of Variability (Sea Level Pressure modes)
pmp/extratropical-modes-of-variability-nao:
reference_datasets:
- - slug: "obs4mips.20CR"
- description: "20th Century Reanalysis - Sea level pressure for North Atlantic Oscillation"
+ - slug: "obs4mips.20CR-V2"
+ source_type: "obs4mips"
+ description: "NOAA 20th Century Reanalysis v2. Supplies psl."
type: "primary"
display_name: "North Atlantic Oscillation (NAO)"
description: >-
@@ -125,8 +114,9 @@ pmp/extratropical-modes-of-variability-nao:
pmp/extratropical-modes-of-variability-nam:
reference_datasets:
- - slug: "obs4mips.20CR"
- description: "20th Century Reanalysis - Sea level pressure for Northern Annular Mode"
+ - slug: "obs4mips.20CR-V2"
+ source_type: "obs4mips"
+ description: "NOAA 20th Century Reanalysis v2. Supplies psl."
type: "primary"
display_name: "Northern Annular Mode (NAM)"
description: >-
@@ -136,8 +126,9 @@ pmp/extratropical-modes-of-variability-nam:
pmp/extratropical-modes-of-variability-pna:
reference_datasets:
- - slug: "obs4mips.20CR"
- description: "20th Century Reanalysis - Sea level pressure for Pacific North American Pattern"
+ - slug: "obs4mips.20CR-V2"
+ source_type: "obs4mips"
+ description: "NOAA 20th Century Reanalysis v2. Supplies psl."
type: "primary"
display_name: "Pacific North American Pattern (PNA)"
description: >-
@@ -147,8 +138,9 @@ pmp/extratropical-modes-of-variability-pna:
pmp/extratropical-modes-of-variability-npo:
reference_datasets:
- - slug: "obs4mips.20CR"
- description: "20th Century Reanalysis - Sea level pressure for North Pacific Oscillation"
+ - slug: "obs4mips.20CR-V2"
+ source_type: "obs4mips"
+ description: "NOAA 20th Century Reanalysis v2. Supplies psl."
type: "primary"
display_name: "North Pacific Oscillation (NPO)"
description: >-
@@ -158,8 +150,9 @@ pmp/extratropical-modes-of-variability-npo:
pmp/extratropical-modes-of-variability-sam:
reference_datasets:
- - slug: "obs4mips.20CR"
- description: "20th Century Reanalysis - Sea level pressure for Southern Annular Mode"
+ - slug: "obs4mips.20CR-V2"
+ source_type: "obs4mips"
+ description: "NOAA 20th Century Reanalysis v2. Supplies psl."
type: "primary"
display_name: "Southern Annular Mode (SAM)"
description: >-
diff --git a/backend/tests/test_core/test_diagnostic_metadata.py b/backend/tests/test_core/test_diagnostic_metadata.py
index 53b448a5..a2950cc2 100644
--- a/backend/tests/test_core/test_diagnostic_metadata.py
+++ b/backend/tests/test_core/test_diagnostic_metadata.py
@@ -290,3 +290,73 @@ def test_optional_fields(self, tmp_path: Path):
assert full.tags == ["tag1", "tag2"]
assert full.reference_datasets is not None
assert full.reference_datasets[0].description == "A test dataset"
+
+ def test_source_type_is_optional_and_validated(self, tmp_path: Path):
+ """Test that the reference dataset source type is parsed and constrained to known values."""
+ yaml_content = {
+ "with/source": {
+ "reference_datasets": [
+ {"slug": "obs4mips.HadISST-1-1", "type": "primary", "source_type": "obs4ref"},
+ {"slug": "obs4mips.ERA-5", "type": "primary", "source_type": "obs4mips"},
+ ],
+ },
+ "without/source": {
+ "reference_datasets": [{"slug": "obs4mips.ERA-5", "type": "primary"}],
+ },
+ "bad/source": {
+ "reference_datasets": [
+ {"slug": "obs4mips.ERA-5", "type": "primary", "source_type": "not-a-collection"}
+ ],
+ },
+ }
+
+ yaml_path = tmp_path / "sources.yaml"
+ with open(yaml_path, "w") as f:
+ yaml.dump(yaml_content, f)
+
+ result = load_diagnostic_metadata(yaml_path)
+
+ with_source_type = result["with/source"].reference_datasets
+ assert with_source_type is not None
+ assert [rd.source_type for rd in with_source_type] == ["obs4ref", "obs4mips"]
+
+ without_source_type = result["without/source"].reference_datasets
+ assert without_source_type is not None
+ assert without_source_type[0].source_type is None
+
+ # An unknown source type fails validation, so the entry is skipped rather than served.
+ assert "bad/source" not in result
+
+
+class TestStaticDiagnosticMetadata:
+ """Test the metadata files that ship with the backend."""
+
+ def test_every_reference_dataset_declares_a_source_type(self):
+ """Test that each shipped reference dataset says where its data comes from."""
+ metadata = load_diagnostic_metadata(Path(__file__).parents[2] / "static" / "diagnostics")
+
+ assert metadata
+
+ missing = [
+ (key, rd.slug)
+ for key, entry in metadata.items()
+ for rd in entry.reference_datasets or []
+ if rd.source_type is None
+ ]
+ assert missing == []
+
+ def test_slug_prefix_agrees_with_the_source_type(self):
+ """Test that the name a reader sees says the same thing as the structured source type."""
+ metadata = load_diagnostic_metadata(Path(__file__).parents[2] / "static" / "diagnostics")
+
+ # The datasets named inside a recipe keep the provider's prefix rather than "recipe".
+ prefix_for_source_type = {"recipe": "esmvaltool"}
+
+ mismatched = [
+ (key, rd.slug, rd.source_type)
+ for key, entry in metadata.items()
+ for rd in entry.reference_datasets or []
+ if rd.source_type is not None
+ and rd.slug.partition(".")[0] != prefix_for_source_type.get(rd.source_type, rd.source_type)
+ ]
+ assert mismatched == []
diff --git a/changelog/82.improvement.md b/changelog/82.improvement.md
new file mode 100644
index 00000000..e0462aec
--- /dev/null
+++ b/changelog/82.improvement.md
@@ -0,0 +1,6 @@
+Updates the diagnostic reference dataset metadata to match the datasets the REF providers actually select, and separates published obs4MIPs data from our own obs4REF data.
+
+- Names each dataset by who supplies it, so a dataset required as `obs4mips` but supplied from obs4REF now reads `obs4ref.HadISST-1-1`. A new `source_type` field carries the same thing in a structured form for API consumers.
+- Corrects the source ids to match the data requirements, so ILAMB and PMP datasets are labelled from the providers rather than from a mix of guesses.
+- Adds the five ozone diagnostics, and drops `pmp/enso_perf` and the AMO mode, which the providers no longer register.
+- Fixes the ILAMB `emp-gleamgpcp2.3` key, which never matched the diagnostic slug.
diff --git a/changelog/82.trivial.md b/changelog/82.trivial.md
new file mode 100644
index 00000000..e313c620
--- /dev/null
+++ b/changelog/82.trivial.md
@@ -0,0 +1 @@
+Fixes the Biome pre-commit hooks, which passed staged filenames to `bash -lc` rather than to Biome and so checked the whole frontend tree on every commit. The hooks now run the Biome and TypeScript versions pinned in `frontend/package.json`.
diff --git a/frontend/biome.json b/frontend/biome.json
index b315edfd..775503fc 100644
--- a/frontend/biome.json
+++ b/frontend/biome.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://biomejs.dev/schemas/2.4.10/schema.json",
+ "$schema": "https://biomejs.dev/schemas/2.5.11/schema.json",
"vcs": {
"enabled": false,
"clientKind": "git",
@@ -10,6 +10,7 @@
"includes": [
"**",
"!**/dist",
+ "!**/public",
"!**/coverage",
"!**/src/client",
"!**/*.gen.ts",
diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts
index b2819045..8e7656ac 100644
--- a/frontend/src/client/types.gen.ts
+++ b/frontend/src/client/types.gen.ts
@@ -995,7 +995,7 @@ export type ReferenceDatasetLink = {
/**
* Slug
*
- * Unique identifier for the dataset(e.g., 'obs4mips.CERES-EBAF.v4.2')
+ * Unique identifier for the dataset, `.`. The prefix names who supplies the data rather than the source type the requirement asks for, so a dataset required as obs4mips but supplied from obs4REF reads 'obs4ref.HadISST-1-1'
*/
slug: string;
/**
@@ -1004,6 +1004,18 @@ export type ReferenceDatasetLink = {
* Description of how this dataset is used in the diagnostic
*/
description?: string | null;
+ /**
+ * Source Type
+ *
+ * The source type of the data actually supplied, which is not always the one the requirement asks for:
+ * - 'obs4mips': published on ESGF obs4MIPs
+ * - 'obs4ref': reference data served by the REF, not yet published on obs4MIPs
+ * - 'pmp-climatology': the PMP climatology registry
+ * - 'esmvaltool-reference': the ESMValTool reference registry
+ * - 'ilamb': the ILAMB registry
+ * - 'recipe': named inside the provider's recipe rather than ingested by the REF
+ */
+ source_type?: 'obs4mips' | 'obs4ref' | 'pmp-climatology' | 'esmvaltool-reference' | 'ilamb' | 'recipe' | null;
/**
* Type
*
diff --git a/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/route.tsx b/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/route.tsx
index a85225de..a514aae7 100644
--- a/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/route.tsx
+++ b/frontend/src/routes/_app/diagnostics.$providerSlug.$diagnosticSlug/route.tsx
@@ -293,7 +293,9 @@ const DiagnosticInfoLayout = () => {
Observational or reference datasets that this diagnostic
uses for comparison. Primary references are essential
for the diagnostic to run, while secondary references
- provide additional context.
+ provide additional context. The dataset name says where
+ the data comes from. obs4MIPs data is published, while
+ obs4REF data is not on obs4MIPs yet.
@@ -302,14 +304,6 @@ const DiagnosticInfoLayout = () => {
{data.reference_datasets.map((ref: ReferenceDatasetLink) => (
-
- {ref.type}
-
{ref.slug}
@@ -320,6 +314,13 @@ const DiagnosticInfoLayout = () => {
)}
+
+ {ref.type}
+
))}