auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/654-zonal_mean_xy_cdat_regression_test_netcdf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CDAT Migration Regression Testing Notebook (`.nc` files)\n",
    "\n",
    "This notebook is used to perform regression testing between the development and\n",
    "production versions of a diagnostic set.\n",
    "\n",
    "## How it works\n",
    "\n",
    "It compares the relative differences (%) between ref and test variables between\n",
    "the dev and `main` branches.\n",
    "\n",
    "## How to use\n",
    "\n",
    "PREREQUISITE: The diagnostic set's netCDF stored in `.json` files in two directories\n",
    "(dev and `main` branches).\n",
    "\n",
    "1. Make a copy of this notebook under `auxiliary_tools/cdat_regression_testing/<DIR_NAME>`.\n",
    "2. Run `mamba create -n cdat_regression_test -y -c conda-forge \"python<3.12\" xarray netcdf4 dask pandas matplotlib-base ipykernel`\n",
    "3. Run `mamba activate cdat_regression_test`\n",
    "4. Update `SET_DIR` and `SET_NAME` in the copy of your notebook.\n",
    "5. Run all cells IN ORDER.\n",
    "6. Review results for any outstanding differences (>=1e-5 relative tolerance).\n",
    "   - Debug these differences (e.g., bug in metrics functions, incorrect variable references, etc.)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Code\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of files do not match at DEV_PATH (282) and MAIN_PATH 390.\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "import glob\n",
    "\n",
    "import numpy as np\n",
    "import xarray as xr\n",
    "\n",
    "# TODO: Update SET_NAME and SET_DIR\n",
    "SET_NAME = \"zonal_mean_xy\"\n",
    "SET_DIR = \"654-zonal_mean_xy\"\n",
    "\n",
    "DEV_PATH = f\"/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/{SET_DIR}/{SET_NAME}/**\"\n",
    "MAIN_PATH = f\"/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/{SET_NAME}/**\"\n",
    "\n",
    "DEV_GLOB = sorted(glob.glob(DEV_PATH + \"/*.nc\"))\n",
    "MAIN_GLOB = sorted(glob.glob(MAIN_PATH + \"/*.nc\"))\n",
    "\n",
    "if len(DEV_GLOB) == 0 or len(MAIN_GLOB) == 0:\n",
    "    raise IOError(\"No files found at DEV_PATH and/or MAIN_PATH.\")\n",
    "\n",
    "if len(DEV_GLOB) != len(MAIN_GLOB):\n",
    "    #raise IOError(f\"Number of files do not match at DEV_PATH ({len(DEV_GLOB)}) and MAIN_PATH {len(MAIN_GLOB)}.\")\n",
    "    print(f\"Number of files do not match at DEV_PATH ({len(DEV_GLOB)}) and MAIN_PATH {len(MAIN_GLOB)}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-NETCF-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-NETCF-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-NETCF-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-NETCF-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-NETCF-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-NETCF-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-RESTOM-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-RESTOM-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-RESTOM-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-RESTOM-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-RESTOM-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-RESTOM-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDSC-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDSC-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDSC-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDSC-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDSC-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDSC-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_9.4_ISCCP-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_9.4_ISCCP-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_9.4_ISCCP-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_9.4_ISCCP-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_9.4_ISCCP-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_9.4_ISCCP-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_ISCCP-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_ISCCP-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_ISCCP-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_ISCCP-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_ISCCP-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU1.3_ISCCP-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU9.4_ISCCP-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU9.4_ISCCP-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU9.4_ISCCP-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU9.4_ISCCP-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU9.4_ISCCP-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud ISCCP/ISCCPCOSP-CLDTOT_TAU9.4_ISCCP-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_9.4_MISR-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_9.4_MISR-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_9.4_MISR-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_9.4_MISR-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_9.4_MISR-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_9.4_MISR-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_MISR-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_MISR-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_MISR-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_MISR-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_MISR-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU1.3_MISR-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU9.4_MISR-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU9.4_MISR-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU9.4_MISR-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU9.4_MISR-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU9.4_MISR-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDLOW_TAU9.4_MISR-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_MISR-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_MISR-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_MISR-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_MISR-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_MISR-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_MISR-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU9.4_MISR-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU9.4_MISR-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU9.4_MISR-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU9.4_MISR-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU9.4_MISR-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MISR/MISRCOSP-CLDTOT_TAU9.4_MISR-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_9.4_MODIS-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_9.4_MODIS-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_9.4_MODIS-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_9.4_MODIS-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_9.4_MODIS-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_9.4_MODIS-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_MODIS-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_MODIS-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_MODIS-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_MODIS-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_MODIS-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU1.3_MODIS-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU9.4_MODIS-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU9.4_MODIS-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU9.4_MODIS-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU9.4_MODIS-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU9.4_MODIS-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDHGH_TAU9.4_MODIS-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_9.4_MODIS-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_9.4_MODIS-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_9.4_MODIS-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_9.4_MODIS-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_9.4_MODIS-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_9.4_MODIS-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_MODIS-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_MODIS-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_MODIS-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_MODIS-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_MODIS-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU1.3_MODIS-JJA-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU9.4_MODIS-ANN-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU9.4_MODIS-ANN-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU9.4_MODIS-ANN-global_test.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU9.4_MODIS-JJA-global_diff.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU9.4_MODIS-JJA-global_ref.nc!\n",
      "No file found to compare with /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud MODIS/MODISCOSP-CLDTOT_TAU9.4_MODIS-JJA-global_test.nc!\n",
      "Number of files missing: 108\n"
     ]
    }
   ],
   "source": [
    "missing_count = 0\n",
    "for filepath_main in MAIN_GLOB:\n",
    "    filepath_dev = filepath_main.replace(\"main\", SET_DIR)\n",
    "    try:\n",
    "        ds = xr.open_dataset(filepath_dev)\n",
    "    except OSError:\n",
    "        print(f\"No file found to compare with {filepath_main}!\")\n",
    "        missing_count += 1\n",
    "print(f\"Number of files missing: {missing_count}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Compare the netCDF files between branches\n",
    "\n",
    "- Compare \"ref\" and \"test\" files\n",
    "- \"diff\" files are ignored because getting relative diffs for these does not make sense (relative diff will be above tolerance)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-ANN-global_ref.nc\n",
      "var_key ALBEDO\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-ANN-global_test.nc\n",
      "var_key ALBEDO\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-JJA-global_ref.nc\n",
      "var_key ALBEDO\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "x and y nan location mismatch:\n",
      " x: array([     inf,      inf,      inf,      inf,      inf,      inf,\n",
      "            inf,      inf,      inf, 4.905806, 1.104448, 0.486332,\n",
      "       1.070643, 0.9697  , 0.895325, 0.848754, 0.816032, 0.794203,...\n",
      " y: array([         nan,          nan,          nan,          nan,\n",
      "                nan,          nan,          nan,          nan,\n",
      "       2.497374e+06, 4.905807e+00, 1.104448e+00, 4.863323e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDO-JJA-global_test.nc\n",
      "var_key ALBEDO\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-ANN-global_ref.nc\n",
      "var_key ALBEDOC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-ANN-global_test.nc\n",
      "var_key ALBEDOC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-JJA-global_ref.nc\n",
      "var_key ALBEDOC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "x and y nan location mismatch:\n",
      " x: array([     inf,      inf,      inf,      inf,      inf,      inf,\n",
      "            inf,      inf,      inf, 2.820785, 2.415148, 2.439055,\n",
      "       2.429912, 2.546782, 2.106862, 1.591239, 1.257344, 1.041096,...\n",
      " y: array([         nan,          nan,          nan,          nan,\n",
      "                nan,          nan,          nan,          nan,\n",
      "       2.353753e+04, 2.820784e+00, 2.415149e+00, 2.439055e+00,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-ALBEDOC-JJA-global_test.nc\n",
      "var_key ALBEDOC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-ANN-global_ref.nc\n",
      "var_key FLUT\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 179 / 180 (99.4%)\n",
      "Max absolute difference: 7.79999068\n",
      "Max relative difference: 0.03936372\n",
      " x: array([141.246109, 142.448908, 144.944363, 148.769939, 151.618505,\n",
      "       154.311928, 156.204058, 157.14579 , 157.79317 , 158.203278,\n",
      "       158.684151, 159.585104, 161.486325, 163.265739, 165.540443,...\n",
      " y: array([141.193008, 141.980984, 145.626382, 148.877415, 152.39056 ,\n",
      "       155.563024, 156.842727, 157.492508, 157.808824, 157.700497,\n",
      "       157.710477, 158.174671, 159.525564, 161.255139, 163.758747,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-ANN-global_test.nc\n",
      "var_key FLUT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-JJA-global_ref.nc\n",
      "var_key FLUT\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 13.14937265\n",
      "Max relative difference: 0.05631396\n",
      " x: array([118.728521, 120.124598, 122.939272, 126.806588, 129.464821,\n",
      "       131.808891, 133.047425, 133.685549, 134.117062, 134.603681,\n",
      "       135.341986, 136.71499 , 139.38547 , 141.552625, 144.099721,...\n",
      " y: array([113.994202, 114.798511, 118.883746, 122.433748, 126.118669,\n",
      "       129.182751, 130.391198, 130.980806, 131.305683, 131.209387,\n",
      "       131.431461, 132.321879, 134.364658, 136.657223, 139.62437 ,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUT-JJA-global_test.nc\n",
      "var_key FLUT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-ANN-global_ref.nc\n",
      "var_key FLUTC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 6.33922403\n",
      "Max relative difference: 0.04153241\n",
      " x: array([139.868149, 141.256436, 144.188424, 148.897385, 152.484459,\n",
      "       155.804953, 158.166585, 159.384443, 160.23524 , 160.883335,\n",
      "       161.692296, 163.057875, 165.718024, 168.099169, 171.192569,...\n",
      " y: array([144.134415, 146.003993, 150.43641 , 154.41483 , 158.823683,\n",
      "       161.909077, 162.810828, 163.451103, 163.773736, 163.820472,\n",
      "       164.54419 , 165.963616, 168.190604, 171.259626, 174.878848,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-ANN-global_test.nc\n",
      "var_key FLUTC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-JJA-global_ref.nc\n",
      "var_key FLUTC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 5.78375481\n",
      "Max relative difference: 0.03386614\n",
      " x: array([115.912369, 117.501384, 120.76307 , 125.421667, 128.716684,\n",
      "       131.560534, 133.153934, 133.964388, 134.496953, 135.085091,\n",
      "       136.033789, 137.792704, 141.162996, 143.932394, 147.355409,...\n",
      " y: array([114.974915, 117.173111, 122.874888, 127.662063, 132.643509,\n",
      "       135.951245, 136.719533, 137.339831, 137.729988, 137.718733,\n",
      "       138.824733, 140.62775 , 143.507289, 147.499592, 151.622775,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FLUTC-JJA-global_test.nc\n",
      "var_key FLUTC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-ANN-global_ref.nc\n",
      "var_key FSNTOA\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-ANN-global_test.nc\n",
      "var_key FSNTOA\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-JJA-global_ref.nc\n",
      "var_key FSNTOA\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOA-JJA-global_test.nc\n",
      "var_key FSNTOA\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-ANN-global_ref.nc\n",
      "var_key FSNTOAC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-ANN-global_test.nc\n",
      "var_key FSNTOAC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-JJA-global_ref.nc\n",
      "var_key FSNTOAC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-FSNTOAC-JJA-global_test.nc\n",
      "var_key FSNTOAC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-ANN-global_ref.nc\n",
      "var_key LWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-ANN-global_test.nc\n",
      "var_key LWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-JJA-global_ref.nc\n",
      "var_key LWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-LWCF-JJA-global_test.nc\n",
      "var_key LWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-ANN-global_ref.nc\n",
      "var_key SOLIN\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 176 / 180 (97.8%)\n",
      "Max absolute difference: 0.25607177\n",
      "Max relative difference: 0.00121066\n",
      " x: array([172.379822, 172.478048, 172.680255, 173.084065, 173.487471,\n",
      "       174.045946, 174.811961, 175.544428, 176.461651, 177.601705,\n",
      "       178.681469, 179.973292, 181.508563, 182.961183, 184.655792,...\n",
      " y: array([172.254196, 172.367507, 172.595506, 172.936009, 173.391881,\n",
      "       173.964149, 174.65194 , 175.459979, 176.385718, 177.433514,\n",
      "       178.605712, 179.90479 , 181.332968, 182.894449, 184.594455,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-ANN-global_test.nc\n",
      "var_key SOLIN\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-JJA-global_ref.nc\n",
      "var_key SOLIN\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 172 / 180 (95.6%)\n",
      "Max absolute difference: 2.60589862\n",
      "Max relative difference: 2203.43224529\n",
      " x: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       1.299852e-03, 3.566784e-02, 1.439924e-01, 3.679212e-01,...\n",
      " y: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       5.896539e-07, 3.771837e-02, 1.805163e-01, 4.534470e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SOLIN-JJA-global_test.nc\n",
      "var_key SOLIN\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-ANN-global_ref.nc\n",
      "var_key SWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-ANN-global_test.nc\n",
      "var_key SWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-JJA-global_ref.nc\n",
      "var_key SWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-TOA-v4.1/ceres_ebaf_toa_v4.1-SWCF-JJA-global_test.nc\n",
      "var_key SWCF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-ANN-global_ref.nc\n",
      "var_key ALBEDO_SRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-ANN-global_test.nc\n",
      "var_key ALBEDO_SRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-JJA-global_ref.nc\n",
      "var_key ALBEDO_SRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-ALBEDO_SRF-JJA-global_test.nc\n",
      "var_key ALBEDO_SRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-ANN-global_ref.nc\n",
      "var_key FLDS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 22.30185886\n",
      "Max relative difference: 0.17424603\n",
      " x: array([ 98.603769, 100.013104, 103.124255, 109.477082, 114.828849,\n",
      "       120.367467, 125.103684, 127.989808, 130.523101, 132.72756 ,\n",
      "       134.853978, 137.151021, 141.412339, 145.471684, 151.087881,...\n",
      " y: array([119.410591, 118.31206 , 123.309446, 128.200745, 133.780149,\n",
      "       135.215224, 136.666433, 137.553054, 137.775374, 137.603672,\n",
      "       138.024805, 140.406558, 146.011641, 151.996392, 159.280968,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-ANN-global_test.nc\n",
      "var_key FLDS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-JJA-global_ref.nc\n",
      "var_key FLDS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 30.02145386\n",
      "Max relative difference: 0.21021598\n",
      " x: array([ 86.579041,  87.773002,  90.297814,  94.94907 ,  98.532159,\n",
      "       101.821095, 104.307123, 105.902524, 107.730424, 109.863227,\n",
      "       112.170159, 114.71559 , 119.379931, 123.518244, 128.946971,...\n",
      " y: array([105.692902, 104.562911, 114.149889, 120.14752 , 124.75836 ,\n",
      "       123.85244 , 122.906467, 122.710191, 123.046583, 123.173363,\n",
      "       124.120997, 127.510116, 131.521946, 135.512223, 142.274537,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLDS-JJA-global_test.nc\n",
      "var_key FLDS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-ANN-global_ref.nc\n",
      "var_key FLNS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 14.81529866\n",
      "Max relative difference: 0.50590681\n",
      " x: array([38.672388, 39.632284, 41.554582, 43.501992, 44.63556 , 45.100002,\n",
      "       44.525974, 43.693555, 42.575008, 41.432901, 40.768997, 41.093413,\n",
      "       42.111652, 42.513002, 42.677124, 42.86597 , 42.733227, 41.992849,...\n",
      " y: array([25.680466, 28.040907, 30.184382, 32.143437, 34.077625, 37.068989,\n",
      "       36.722073, 36.567483, 36.857023, 36.904296, 37.098863, 36.580727,\n",
      "       36.892113, 37.273634, 37.062925, 36.622801, 35.768733, 36.590338,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-ANN-global_test.nc\n",
      "var_key FLNS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-JJA-global_ref.nc\n",
      "var_key FLNS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 31.08156996\n",
      "Max relative difference: 1.99768489\n",
      " x: array([23.173501, 24.600908, 27.443069, 30.426269, 32.228169, 33.389323,\n",
      "       32.921439, 32.165488, 30.741161, 29.256816, 28.440312, 29.063455,\n",
      "       30.711824, 31.401851, 31.815865, 32.276679, 32.631757, 32.746176,...\n",
      " y: array([ 9.407887, 10.994175,  9.154754, 10.756408, 13.533618, 17.267534,\n",
      "       18.091678, 18.120708, 17.883574, 16.984547, 17.261472, 16.838912,\n",
      "       19.529889, 22.028511, 21.843096, 22.190228, 22.273543, 24.49743 ,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNS-JJA-global_test.nc\n",
      "var_key FLNS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-ANN-global_ref.nc\n",
      "var_key FLNSC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 10.86564869\n",
      "Max relative difference: 0.16694748\n",
      " x: array([50.142589, 51.490356, 54.218586, 57.809375, 59.661539, 60.657781,\n",
      "       60.325538, 59.89425 , 59.303388, 59.115358, 59.632704, 61.077648,\n",
      "       64.096028, 65.588617, 66.933318, 68.847662, 70.85666 , 73.230059,...\n",
      " y: array([57.019058, 60.835803, 65.084235, 67.661385, 69.16662 , 70.807613,\n",
      "       69.408429, 68.197493, 67.820274, 67.731022, 68.192893, 69.525411,\n",
      "       71.076735, 73.087424, 74.88605 , 76.617546, 78.463812, 80.802389,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-ANN-global_test.nc\n",
      "var_key FLNSC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-JJA-global_ref.nc\n",
      "var_key FLNSC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 13.06876658\n",
      "Max relative difference: 0.16989632\n",
      " x: array([ 35.083467,  36.80715 ,  40.184035,  44.132413,  46.014228,\n",
      "        46.854808,  45.771593,  44.871804,  43.837011,  43.44405 ,\n",
      "        43.982251,  45.823333,  49.76725 ,  51.703341,  53.384169,...\n",
      " y: array([36.823761, 40.103637, 46.437629, 49.795666, 51.668213, 52.729509,\n",
      "       50.523095, 48.817463, 48.325775, 47.815335, 48.943311, 51.562074,\n",
      "       55.116418, 59.465276, 62.162868, 64.817275, 68.520103, 72.965333,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FLNSC-JJA-global_test.nc\n",
      "var_key FLNSC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-ANN-global_ref.nc\n",
      "var_key FSDS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 27.15617752\n",
      "Max relative difference: 0.2734143\n",
      " x: array([134.500011, 134.207817, 133.496102, 131.715675, 130.5824  ,\n",
      "       129.71912 , 129.328282, 129.17619 , 129.406694, 129.803844,\n",
      "       129.961034, 130.286989, 129.706923, 129.881992, 129.960545,...\n",
      " y: array([127.133133, 128.091594, 130.511359, 130.840244, 130.630491,\n",
      "       130.691847, 130.441245, 130.023419, 130.196103, 130.54729 ,\n",
      "       131.20615 , 132.222195, 131.048702, 129.485736, 129.345354,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-ANN-global_test.nc\n",
      "var_key FSDS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-JJA-global_ref.nc\n",
      "var_key FSDS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 172 / 180 (95.6%)\n",
      "Max absolute difference: 81.9628933\n",
      "Max relative difference: 755.00534282\n",
      " x: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       4.101190e-04, 1.220662e-02, 5.444169e-02, 1.496037e-01,...\n",
      " y: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       5.424817e-07, 1.790796e-02, 7.905615e-02, 2.006905e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDS-JJA-global_test.nc\n",
      "var_key FSDS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-ANN-global_ref.nc\n",
      "var_key FSDSC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 12.02716031\n",
      "Max relative difference: 0.04614967\n",
      " x: array([142.169182, 142.120644, 141.970673, 141.508112, 141.057584,\n",
      "       140.704173, 140.652836, 140.953918, 141.536291, 142.537232,\n",
      "       143.507893, 144.709581, 145.840567, 146.883002, 147.91992 ,...\n",
      " y: array([145.749074, 146.426376, 146.247775, 145.939056, 145.553679,\n",
      "       145.168239, 145.319736, 145.761183, 146.50989 , 147.55443 ,\n",
      "       148.684689, 150.032874, 150.800413, 151.727249, 152.64788 ,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-ANN-global_test.nc\n",
      "var_key FSDSC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-JJA-global_ref.nc\n",
      "var_key FSDSC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 172 / 180 (95.6%)\n",
      "Max absolute difference: 15.55342543\n",
      "Max relative difference: 795.42544402\n",
      " x: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       4.320462e-04, 1.325242e-02, 6.056289e-02, 1.684320e-01,...\n",
      " y: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       5.424817e-07, 2.319665e-02, 1.040350e-01, 2.660579e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSDSC-JJA-global_test.nc\n",
      "var_key FSDSC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-ANN-global_ref.nc\n",
      "var_key FSNS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 13.59378399\n",
      "Max relative difference: 0.40684139\n",
      " x: array([ 24.569824,  24.504611,  24.495851,  25.072746,  25.628108,\n",
      "        25.809881,  26.131333,  26.349462,  26.797732,  27.032531,\n",
      "        27.58193 ,  28.082599,  30.027566,  30.689215,  31.548233,...\n",
      " y: array([ 24.888004,  26.214967,  28.394634,  29.318973,  29.450419,\n",
      "        28.956498,  28.067234,  27.375691,  26.946654,  26.912544,\n",
      "        26.744005,  27.60545 ,  30.166636,  31.077168,  30.888393,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-ANN-global_test.nc\n",
      "var_key FSNS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-JJA-global_ref.nc\n",
      "var_key FSNS\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 172 / 180 (95.6%)\n",
      "Max absolute difference: 37.6358188\n",
      "Max relative difference: 0.53590547\n",
      " x: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       6.052015e-05, 1.892413e-03, 8.475397e-03, 2.353338e-02,...\n",
      " y: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 3.734292e-03, 1.625087e-02, 4.703653e-02,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNS-JJA-global_test.nc\n",
      "var_key FSNS\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-ANN-global_ref.nc\n",
      "var_key FSNSC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 14.79101111\n",
      "Max relative difference: 0.26783535\n",
      " x: array([ 26.328924,  26.334813,  26.497151,  27.495192,  28.312105,\n",
      "        28.686906,  29.16632 ,  29.573289,  30.197356,  30.663855,\n",
      "        31.565477,  32.472693,  35.416757,  36.530502,  37.93983 ,...\n",
      " y: array([ 32.733452,  33.532551,  33.536793,  34.557571,  34.456016,\n",
      "        33.777616,  33.078005,  32.97839 ,  32.825829,  33.044484,\n",
      "        33.229347,  34.43668 ,  39.107409,  42.050515,  42.093438,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-ANN-global_test.nc\n",
      "var_key FSNSC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-JJA-global_ref.nc\n",
      "var_key FSNSC\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 172 / 180 (95.6%)\n",
      "Max absolute difference: 35.53302786\n",
      "Max relative difference: 10.08476802\n",
      " x: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       6.221870e-05, 1.953278e-03, 8.905534e-03, 2.504571e-02,...\n",
      " y: array([0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
      "       0.000000e+00, 4.828696e-03, 1.949154e-02, 3.929949e-02,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-FSNSC-JJA-global_test.nc\n",
      "var_key FSNSC\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-ANN-global_ref.nc\n",
      "var_key LWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-ANN-global_test.nc\n",
      "var_key LWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-JJA-global_ref.nc\n",
      "var_key LWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-LWCFSRF-JJA-global_test.nc\n",
      "var_key LWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-ANN-global_ref.nc\n",
      "var_key SWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-ANN-global_test.nc\n",
      "var_key SWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-JJA-global_ref.nc\n",
      "var_key SWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/CERES-EBAF-surface-v4.1/ceres_ebaf_surface_v4.1-SWCFSRF-JJA-global_test.nc\n",
      "var_key SWCFSRF\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-ANN-global_ref.nc\n",
      "var_key PminusE\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 169 / 180 (93.9%)\n",
      "Max absolute difference: 2.68669262\n",
      "Max relative difference: 0.99998843\n",
      " x: array([          nan,           nan,           nan,           nan,\n",
      "                 nan,           nan,           nan,           nan,\n",
      "                 nan,           nan,           nan, -1.362539e-06,...\n",
      " y: array([      nan,       nan,       nan,       nan,       nan,       nan,\n",
      "             nan,       nan,       nan,       nan,       nan, -0.117723,\n",
      "       -0.275558, -0.151884, -0.083765, -0.021892,  0.034201,  0.038609,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-ANN-global_test.nc\n",
      "var_key PminusE\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-JJA-global_ref.nc\n",
      "var_key PminusE\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 169 / 180 (93.9%)\n",
      "Max absolute difference: 4.20649947\n",
      "Max relative difference: 0.99998843\n",
      " x: array([          nan,           nan,           nan,           nan,\n",
      "                 nan,           nan,           nan,           nan,\n",
      "                 nan,           nan,           nan, -3.314214e-06,...\n",
      " y: array([      nan,       nan,       nan,       nan,       nan,       nan,\n",
      "             nan,       nan,       nan,       nan,       nan, -0.286348,\n",
      "       -0.466065, -0.300971, -0.246707, -0.202872, -0.148632, -0.135121,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/COREv2_Flux/COREv2_Flux-PminusE-JJA-global_test.nc\n",
      "var_key PminusE\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-ANN-global_ref.nc\n",
      "var_key CLDHGH_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-ANN-global_test.nc\n",
      "var_key CLDHGH_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-JJA-global_ref.nc\n",
      "var_key CLDHGH_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDHGH_CAL-JJA-global_test.nc\n",
      "var_key CLDHGH_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-ANN-global_ref.nc\n",
      "var_key CLDLOW_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-ANN-global_test.nc\n",
      "var_key CLDLOW_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-JJA-global_ref.nc\n",
      "var_key CLDLOW_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDLOW_CAL-JJA-global_test.nc\n",
      "var_key CLDLOW_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-ANN-global_ref.nc\n",
      "var_key CLDMED_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-ANN-global_test.nc\n",
      "var_key CLDMED_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-JJA-global_ref.nc\n",
      "var_key CLDMED_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDMED_CAL-JJA-global_test.nc\n",
      "var_key CLDMED_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-ANN-global_ref.nc\n",
      "var_key CLDTOT_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-ANN-global_test.nc\n",
      "var_key CLDTOT_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-JJA-global_ref.nc\n",
      "var_key CLDTOT_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/Cloud Calipso/CALIPSOCOSP-CLDTOT_CAL-JJA-global_test.nc\n",
      "var_key CLDTOT_CAL\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-LHFLX-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-LHFLX-ANN-global_ref.nc\n",
      "var_key LHFLX\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "(shapes (180,), (721,) mismatch)\n",
      " x: array([-1.262214e-01, -1.335355e-01, -1.205506e-01,  1.119733e-01,\n",
      "        4.583618e-01,  5.695841e-01,  4.694749e-01,  3.887818e-01,\n",
      "        3.957684e-01,  4.298707e-01,  6.209118e-01,  8.169651e-01,...\n",
      " y: array([-2.811061e-01, -2.863868e-01, -2.983699e-01, -3.009214e-01,\n",
      "       -3.107236e-01, -3.250959e-01, -3.408958e-01, -3.480319e-01,\n",
      "       -3.395443e-01, -3.178686e-01, -2.870451e-01, -2.483378e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-LHFLX-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-LHFLX-ANN-global_test.nc\n",
      "var_key LHFLX\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 0.31178502\n",
      "Max relative difference: 0.21642227\n",
      " x: array([-1.262214e-01, -1.335355e-01, -1.205506e-01,  1.119733e-01,\n",
      "        4.583618e-01,  5.695841e-01,  4.694749e-01,  3.887818e-01,\n",
      "        3.957684e-01,  4.298707e-01,  6.209118e-01,  8.169651e-01,...\n",
      " y: array([-1.096475e-01, -1.151044e-01, -9.910261e-02,  1.263154e-01,\n",
      "        4.567322e-01,  5.734855e-01,  4.874444e-01,  4.170845e-01,\n",
      "        4.309056e-01,  4.696122e-01,  6.584857e-01,  8.427438e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-LHFLX-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-LHFLX-JJA-global_ref.nc\n",
      "var_key LHFLX\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "(shapes (180,), (721,) mismatch)\n",
      " x: array([-7.812247e-02, -1.082585e-01, -1.767612e-01, -2.388224e-01,\n",
      "       -1.404205e-01, -1.636696e-01, -3.024474e-01, -3.957535e-01,\n",
      "       -4.690657e-01, -5.274440e-01, -5.584010e-01, -5.479116e-01,...\n",
      " y: array([-3.225471e-01, -3.244649e-01, -3.344758e-01, -3.357413e-01,\n",
      "       -3.388889e-01, -3.504171e-01, -3.711881e-01, -3.928775e-01,\n",
      "       -4.071674e-01, -4.138297e-01, -4.151484e-01, -4.082568e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-LHFLX-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-LHFLX-JJA-global_test.nc\n",
      "var_key LHFLX\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "Mismatched elements: 180 / 180 (100%)\n",
      "Max absolute difference: 0.56433213\n",
      "Max relative difference: 3.49069991\n",
      " x: array([-7.812247e-02, -1.082585e-01, -1.767612e-01, -2.388224e-01,\n",
      "       -1.404205e-01, -1.636696e-01, -3.024474e-01, -3.957535e-01,\n",
      "       -4.690657e-01, -5.274440e-01, -5.584010e-01, -5.479116e-01,...\n",
      " y: array([-6.889836e-02, -9.547618e-02, -1.558905e-01, -2.100813e-01,\n",
      "       -1.210395e-01, -1.408228e-01, -2.651606e-01, -3.479036e-01,\n",
      "       -4.129178e-01, -4.647684e-01, -4.920337e-01, -4.808924e-01,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-PRECT-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-PRECT-ANN-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-PRECT-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-PRECT-ANN-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-PRECT-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-PRECT-JJA-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-PRECT-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-PRECT-JJA-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-200-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-200-ANN-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-200-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-200-ANN-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-200-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-200-JJA-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-200-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-200-JJA-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-850-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-850-ANN-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-850-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-850-ANN-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-850-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-850-JJA-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-T-850-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-T-850-JJA-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TMQ-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TMQ-ANN-global_ref.nc\n",
      "var_key TMQ\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "(shapes (180,), (721,) mismatch)\n",
      " x: array([ 0.496739,  0.51491 ,  0.559682,  0.680788,  0.830239,  1.019754,\n",
      "        1.221377,  1.343748,  1.457112,  1.539496,  1.599018,  1.65859 ,\n",
      "        1.760001,  1.902417,  2.129324,  2.408657,  2.695891,  3.007882,...\n",
      " y: array([ 0.510452,  0.50892 ,  0.505115,  0.503489,  0.50133 ,  0.500357,\n",
      "        0.501718,  0.506679,  0.515499,  0.527265,  0.541239,  0.557384,\n",
      "        0.574094,  0.592596,  0.612698,  0.630298,  0.651497,  0.692578,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TMQ-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TMQ-ANN-global_test.nc\n",
      "var_key TMQ\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TMQ-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TMQ-JJA-global_ref.nc\n",
      "var_key TMQ\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "(shapes (180,), (721,) mismatch)\n",
      " x: array([ 0.326149,  0.338786,  0.36922 ,  0.444122,  0.531779,  0.639072,\n",
      "        0.745944,  0.81095 ,  0.871856,  0.922061,  0.966205,  1.016131,\n",
      "        1.096865,  1.203753,  1.369331,  1.574277,  1.786222,  2.016905,...\n",
      " y: array([ 0.307267,  0.305856,  0.30437 ,  0.303801,  0.303308,  0.303602,\n",
      "        0.305342,  0.309214,  0.315328,  0.323192,  0.332349,  0.342588,\n",
      "        0.35346 ,  0.365171,  0.377638,  0.388869,  0.403474,  0.430992,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TMQ-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TMQ-JJA-global_test.nc\n",
      "var_key TMQ\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TREFHT-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TREFHT-ANN-global_ref.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TREFHT-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TREFHT-ANN-global_test.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TREFHT-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TREFHT-JJA-global_ref.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-TREFHT-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-TREFHT-JJA-global_test.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-Z3-500-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-Z3-500-ANN-global_ref.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-Z3-500-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-Z3-500-ANN-global_test.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-Z3-500-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-Z3-500-JJA-global_ref.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/ERA5/ERA5-Z3-500-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/ERA5/ERA5-Z3-500-JJA-global_test.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-ANN-global_ref.nc\n",
      "var_key PminusE\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-ANN-global_test.nc\n",
      "var_key PminusE\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-JJA-global_ref.nc\n",
      "var_key PminusE\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_OAFLux/GPCP_OAFLux-PminusE-JJA-global_test.nc\n",
      "var_key PminusE\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-ANN-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-ANN-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-JJA-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v2.3/GPCP_v2.3-PRECT-JJA-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-ANN-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-ANN-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-JJA-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/GPCP_v3.2/GPCP_v3.2-PRECT-JJA-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-PRECT-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-PRECT-ANN-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-PRECT-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-PRECT-ANN-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-PRECT-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-PRECT-JJA-global_ref.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-PRECT-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-PRECT-JJA-global_test.nc\n",
      "var_key PRECT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-200-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-200-ANN-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-200-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-200-ANN-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-200-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-200-JJA-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-200-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-200-JJA-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-850-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-850-ANN-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-850-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-850-ANN-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-850-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-850-JJA-global_ref.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-T-850-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-T-850-JJA-global_test.nc\n",
      "var_key T\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TMQ-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TMQ-ANN-global_ref.nc\n",
      "var_key TMQ\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "(shapes (180,), (361,) mismatch)\n",
      " x: array([ 0.496739,  0.51491 ,  0.559682,  0.680788,  0.830239,  1.019754,\n",
      "        1.221377,  1.343748,  1.457112,  1.539496,  1.599018,  1.65859 ,\n",
      "        1.760001,  1.902417,  2.129324,  2.408657,  2.695891,  3.007882,...\n",
      " y: array([ 0.473938,  0.472531,  0.469436,  0.470072,  0.482709,  0.50715 ,\n",
      "        0.539403,  0.576001,  0.627298,  0.719738,  0.822251,  0.909516,\n",
      "        0.981075,  1.038113,  1.09093 ,  1.1499  ,  1.186421,  1.223845,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TMQ-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TMQ-ANN-global_test.nc\n",
      "var_key TMQ\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TMQ-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TMQ-JJA-global_ref.nc\n",
      "var_key TMQ\n",
      "\n",
      "Not equal to tolerance rtol=1e-05, atol=0\n",
      "\n",
      "(shapes (180,), (361,) mismatch)\n",
      " x: array([ 0.326149,  0.338786,  0.36922 ,  0.444122,  0.531779,  0.639072,\n",
      "        0.745944,  0.81095 ,  0.871856,  0.922061,  0.966205,  1.016131,\n",
      "        1.096865,  1.203753,  1.369331,  1.574277,  1.786222,  2.016905,...\n",
      " y: array([ 0.2718  ,  0.271416,  0.270968,  0.272947,  0.281705,  0.297261,\n",
      "        0.317342,  0.340431,  0.373077,  0.43018 ,  0.492562,  0.546491,\n",
      "        0.591226,  0.627795,  0.660497,  0.695997,  0.719053,  0.743398,...\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TMQ-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TMQ-JJA-global_test.nc\n",
      "var_key TMQ\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFHT-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFHT-ANN-global_ref.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFHT-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFHT-ANN-global_test.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFHT-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFHT-JJA-global_ref.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFHT-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFHT-JJA-global_test.nc\n",
      "var_key TREFHT\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-ANN-global_ref.nc\n",
      "var_key TREFMNAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-ANN-global_test.nc\n",
      "var_key TREFMNAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-JJA-global_ref.nc\n",
      "var_key TREFMNAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMNAV-JJA-global_test.nc\n",
      "var_key TREFMNAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-ANN-global_ref.nc\n",
      "var_key TREFMXAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-ANN-global_test.nc\n",
      "var_key TREFMXAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-JJA-global_ref.nc\n",
      "var_key TREFMXAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-TREFMXAV-JJA-global_test.nc\n",
      "var_key TREFMXAV\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-200-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-200-ANN-global_ref.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-200-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-200-ANN-global_test.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-200-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-200-JJA-global_ref.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-200-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-200-JJA-global_test.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-850-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-850-ANN-global_ref.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-850-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-850-ANN-global_test.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-850-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-850-JJA-global_ref.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-U-850-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-U-850-JJA-global_test.nc\n",
      "var_key U\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-Z3-500-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-Z3-500-ANN-global_ref.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-Z3-500-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-Z3-500-ANN-global_test.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-Z3-500-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-Z3-500-JJA-global_ref.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/MERRA2/MERRA2-Z3-500-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/MERRA2/MERRA2-Z3-500-JJA-global_test.nc\n",
      "var_key Z3\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-ANN-global_ref.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-ANN-global_test.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-JJA-global_ref.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_CL_HadISST/HadISST_CL-SST-JJA-global_test.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-ANN-global_ref.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-ANN-global_test.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-JJA-global_ref.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PD_HadISST/HadISST_PD-SST-JJA-global_test.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-ANN-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-ANN-global_ref.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-ANN-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-ANN-global_test.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-JJA-global_ref.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-JJA-global_ref.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n",
      "Comparing:\n",
      "/global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/654-zonal_mean_xy/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-JJA-global_test.nc \n",
      " /global/cfs/projectdirs/e3sm/e3sm_diags_cdat_test/main/zonal_mean_xy/SST_PI_HadISST/HadISST_PI-SST-JJA-global_test.nc\n",
      "var_key SST\n",
      "   * All close and within relative tolerance (1e-05)\n"
     ]
    }
   ],
   "source": [
    "# We are mainly focusing on relative tolerance here (in percentage terms).\n",
    "atol = 0\n",
    "rtol = 1e-5\n",
    "\n",
    "for filepath_dev in DEV_GLOB:\n",
    "    if ('test.nc' in filepath_dev or 'ref.nc' in filepath_dev):\n",
    "        filepath_main = filepath_dev.replace(SET_DIR, \"main\")\n",
    "        print(\"Comparing:\")\n",
    "        print(filepath_dev, \"\\n\", filepath_main)\n",
    "        ds1 = xr.open_dataset(filepath_dev)\n",
    "        ds2 = xr.open_dataset(filepath_main)\n",
    "\n",
    "        var_key = filepath_dev.split(\"-\")[-3] \n",
    "        # for 3d vars such as T-200\n",
    "        var_key.isdigit()\n",
    "        if var_key.isdigit(): \n",
    "            var_key = filepath_dev.split(\"-\")[-4] \n",
    "        \n",
    "        print('var_key', var_key)\n",
    "        try:\n",
    "            np.testing.assert_allclose(\n",
    "                ds1[var_key].values,\n",
    "                ds2[var_key].values,\n",
    "                atol=atol,\n",
    "                rtol=rtol,\n",
    "            )\n",
    "        except AssertionError as e:\n",
    "            print(e)\n",
    "        else:\n",
    "            print(f\"   * All close and within relative tolerance ({rtol})\")\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "\n",
    "- The relative tolerance of all files are 1e-05, which means things should be good to go.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cdat_regression_test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}