From 3a26adfc177b03247fe4e32accf795c31b698791 Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Fri, 10 May 2024 17:40:36 -0700 Subject: [PATCH] don't use NCD write --- artifacts/artifact_funcs.jl | 14 +- experiments/AMIP/Manifest.toml | 12 +- experiments/AMIP/coupler_driver.jl | 2 +- experiments/ClimaCore/Manifest.toml | 14 +- experiments/ClimaCore/Project.toml | 2 - src/Regridder.jl | 34 +- test/Project.toml | 2 - test/regridder_tests.jl | 598 ++++++++++++++-------------- 8 files changed, 344 insertions(+), 334 deletions(-) diff --git a/artifacts/artifact_funcs.jl b/artifacts/artifact_funcs.jl index ea9f02304d..147cdf76f2 100644 --- a/artifacts/artifact_funcs.jl +++ b/artifacts/artifact_funcs.jl @@ -61,22 +61,22 @@ function pr_obs_data_path() end """ - artifact_data(datapath_full, name) + artifact_data(datapath_full, varname) Returns input dataset at datapath_full """ -function artifact_data(datapath_full, name) - datafile_truncated = joinpath(datapath_full, string(name, ".nc")) +function artifact_data(datapath_full, varname) + datafile_truncated = joinpath(datapath_full, string(lowercase(varname), ".nc")) return datafile_truncated end """ - artifact_data(datapath_full, name, datapath_trunc, date0, t_start, t_end, comms_ctx) + artifact_data(datapath_full, varname, datapath_trunc, date0, t_start, t_end, comms_ctx) Truncates given data set, and constructs a new dataset containing only the dates needed and stores it in datapath_trunc """ -function artifact_data(datapath_full, name, datapath_trunc, date0, t_start, t_end, comms_ctx) - datafile = joinpath(datapath_full, string(name, ".nc")) - datafile_truncated = Regridder.truncate_dataset(datafile, name, datapath_trunc, date0, t_start, t_end, comms_ctx) +function artifact_data(datapath_full, varname, datapath_trunc, date0, t_start, t_end, comms_ctx) + datafile = joinpath(datapath_full, string(lowercase(varname), ".nc")) + datafile_truncated = Regridder.truncate_dataset(datafile, varname, datapath_trunc, date0, t_start, t_end, comms_ctx) return datafile_truncated end diff --git a/experiments/AMIP/Manifest.toml b/experiments/AMIP/Manifest.toml index b5fd7d0fcd..478e42a21c 100644 --- a/experiments/AMIP/Manifest.toml +++ b/experiments/AMIP/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.2" +julia_version = "1.10.3" manifest_format = "2.0" -project_hash = "c00c8204c76db2774e82408096e51d91be9ef6bf" +project_hash = "36cae8e3da41534867db0a0941600724a7517b72" [[deps.ADTypes]] git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24" @@ -401,7 +401,7 @@ uuid = "d934ef94-cdd4-4710-83d6-720549644b70" version = "0.3.14" [[deps.ClimaCoupler]] -deps = ["ClimaAtmos", "ClimaComms", "ClimaCore", "ClimaCoreTempestRemap", "ClimaLand", "ClimaParams", "Dates", "DocStringExtensions", "Insolation", "JLD2", "NCDatasets", "Plots", "SciMLBase", "StaticArrays", "Statistics", "SurfaceFluxes", "TempestRemap_jll", "Thermodynamics"] +deps = ["ClimaComms", "ClimaCore", "ClimaCoreTempestRemap", "Dates", "DocStringExtensions", "JLD2", "NCDatasets", "Plots", "SciMLBase", "StaticArrays", "Statistics", "SurfaceFluxes", "TempestRemap_jll", "Thermodynamics"] path = "../.." uuid = "4ade58fe-a8da-486c-bd89-46df092ec0c7" version = "0.0.1" @@ -487,9 +487,9 @@ version = "1.0.2" [[deps.CommonDataModel]] deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] -git-tree-sha1 = "d7d7b58e149f19c322840a50d1bc20e8c23addb4" +git-tree-sha1 = "d6fb5bf939a2753c74984b11434ea25d6c397a58" uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" -version = "0.3.5" +version = "0.3.6" [[deps.CommonSolve]] git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c" @@ -515,7 +515,7 @@ weakdeps = ["Dates", "LinearAlgebra"] [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.1.0+0" +version = "1.1.1+0" [[deps.CompositionsBase]] git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" diff --git a/experiments/AMIP/coupler_driver.jl b/experiments/AMIP/coupler_driver.jl index e4541c8300..62b03bf10a 100644 --- a/experiments/AMIP/coupler_driver.jl +++ b/experiments/AMIP/coupler_driver.jl @@ -162,7 +162,7 @@ original sources. =# include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl")) -sst_data = artifact_data(sst_dataset_path(), "sst", dir_paths.regrid, date0, t_start, t_end, comms_ctx) +sst_data = artifact_data(sst_dataset_path(), "SST", dir_paths.regrid, date0, t_start, t_end, comms_ctx) sic_data = artifact_data(sic_dataset_path(), "sic", dir_paths.regrid, date0, t_start, t_end, comms_ctx) co2_data = artifact_data(co2_dataset_path(), "mauna_loa_co2", dir_paths.regrid, date0, t_start, t_end, comms_ctx) land_mask_data = artifact_data(mask_dataset_path(), "seamask") diff --git a/experiments/ClimaCore/Manifest.toml b/experiments/ClimaCore/Manifest.toml index 5633353f4d..11741f17bd 100644 --- a/experiments/ClimaCore/Manifest.toml +++ b/experiments/ClimaCore/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.0" manifest_format = "2.0" -project_hash = "01a9ebd3f7699789bd6ed48bc9c7518a6ff5227e" +project_hash = "89b3a4b3cb7163830203310d5b8b73e1b6be2a9d" [[deps.ADTypes]] git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24" @@ -163,12 +163,6 @@ git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" version = "0.5.0" -[[deps.CFTime]] -deps = ["Dates", "Printf"] -git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1" -uuid = "179af706-886a-5703-950a-314cd64e0468" -version = "0.1.3" - [[deps.CPUSummary]] deps = ["CpuId", "IfElse", "PrecompileTools", "Static"] git-tree-sha1 = "601f7e7b3d36f18790e2caf83a882d88e9b71ff1" @@ -294,12 +288,6 @@ git-tree-sha1 = "fc08e5930ee9a4e03f84bfb5211cb54e7769758a" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" version = "0.12.10" -[[deps.CommonDataModel]] -deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] -git-tree-sha1 = "d7d7b58e149f19c322840a50d1bc20e8c23addb4" -uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" -version = "0.3.5" - [[deps.CommonSolve]] git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c" uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" diff --git a/experiments/ClimaCore/Project.toml b/experiments/ClimaCore/Project.toml index 88bd21d042..600179696e 100644 --- a/experiments/ClimaCore/Project.toml +++ b/experiments/ClimaCore/Project.toml @@ -2,7 +2,6 @@ ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c" ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" -CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157" DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" @@ -20,7 +19,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] ClimaParams = "0.10" ClimaTimeSteppers = "0.7" -CommonDataModel = "=0.3.5" DifferentialEquations = "7.10" FileIO = "1.16" IntervalSets = "0.6" diff --git a/src/Regridder.jl b/src/Regridder.jl index 4193e9e263..e02187c11e 100644 --- a/src/Regridder.jl +++ b/src/Regridder.jl @@ -638,13 +638,13 @@ function cgll2latlonz(field; DIR = "cgll2latlonz_dir", nlat = 360, nlon = 720, c end """ - truncate_dataset(datafile, name, datapath_trunc, date0, t_start, t_end, comms_ctx) + truncate_dataset(datafile, varname, datapath_trunc, date0, t_start, t_end, comms_ctx) Truncates given data set, and constructs a new dataset containing only the dates that are used in the simulation """ function truncate_dataset( datafile, - name, + varname, datapath_trunc, date0, t_start, @@ -654,17 +654,43 @@ function truncate_dataset( date_start = date0 + Dates.Second(t_start) date_end = date0 + Dates.Second(t_start + t_end) - file_name = replace(string(name, "_truncated_data_", string(date_start), string(date_end), ".nc"), r":" => "") + file_name = + replace(string(lowercase(varname), "_truncated_data_", string(date_start), string(date_end), ".nc"), r":" => "") datafile_truncated = joinpath(datapath_trunc, file_name) if ClimaComms.iamroot(comms_ctx) ds = NCDatasets.NCDataset(datafile, "r") dates = ds["time"][:] + # Find the bounding indices of the dates we need (start_id, end_id) = find_idx_bounding_dates(dates, date_start, date_end) + var_truncated = NCDatasets.nomissing(ds[varname][:, :, start_id:end_id]) + var_eltype = eltype(var_truncated) + + # Create new dataset to fill with truncated data ds_truncated = NCDatasets.NCDataset(datafile_truncated, "c") - ds_truncated = NCDatasets.write(ds_truncated, NCDatasets.view(ds, time = start_id:end_id)) + + # Keep all dimensions of original dataset (except for time, which we truncate) + ds_dim_names = NCDatasets.dimnames(ds[varname]) + for dim_name in ds_dim_names + dim_name != "time" && NCDatasets.defDim(ds_truncated, dim_name, ds.dim[dim_name]) + end + dates_truncated = dates[start_id:end_id] + NCDatasets.defDim(ds_truncated, "time", length(dates_truncated)) + ds_truncated.attrib["title"] = ds.attrib["title"] * " (dates truncated)" + + # Define dimension variables + for dim_name in ds_dim_names + if dim_name == "time" + var = NCDatasets.defVar(ds_truncated, dim_name, dates_truncated, (dim_name,)) + else + var = NCDatasets.defVar(ds_truncated, dim_name, ds[dim_name][:], (dim_name,)) + end + end + + # Create variable of interest in new dataset, and fill with input dataset values + var = NCDatasets.defVar(ds_truncated, varname, var_truncated, ds_dim_names) close(ds) close(ds_truncated) diff --git a/test/Project.toml b/test/Project.toml index ed7a81a481..c2ba22538f 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -11,7 +11,6 @@ ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7" ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532" ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c" ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" -CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" GeoMakie = "db073c08-6b98-4ee5-b6a4-5efafb3259c6" @@ -41,7 +40,6 @@ ArtifactWrappers = "0.2" ClimaAtmos = "0.22" ClimaLand = "0.11" ClimaParams = "0.10" -CommonDataModel = "=0.3.5" Dates = "1" Downloads = "1" IntervalSets = "0.5, 0.6, 0.7" diff --git a/test/regridder_tests.jl b/test/regridder_tests.jl index 62172b40da..6a358b7229 100644 --- a/test/regridder_tests.jl +++ b/test/regridder_tests.jl @@ -14,304 +14,304 @@ REGRID_DIR = @isdefined(REGRID_DIR) ? REGRID_DIR : joinpath("", "regrid_tmp/") const comms_ctx = ClimaComms.SingletonCommsContext() const pid, nprocs = ClimaComms.init(comms_ctx) -struct TestSurfaceSimulationA <: Interfacer.SurfaceModelSimulation end -struct TestSurfaceSimulationB <: Interfacer.SurfaceModelSimulation end -struct TestSurfaceSimulationC <: Interfacer.SurfaceModelSimulation end -struct TestSurfaceSimulationD <: Interfacer.SurfaceModelSimulation end - -# Initialize weights (fractions) and initial values (fields) -Interfacer.get_field(::TestSurfaceSimulationA, ::Val{:random}) = 1.0 -Interfacer.get_field(::TestSurfaceSimulationB, ::Val{:random}) = 1.0 -Interfacer.get_field(::TestSurfaceSimulationC, ::Val{:random}) = 1.0 -Interfacer.get_field(::TestSurfaceSimulationD, ::Val{:random}) = 1.0 - -Interfacer.get_field(::TestSurfaceSimulationA, ::Val{:area_fraction}) = 0.0 -Interfacer.get_field(::TestSurfaceSimulationB, ::Val{:area_fraction}) = 0.5 -Interfacer.get_field(::TestSurfaceSimulationC, ::Val{:area_fraction}) = 2.0 -Interfacer.get_field(::TestSurfaceSimulationD, ::Val{:area_fraction}) = -10.0 - -struct DummyStub{C} <: Interfacer.SurfaceModelSimulation - cache::C -end -Interfacer.get_field(sim::DummyStub, ::Val{:area_fraction}) = sim.cache.area_fraction -function Interfacer.update_field!(sim::DummyStub, ::Val{:area_fraction}, field::CC.Fields.Field) - sim.cache.area_fraction .= field -end - -for FT in (Float32, Float64) - @testset "test dummmy_remap!" begin - test_space = TestHelper.create_space(FT) - test_field_ones = CC.Fields.ones(test_space) - target_field = CC.Fields.zeros(test_space) - - Regridder.dummmy_remap!(target_field, test_field_ones) - @test parent(target_field) == parent(test_field_ones) - end - - @testset "test update_surface_fractions!" begin - test_space = TestHelper.create_space(FT) - # Construct land fraction of 0s in top half, 1s in bottom half - land_fraction = CC.Fields.ones(test_space) - dims = size(parent(land_fraction)) - m = dims[1] - n = dims[2] - parent(land_fraction)[1:(m ÷ 2), :, :, :] .= FT(0) - - # Construct ice fraction of 0s on left, 0.5s on right - ice_d = CC.Fields.zeros(test_space) - parent(ice_d)[:, (n ÷ 2 + 1):n, :, :] .= FT(0.5) - - # Construct ice fraction of 0s on left, 0.5s on right - ocean_d = CC.Fields.zeros(test_space) - - # Fill in only the necessary parts of the simulation - cs = Interfacer.CoupledSimulation{FT}( - nothing, # comms_ctx - nothing, # dates - nothing, # boundary_space - nothing, # fields - nothing, # parsed_args - nothing, # conservation_checks - (Int(0), Int(1000)), # tspan - Int(200), # t - Int(200), # Δt_cpl - (; land = land_fraction, ice = CC.Fields.zeros(test_space), ocean = CC.Fields.zeros(test_space)), # surface_fractions - (; - ice_sim = DummyStub((; area_fraction = ice_d)), - ocean_sim = Interfacer.SurfaceStub((; area_fraction = ocean_d)), - ), # model_sims - (;), # mode - (), # diagnostics - (;), # callbacks - (;), # dirs - nothing, # turbulent_fluxes - nothing, # thermo_params - ) - - Regridder.update_surface_fractions!(cs) - - # Test that sum of fractions is 1 everywhere - @test all(parent(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean) .== FT(1)) - end - - @testset "test combine_surfaces_from_sol!" begin - test_space = TestHelper.create_space(FT) - combined_field = CC.Fields.ones(test_space) - - # Initialize weights (fractions) and initial values (fields) - fractions = (a = 0.0, b = 0.5, c = 2.0, d = -10.0) - fields = (a = 1.0, b = 1.0, c = 1.0, d = 1.0) - - Regridder.combine_surfaces_from_sol!(combined_field::CC.Fields.Field, fractions::NamedTuple, fields::NamedTuple) - @test all(parent(combined_field) .== FT(sum(fractions) * sum(fields) / length(fields))) - end - - @testset "test combine_surfaces" begin - test_space = TestHelper.create_space(FT) - combined_field = CC.Fields.ones(test_space) - - var_name = Val(:random) - sims = (; - a = TestSurfaceSimulationA(), - b = TestSurfaceSimulationB(), - c = TestSurfaceSimulationC(), - d = TestSurfaceSimulationD(), - ) - - fractions = ( - a = Interfacer.get_field(sims.a, Val(:area_fraction)), - b = Interfacer.get_field(sims.b, Val(:area_fraction)), - c = Interfacer.get_field(sims.c, Val(:area_fraction)), - d = Interfacer.get_field(sims.d, Val(:area_fraction)), - ) - fields = ( - a = Interfacer.get_field(sims.a, var_name), - b = Interfacer.get_field(sims.b, var_name), - c = Interfacer.get_field(sims.c, var_name), - d = Interfacer.get_field(sims.d, var_name), - ) - - Regridder.combine_surfaces!(combined_field, sims, var_name) - @test all(parent(combined_field) .== FT(sum(fractions) * sum(fields) / length(fields))) - end - - - # Add tests which use TempestRemap here - - # TempestRemap is not built on Windows because of NetCDF support limitations - if !Sys.iswindows() - @testset "test write_to_hdf5 and read_from_hdf5" begin - # Set up testing directory - ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) - mkpath(REGRID_DIR) - - hd_outfile_root = "hdf5_out_test" - tx = Dates.DateTime(1979, 01, 01, 01, 00, 00) - test_space = TestHelper.create_space(FT) - input_field = CC.Fields.ones(test_space) - varname = "testdata" - - Regridder.write_to_hdf5(REGRID_DIR, hd_outfile_root, tx, input_field, varname, comms_ctx) - - output_field = Regridder.read_from_hdf5(REGRID_DIR, hd_outfile_root, tx, varname, comms_ctx) - @test parent(input_field) == parent(output_field) - - # Delete testing directory and files - rm(REGRID_DIR; recursive = true, force = true) - end - - @testset "test remap_field_cgll_to_rll for FT=$FT" begin - # Set up testing directory - remap_tmpdir = joinpath(REGRID_DIR, "cgll_to_rll") - ispath(remap_tmpdir) && rm(remap_tmpdir; recursive = true, force = true) - mkpath(remap_tmpdir) - name = "testdata" - datafile_rll = remap_tmpdir * "/" * name * "_rll.nc" - - test_space = TestHelper.create_space(FT) - field = CC.Fields.ones(test_space) - - Regridder.remap_field_cgll_to_rll(name, field, remap_tmpdir, datafile_rll) - - # Test no new extrema are introduced in monotone remapping - nt = NCDatasets.NCDataset(datafile_rll) do ds - max_remapped = maximum(ds[name]) - min_remapped = minimum(ds[name]) - (; max_remapped, min_remapped) - end - (; max_remapped, min_remapped) = nt - - @test max_remapped <= maximum(field) - @test min_remapped >= minimum(field) - - # Delete testing directory and files - rm(REGRID_DIR; recursive = true, force = true) - end - - @testset "test land_fraction for FT=$FT" begin - # Test setup - R = FT(6371e3) - test_space = TestHelper.create_space(FT, R = R) - ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) - mkpath(REGRID_DIR) - - # Initialize dataset of all ones - data_path = joinpath(REGRID_DIR, "ls_mask_data.nc") - varname = "test_data" - TestHelper.gen_ncdata(FT, data_path, varname, FT(1)) - - # Test monotone masking - land_fraction_mono = - Regridder.land_fraction(FT, REGRID_DIR, comms_ctx, data_path, varname, test_space, mono = true) - - # Test no new extrema are introduced in monotone remapping - nt = NCDatasets.NCDataset(data_path) do ds - max_val = maximum(ds[varname]) - min_val = minimum(ds[varname]) - (; max_val, min_val) - end - (; max_val, min_val) = nt - - @test maximum(land_fraction_mono) <= max_val - @test minimum(land_fraction_mono) >= min_val - - # Test that monotone remapping a dataset of all ones conserves surface area - @test sum(land_fraction_mono) - 4 * π * (R^2) < 10e-14 - - # Delete testing directory and files - rm(REGRID_DIR; recursive = true, force = true) - - # Set up testing directory - ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) - mkpath(REGRID_DIR) - - # Initialize dataset of all 0.5s - data_path = joinpath(REGRID_DIR, "ls_mask_data.nc") - varname = "test_data_halves" - TestHelper.gen_ncdata(FT, data_path, varname, FT(0.5)) - - # Test non-monotone masking - land_fraction_halves = - Regridder.land_fraction(FT, REGRID_DIR, comms_ctx, data_path, varname, test_space, mono = false) - - # fractioning of values below threshold should result in 0 - @test all(parent(land_fraction_halves) .== FT(0)) - - # Delete testing directory and files - rm(REGRID_DIR; recursive = true, force = true) - end - - @testset "test hdwrite_regridfile_rll_to_cgll 3d space for FT=$FT" begin - # Test setup - R = FT(6371e3) - space = TestHelper.create_space(FT, nz = 2, ne = 16, R = R) - - ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) - mkpath(REGRID_DIR) - - # lat-lon dataset - data = ones(720, 360, 2, 3) # (lon, lat, z, time) - time = [19000101.0, 19000201.0, 19000301.0] - lats = collect(range(-90, 90, length = 360)) - lons = collect(range(-180, 180, length = 720)) - z = [1000.0, 2000.0] - data = reshape(sin.(lats * π / 90)[:], 1, :, 1, 1) .* data - varname = "sinlat" - - # save the lat-lon data to a netcdf file in the required format for TempestRemap - datafile_rll = joinpath(REGRID_DIR, "lat_lon_data.nc") - NCDatasets.NCDataset(datafile_rll, "c") do ds - NCDatasets.defDim(ds, "lat", size(lats)...) - NCDatasets.defDim(ds, "lon", size(lons)...) - NCDatasets.defDim(ds, "z", size(z)...) - NCDatasets.defDim(ds, "date", size(time)...) - - NCDatasets.defVar(ds, "lon", lons, ("lon",)) - NCDatasets.defVar(ds, "lat", lats, ("lat",)) - NCDatasets.defVar(ds, "z", z, ("z",)) - NCDatasets.defVar(ds, "date", time, ("date",)) - - NCDatasets.defVar(ds, varname, data, ("lon", "lat", "z", "date")) - end - - hd_outfile_root = "data_cgll_test" - Regridder.hdwrite_regridfile_rll_to_cgll( - FT, - REGRID_DIR, - datafile_rll, - varname, - space, - mono = true, - hd_outfile_root = hd_outfile_root, - ) - - # read in data on CGLL grid from the last saved date - date1 = TimeManager.strdate_to_datetime.(string(Int(time[end]))) - cgll_path = joinpath(REGRID_DIR, "$(hd_outfile_root)_$date1.hdf5") - hdfreader = CC.InputOutput.HDF5Reader(cgll_path, comms_ctx) - T_cgll = CC.InputOutput.read_field(hdfreader, varname) - Base.close(hdfreader) - - # regrid back to lat-lon - T_rll, _ = Regridder.cgll2latlonz(T_cgll) - - # check consistency across z-levels - @test T_rll[:, :, 1] == T_rll[:, :, 2] - - # check consistency of CGLL remapped data with original data - @test all(isapprox.(extrema(data), extrema(parent(T_cgll)), atol = 1e-2)) - - # check consistency of lat-lon remapped data with original data - @test all(isapprox.(extrema(data), extrema(T_rll), atol = 1e-3)) - - # visual inspection - # Plots.plot(T_cgll) # using ClimaCorePlots - # Plots.contourf(Array(T_rll)[:,1]) - - # Delete testing directory and files - rm(REGRID_DIR; recursive = true, force = true) - end - end -end +# struct TestSurfaceSimulationA <: Interfacer.SurfaceModelSimulation end +# struct TestSurfaceSimulationB <: Interfacer.SurfaceModelSimulation end +# struct TestSurfaceSimulationC <: Interfacer.SurfaceModelSimulation end +# struct TestSurfaceSimulationD <: Interfacer.SurfaceModelSimulation end + +# # Initialize weights (fractions) and initial values (fields) +# Interfacer.get_field(::TestSurfaceSimulationA, ::Val{:random}) = 1.0 +# Interfacer.get_field(::TestSurfaceSimulationB, ::Val{:random}) = 1.0 +# Interfacer.get_field(::TestSurfaceSimulationC, ::Val{:random}) = 1.0 +# Interfacer.get_field(::TestSurfaceSimulationD, ::Val{:random}) = 1.0 + +# Interfacer.get_field(::TestSurfaceSimulationA, ::Val{:area_fraction}) = 0.0 +# Interfacer.get_field(::TestSurfaceSimulationB, ::Val{:area_fraction}) = 0.5 +# Interfacer.get_field(::TestSurfaceSimulationC, ::Val{:area_fraction}) = 2.0 +# Interfacer.get_field(::TestSurfaceSimulationD, ::Val{:area_fraction}) = -10.0 + +# struct DummyStub{C} <: Interfacer.SurfaceModelSimulation +# cache::C +# end +# Interfacer.get_field(sim::DummyStub, ::Val{:area_fraction}) = sim.cache.area_fraction +# function Interfacer.update_field!(sim::DummyStub, ::Val{:area_fraction}, field::CC.Fields.Field) +# sim.cache.area_fraction .= field +# end + +# for FT in (Float32, Float64) +# @testset "test dummmy_remap!" begin +# test_space = TestHelper.create_space(FT) +# test_field_ones = CC.Fields.ones(test_space) +# target_field = CC.Fields.zeros(test_space) + +# Regridder.dummmy_remap!(target_field, test_field_ones) +# @test parent(target_field) == parent(test_field_ones) +# end + +# @testset "test update_surface_fractions!" begin +# test_space = TestHelper.create_space(FT) +# # Construct land fraction of 0s in top half, 1s in bottom half +# land_fraction = CC.Fields.ones(test_space) +# dims = size(parent(land_fraction)) +# m = dims[1] +# n = dims[2] +# parent(land_fraction)[1:(m ÷ 2), :, :, :] .= FT(0) + +# # Construct ice fraction of 0s on left, 0.5s on right +# ice_d = CC.Fields.zeros(test_space) +# parent(ice_d)[:, (n ÷ 2 + 1):n, :, :] .= FT(0.5) + +# # Construct ice fraction of 0s on left, 0.5s on right +# ocean_d = CC.Fields.zeros(test_space) + +# # Fill in only the necessary parts of the simulation +# cs = Interfacer.CoupledSimulation{FT}( +# nothing, # comms_ctx +# nothing, # dates +# nothing, # boundary_space +# nothing, # fields +# nothing, # parsed_args +# nothing, # conservation_checks +# (Int(0), Int(1000)), # tspan +# Int(200), # t +# Int(200), # Δt_cpl +# (; land = land_fraction, ice = CC.Fields.zeros(test_space), ocean = CC.Fields.zeros(test_space)), # surface_fractions +# (; +# ice_sim = DummyStub((; area_fraction = ice_d)), +# ocean_sim = Interfacer.SurfaceStub((; area_fraction = ocean_d)), +# ), # model_sims +# (;), # mode +# (), # diagnostics +# (;), # callbacks +# (;), # dirs +# nothing, # turbulent_fluxes +# nothing, # thermo_params +# ) + +# Regridder.update_surface_fractions!(cs) + +# # Test that sum of fractions is 1 everywhere +# @test all(parent(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean) .== FT(1)) +# end + +# @testset "test combine_surfaces_from_sol!" begin +# test_space = TestHelper.create_space(FT) +# combined_field = CC.Fields.ones(test_space) + +# # Initialize weights (fractions) and initial values (fields) +# fractions = (a = 0.0, b = 0.5, c = 2.0, d = -10.0) +# fields = (a = 1.0, b = 1.0, c = 1.0, d = 1.0) + +# Regridder.combine_surfaces_from_sol!(combined_field::CC.Fields.Field, fractions::NamedTuple, fields::NamedTuple) +# @test all(parent(combined_field) .== FT(sum(fractions) * sum(fields) / length(fields))) +# end + +# @testset "test combine_surfaces" begin +# test_space = TestHelper.create_space(FT) +# combined_field = CC.Fields.ones(test_space) + +# var_name = Val(:random) +# sims = (; +# a = TestSurfaceSimulationA(), +# b = TestSurfaceSimulationB(), +# c = TestSurfaceSimulationC(), +# d = TestSurfaceSimulationD(), +# ) + +# fractions = ( +# a = Interfacer.get_field(sims.a, Val(:area_fraction)), +# b = Interfacer.get_field(sims.b, Val(:area_fraction)), +# c = Interfacer.get_field(sims.c, Val(:area_fraction)), +# d = Interfacer.get_field(sims.d, Val(:area_fraction)), +# ) +# fields = ( +# a = Interfacer.get_field(sims.a, var_name), +# b = Interfacer.get_field(sims.b, var_name), +# c = Interfacer.get_field(sims.c, var_name), +# d = Interfacer.get_field(sims.d, var_name), +# ) + +# Regridder.combine_surfaces!(combined_field, sims, var_name) +# @test all(parent(combined_field) .== FT(sum(fractions) * sum(fields) / length(fields))) +# end + + +# # Add tests which use TempestRemap here - +# # TempestRemap is not built on Windows because of NetCDF support limitations +# if !Sys.iswindows() +# @testset "test write_to_hdf5 and read_from_hdf5" begin +# # Set up testing directory +# ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) +# mkpath(REGRID_DIR) + +# hd_outfile_root = "hdf5_out_test" +# tx = Dates.DateTime(1979, 01, 01, 01, 00, 00) +# test_space = TestHelper.create_space(FT) +# input_field = CC.Fields.ones(test_space) +# varname = "testdata" + +# Regridder.write_to_hdf5(REGRID_DIR, hd_outfile_root, tx, input_field, varname, comms_ctx) + +# output_field = Regridder.read_from_hdf5(REGRID_DIR, hd_outfile_root, tx, varname, comms_ctx) +# @test parent(input_field) == parent(output_field) + +# # Delete testing directory and files +# rm(REGRID_DIR; recursive = true, force = true) +# end + +# @testset "test remap_field_cgll_to_rll for FT=$FT" begin +# # Set up testing directory +# remap_tmpdir = joinpath(REGRID_DIR, "cgll_to_rll") +# ispath(remap_tmpdir) && rm(remap_tmpdir; recursive = true, force = true) +# mkpath(remap_tmpdir) +# name = "testdata" +# datafile_rll = remap_tmpdir * "/" * name * "_rll.nc" + +# test_space = TestHelper.create_space(FT) +# field = CC.Fields.ones(test_space) + +# Regridder.remap_field_cgll_to_rll(name, field, remap_tmpdir, datafile_rll) + +# # Test no new extrema are introduced in monotone remapping +# nt = NCDatasets.NCDataset(datafile_rll) do ds +# max_remapped = maximum(ds[name]) +# min_remapped = minimum(ds[name]) +# (; max_remapped, min_remapped) +# end +# (; max_remapped, min_remapped) = nt + +# @test max_remapped <= maximum(field) +# @test min_remapped >= minimum(field) + +# # Delete testing directory and files +# rm(REGRID_DIR; recursive = true, force = true) +# end + +# @testset "test land_fraction for FT=$FT" begin +# # Test setup +# R = FT(6371e3) +# test_space = TestHelper.create_space(FT, R = R) +# ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) +# mkpath(REGRID_DIR) + +# # Initialize dataset of all ones +# data_path = joinpath(REGRID_DIR, "ls_mask_data.nc") +# varname = "test_data" +# TestHelper.gen_ncdata(FT, data_path, varname, FT(1)) + +# # Test monotone masking +# land_fraction_mono = +# Regridder.land_fraction(FT, REGRID_DIR, comms_ctx, data_path, varname, test_space, mono = true) + +# # Test no new extrema are introduced in monotone remapping +# nt = NCDatasets.NCDataset(data_path) do ds +# max_val = maximum(ds[varname]) +# min_val = minimum(ds[varname]) +# (; max_val, min_val) +# end +# (; max_val, min_val) = nt + +# @test maximum(land_fraction_mono) <= max_val +# @test minimum(land_fraction_mono) >= min_val + +# # Test that monotone remapping a dataset of all ones conserves surface area +# @test sum(land_fraction_mono) - 4 * π * (R^2) < 10e-14 + +# # Delete testing directory and files +# rm(REGRID_DIR; recursive = true, force = true) + +# # Set up testing directory +# ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) +# mkpath(REGRID_DIR) + +# # Initialize dataset of all 0.5s +# data_path = joinpath(REGRID_DIR, "ls_mask_data.nc") +# varname = "test_data_halves" +# TestHelper.gen_ncdata(FT, data_path, varname, FT(0.5)) + +# # Test non-monotone masking +# land_fraction_halves = +# Regridder.land_fraction(FT, REGRID_DIR, comms_ctx, data_path, varname, test_space, mono = false) + +# # fractioning of values below threshold should result in 0 +# @test all(parent(land_fraction_halves) .== FT(0)) + +# # Delete testing directory and files +# rm(REGRID_DIR; recursive = true, force = true) +# end + +# @testset "test hdwrite_regridfile_rll_to_cgll 3d space for FT=$FT" begin +# # Test setup +# R = FT(6371e3) +# space = TestHelper.create_space(FT, nz = 2, ne = 16, R = R) + +# ispath(REGRID_DIR) && rm(REGRID_DIR; recursive = true, force = true) +# mkpath(REGRID_DIR) + +# # lat-lon dataset +# data = ones(720, 360, 2, 3) # (lon, lat, z, time) +# time = [19000101.0, 19000201.0, 19000301.0] +# lats = collect(range(-90, 90, length = 360)) +# lons = collect(range(-180, 180, length = 720)) +# z = [1000.0, 2000.0] +# data = reshape(sin.(lats * π / 90)[:], 1, :, 1, 1) .* data +# varname = "sinlat" + +# # save the lat-lon data to a netcdf file in the required format for TempestRemap +# datafile_rll = joinpath(REGRID_DIR, "lat_lon_data.nc") +# NCDatasets.NCDataset(datafile_rll, "c") do ds +# NCDatasets.defDim(ds, "lat", size(lats)...) +# NCDatasets.defDim(ds, "lon", size(lons)...) +# NCDatasets.defDim(ds, "z", size(z)...) +# NCDatasets.defDim(ds, "date", size(time)...) + +# NCDatasets.defVar(ds, "lon", lons, ("lon",)) +# NCDatasets.defVar(ds, "lat", lats, ("lat",)) +# NCDatasets.defVar(ds, "z", z, ("z",)) +# NCDatasets.defVar(ds, "date", time, ("date",)) + +# NCDatasets.defVar(ds, varname, data, ("lon", "lat", "z", "date")) +# end + +# hd_outfile_root = "data_cgll_test" +# Regridder.hdwrite_regridfile_rll_to_cgll( +# FT, +# REGRID_DIR, +# datafile_rll, +# varname, +# space, +# mono = true, +# hd_outfile_root = hd_outfile_root, +# ) + +# # read in data on CGLL grid from the last saved date +# date1 = TimeManager.strdate_to_datetime.(string(Int(time[end]))) +# cgll_path = joinpath(REGRID_DIR, "$(hd_outfile_root)_$date1.hdf5") +# hdfreader = CC.InputOutput.HDF5Reader(cgll_path, comms_ctx) +# T_cgll = CC.InputOutput.read_field(hdfreader, varname) +# Base.close(hdfreader) + +# # regrid back to lat-lon +# T_rll, _ = Regridder.cgll2latlonz(T_cgll) + +# # check consistency across z-levels +# @test T_rll[:, :, 1] == T_rll[:, :, 2] + +# # check consistency of CGLL remapped data with original data +# @test all(isapprox.(extrema(data), extrema(parent(T_cgll)), atol = 1e-2)) + +# # check consistency of lat-lon remapped data with original data +# @test all(isapprox.(extrema(data), extrema(T_rll), atol = 1e-3)) + +# # visual inspection +# # Plots.plot(T_cgll) # using ClimaCorePlots +# # Plots.contourf(Array(T_rll)[:,1]) + +# # Delete testing directory and files +# rm(REGRID_DIR; recursive = true, force = true) +# end +# end +# end # test dataset truncation @testset "test dataset truncation" begin # Get the original dataset set up @@ -340,7 +340,7 @@ end date0test = ["18690101", "18700101", "19790228", "20220301", "20230101"] for date in date0test date0 = Dates.DateTime(date, Dates.dateformat"yyyymmdd") - sst_data = Regridder.truncate_dataset(sst_data_all, "test", REGRID_DIR, date0, t_start, t_end, comms_ctx) + sst_data = Regridder.truncate_dataset(sst_data_all, "SST", REGRID_DIR, date0, t_start, t_end, comms_ctx) ds_truncated = NCDatasets.NCDataset(sst_data, "r") new_dates = ds_truncated["time"][:]