don't use NCD write

artifacts/artifact_funcs.jl

@@ -61,22 +61,22 @@ function pr_obs_data_path()
 end
 
 """
-    artifact_data(datapath_full, name)
+    artifact_data(datapath_full, varname)
 
 Returns input dataset at datapath_full
 """
-function artifact_data(datapath_full, name)
-    datafile_truncated = joinpath(datapath_full, string(name, ".nc"))
+function artifact_data(datapath_full, varname)
+    datafile_truncated = joinpath(datapath_full, string(lowercase(varname), ".nc"))
     return datafile_truncated
 end
 
 """
-    artifact_data(datapath_full, name, datapath_trunc, date0, t_start, t_end, comms_ctx)
+    artifact_data(datapath_full, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
 
 Truncates given data set, and constructs a new dataset containing only the dates needed and stores it in datapath_trunc
 """
-function artifact_data(datapath_full, name, datapath_trunc, date0, t_start, t_end, comms_ctx)
-    datafile = joinpath(datapath_full, string(name, ".nc"))
-    datafile_truncated = Regridder.truncate_dataset(datafile, name, datapath_trunc, date0, t_start, t_end, comms_ctx)
+function artifact_data(datapath_full, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
+    datafile = joinpath(datapath_full, string(lowercase(varname), ".nc"))
+    datafile_truncated = Regridder.truncate_dataset(datafile, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
     return datafile_truncated
 end

experiments/AMIP/Manifest.toml

@@ -1,8 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.2"
+julia_version = "1.10.3"
 manifest_format = "2.0"
-project_hash = "c00c8204c76db2774e82408096e51d91be9ef6bf"
+project_hash = "36cae8e3da41534867db0a0941600724a7517b72"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24"
@@ -401,7 +401,7 @@ uuid = "d934ef94-cdd4-4710-83d6-720549644b70"
 version = "0.3.14"
 
 [[deps.ClimaCoupler]]
-deps = ["ClimaAtmos", "ClimaComms", "ClimaCore", "ClimaCoreTempestRemap", "ClimaLand", "ClimaParams", "Dates", "DocStringExtensions", "Insolation", "JLD2", "NCDatasets", "Plots", "SciMLBase", "StaticArrays", "Statistics", "SurfaceFluxes", "TempestRemap_jll", "Thermodynamics"]
+deps = ["ClimaComms", "ClimaCore", "ClimaCoreTempestRemap", "Dates", "DocStringExtensions", "JLD2", "NCDatasets", "Plots", "SciMLBase", "StaticArrays", "Statistics", "SurfaceFluxes", "TempestRemap_jll", "Thermodynamics"]
 path = "../.."
 uuid = "4ade58fe-a8da-486c-bd89-46df092ec0c7"
 version = "0.0.1"
@@ -487,9 +487,9 @@ version = "1.0.2"
 
 [[deps.CommonDataModel]]
 deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"]
-git-tree-sha1 = "d7d7b58e149f19c322840a50d1bc20e8c23addb4"
+git-tree-sha1 = "d6fb5bf939a2753c74984b11434ea25d6c397a58"
 uuid = "1fbeeb36-5f17-413c-809b-666fb144f157"
-version = "0.3.5"
+version = "0.3.6"
 
 [[deps.CommonSolve]]
 git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c"
@@ -515,7 +515,7 @@ weakdeps = ["Dates", "LinearAlgebra"]
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "1.1.0+0"
+version = "1.1.1+0"
 
 [[deps.CompositionsBase]]
 git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad"

experiments/AMIP/coupler_driver.jl

@@ -162,8 +162,8 @@ original sources.
 =#
 
 include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl"))
-sst_data = artifact_data(sst_dataset_path(), "sst", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
-sic_data = artifact_data(sic_dataset_path(), "sic", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
+sst_data = artifact_data(sst_dataset_path(), "SST", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
+sic_data = artifact_data(sic_dataset_path(), "SEAICE", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
 co2_data = artifact_data(co2_dataset_path(), "mauna_loa_co2", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
 land_mask_data = artifact_data(mask_dataset_path(), "seamask")
 

experiments/ClimaCore/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.0"
 manifest_format = "2.0"
-project_hash = "01a9ebd3f7699789bd6ed48bc9c7518a6ff5227e"
+project_hash = "89b3a4b3cb7163830203310d5b8b73e1b6be2a9d"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24"
@@ -163,12 +163,6 @@ git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc"
 uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 version = "0.5.0"
 
-[[deps.CFTime]]
-deps = ["Dates", "Printf"]
-git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1"
-uuid = "179af706-886a-5703-950a-314cd64e0468"
-version = "0.1.3"
-
 [[deps.CPUSummary]]
 deps = ["CpuId", "IfElse", "PrecompileTools", "Static"]
 git-tree-sha1 = "601f7e7b3d36f18790e2caf83a882d88e9b71ff1"
@@ -294,12 +288,6 @@ git-tree-sha1 = "fc08e5930ee9a4e03f84bfb5211cb54e7769758a"
 uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
 version = "0.12.10"
 
-[[deps.CommonDataModel]]
-deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"]
-git-tree-sha1 = "d7d7b58e149f19c322840a50d1bc20e8c23addb4"
-uuid = "1fbeeb36-5f17-413c-809b-666fb144f157"
-version = "0.3.5"
-
 [[deps.CommonSolve]]
 git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c"
 uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"

experiments/ClimaCore/Project.toml

@@ -2,7 +2,6 @@
 ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884"
 ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
 DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
@@ -20,7 +19,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 [compat]
 ClimaParams = "0.10"
 ClimaTimeSteppers = "0.7"
-CommonDataModel = "=0.3.5"
 DifferentialEquations = "7.10"
 FileIO = "1.16"
 IntervalSets = "0.6"

src/Regridder.jl

@@ -638,13 +638,13 @@ function cgll2latlonz(field; DIR = "cgll2latlonz_dir", nlat = 360, nlon = 720, c
 end
 
 """
-    truncate_dataset(datafile, name, datapath_trunc, date0, t_start, t_end, comms_ctx)
+    truncate_dataset(datafile, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
 
 Truncates given data set, and constructs a new dataset containing only the dates that are used in the simulation
 """
 function truncate_dataset(
     datafile,
-    name,
+    varname,
     datapath_trunc,
     date0,
     t_start,
@@ -654,17 +654,42 @@ function truncate_dataset(
     date_start = date0 + Dates.Second(t_start)
     date_end = date0 + Dates.Second(t_start + t_end)
 
-    file_name = replace(string(name, "_truncated_data_", string(date_start), string(date_end), ".nc"), r":" => "")
+    file_name =
+        replace(string(lowercase(varname), "_truncated_data_", string(date_start), string(date_end), ".nc"), r":" => "")
     datafile_truncated = joinpath(datapath_trunc, file_name)
 
     if ClimaComms.iamroot(comms_ctx)
         ds = NCDatasets.NCDataset(datafile, "r")
         dates = ds["time"][:]
 
+        # Find the bounding indices of the dates we need
         (start_id, end_id) = find_idx_bounding_dates(dates, date_start, date_end)
 
+        var_truncated = NCDatasets.nomissing(NCDatasets.view(ds, time = start_id:end_id)[varname])
+
+        # Create new dataset to fill with truncated data
         ds_truncated = NCDatasets.NCDataset(datafile_truncated, "c")
-        ds_truncated = NCDatasets.write(ds_truncated, NCDatasets.view(ds, time = start_id:end_id))
+
+        # Keep all dimensions of original dataset (except for time, which we truncate)
+        ds_dim_names = NCDatasets.dimnames(ds[varname])
+        for dim_name in ds_dim_names
+            dim_name != "time" && NCDatasets.defDim(ds_truncated, dim_name, ds.dim[dim_name])
+        end
+        dates_truncated = dates[start_id:end_id]
+        NCDatasets.defDim(ds_truncated, "time", length(dates_truncated))
+        ds_truncated.attrib["title"] = ds.attrib["title"] * " (dates truncated)"
+
+        # Define dimension variables
+        for dim_name in ds_dim_names
+            if dim_name == "time"
+                var = NCDatasets.defVar(ds_truncated, dim_name, dates_truncated, (dim_name,))
+            else
+                var = NCDatasets.defVar(ds_truncated, dim_name, ds[dim_name][:], (dim_name,))
+            end
+        end
+
+        # Create variable of interest in new dataset, and fill with input dataset values
+        var = NCDatasets.defVar(ds_truncated, varname, var_truncated, ds_dim_names)
 
         close(ds)
         close(ds_truncated)

test/Project.toml

@@ -11,7 +11,6 @@ ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7"
 ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532"
 ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 GeoMakie = "db073c08-6b98-4ee5-b6a4-5efafb3259c6"
@@ -41,7 +40,6 @@ ArtifactWrappers = "0.2"
 ClimaAtmos = "0.22"
 ClimaLand = "0.11"
 ClimaParams = "0.10"
-CommonDataModel = "=0.3.5"
 Dates = "1"
 Downloads = "1"
 IntervalSets = "0.5, 0.6, 0.7"

test/regridder_tests.jl

@@ -312,6 +312,7 @@ for FT in (Float32, Float64)
         end
     end
 end
+
 # test dataset truncation
 @testset "test dataset truncation" begin
     # Get the original dataset set up
@@ -340,7 +341,7 @@ end
     date0test = ["18690101", "18700101", "19790228", "20220301", "20230101"]
     for date in date0test
         date0 = Dates.DateTime(date, Dates.dateformat"yyyymmdd")
-        sst_data = Regridder.truncate_dataset(sst_data_all, "test", REGRID_DIR, date0, t_start, t_end, comms_ctx)
+        sst_data = Regridder.truncate_dataset(sst_data_all, "SST", REGRID_DIR, date0, t_start, t_end, comms_ctx)
         ds_truncated = NCDatasets.NCDataset(sst_data, "r")
         new_dates = ds_truncated["time"][:]