notebooks/01_cmip6_netcdf_to_zarr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a626e70d-432a-40c3-b2cd-507acee1646a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import fsspec\n",
    "import s3fs\n",
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "054815d5-ece9-4506-9104-a1b2ff146693",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "target_chunks = {'lat': 600, 'lon': 1440, 'time': 1}\n",
    "temporal_resolution = \"daily\"\n",
    "model = \"ACCESS-CM2\"\n",
    "variable = \"tasmax\"\n",
    "anon = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da68c219-f597-4117-ba49-6708a0fd3285",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_path = f\"s3://nex-gddp-cmip6/NEX-GDDP-CMIP6/{model}/historical/r1i1p1*/{variable}/*\"\n",
    "fs = fsspec.filesystem(\"s3\", anon=anon, skip_instance_cache=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3487b0f9-8a3e-4a5e-b14d-a2509b56f097",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "file_paths = fs.glob(s3_path)\n",
    "fileset = [fs.open(file) for file in file_paths[0:2]]\n",
    "fileset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "580824ed-a875-408c-9a0f-c678a1565d94",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = xr.open_mfdataset(fileset, combine='by_coords', chunks=target_chunks)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "415e8751-3c57-46e6-bee7-31017f35f3f4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if target_chunks['time'] == 29:\n",
    "    store_name = f\"{variable}_day_{model}_historical_r1i1p1f1_gn/data-0a\"\n",
    "elif target_chunks['time'] == 1:\n",
    "    store_name = f\"{variable}_day_{model}_historical_r1i1p1f1_gn/data-1a\"\n",
    "bucket_path = 's3://carbonplan-benchmarks/data'\n",
    "dataset_path = '/'.join(file_paths[0].split('/')[1:-1])\n",
    "target_root = f\"NEX-GDDP-CMIP6/ACCESS-CM2/historical/r1i1p1f1/tasmax/tasmax_day_ACCESS-CM2_historical_r1i1p1f1_gn\"\n",
    "target_store = os.path.join(bucket_path, dataset_path, store_name)\n",
    "target_store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6f0fe82-c5f6-4307-b616-c2b8ada46d79",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.to_zarr(target_store, mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb03c5c1-e66d-4a08-950e-30f0f86a1ef4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:benchmark-data]",
   "language": "python",
   "name": "conda-env-benchmark-data-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}