Skip to content

Commit

Permalink
fill
Browse files Browse the repository at this point in the history
  • Loading branch information
@ramarty
ramarty committed Dec 20, 2023
1 parent b7b4338 commit a292518
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 20 deletions.
61 changes: 44 additions & 17 deletions examples/quality-assessment.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,27 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 2,
"id": "de240c3b-c4bd-4d04-94b7-d042029b618b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found existing installation: blackmarblepy 0.2.2\n",
"Uninstalling blackmarblepy-0.2.2:\n",
" Successfully uninstalled blackmarblepy-0.2.2\n"
]
}
],
"source": [
"!python -m pip uninstall blackmarblepy --yes"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "391def64-3415-43a0-a6b5-dc99450a4576",
"metadata": {},
"outputs": [
Expand All @@ -39,12 +59,12 @@
"output_type": "stream",
"text": [
"Collecting git+https://github.com/worldbank/blackmarblepy@fill-value\n",
" Cloning https://github.com/worldbank/blackmarblepy (to revision fill-value) to /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-uf6i7wc5\n",
" Running command git clone --filter=blob:none --quiet https://github.com/worldbank/blackmarblepy /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-uf6i7wc5\n",
" Cloning https://github.com/worldbank/blackmarblepy (to revision fill-value) to /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-4ybhfwfy\n",
" Running command git clone --filter=blob:none --quiet https://github.com/worldbank/blackmarblepy /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-4ybhfwfy\n",
" Running command git checkout -b fill-value --track origin/fill-value\n",
" Switched to a new branch 'fill-value'\n",
" Branch 'fill-value' set up to track remote branch 'fill-value' from 'origin'.\n",
" Resolved https://github.com/worldbank/blackmarblepy to commit feb64332a3bf7616caf651150de9c9fbfa3c2025\n",
" Resolved https://github.com/worldbank/blackmarblepy to commit b7b433819a1733e43e134acb7e90371b688cd83a\n",
" Installing build dependencies ... \u001b[?25ldone\n",
"\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n",
"\u001b[?25h Installing backend dependencies ... \u001b[?25ldone\n",
Expand Down Expand Up @@ -115,7 +135,14 @@
"Requirement already satisfied: wcwidth in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.2.5)\n",
"Requirement already satisfied: executing in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.8.3)\n",
"Requirement already satisfied: asttokens in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (2.0.5)\n",
"Requirement already satisfied: pure-eval in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.2.2)\n"
"Requirement already satisfied: pure-eval in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.2.2)\n",
"Building wheels for collected packages: blackmarblepy\n",
" Building wheel for blackmarblepy (pyproject.toml) ... \u001b[?25ldone\n",
"\u001b[?25h Created wheel for blackmarblepy: filename=blackmarblepy-0.2.2-py3-none-any.whl size=24227 sha256=bf92e6a1d8692bf779e194622cc2020f7d2ee13933933e17c8803e37053c6b24\n",
" Stored in directory: /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-ephem-wheel-cache-bqvsznh3/wheels/69/92/76/a5f5520f9b2c6aca3ed52b04dcc802937b955f09b0ae2e0f45\n",
"Successfully built blackmarblepy\n",
"Installing collected packages: blackmarblepy\n",
"Successfully installed blackmarblepy-0.2.2\n"
]
}
],
Expand All @@ -125,7 +152,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 4,
"id": "312f1929-0ef8-44ae-904a-38408149ab8b",
"metadata": {
"editable": true,
Expand Down Expand Up @@ -165,7 +192,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 5,
"id": "0f81655a-5476-49e3-b57e-40a3fbd63810",
"metadata": {},
"outputs": [],
Expand All @@ -175,7 +202,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 6,
"id": "72c4977e-793d-4eed-8462-59d312e79d6d",
"metadata": {
"editable": true,
Expand All @@ -193,7 +220,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 7,
"id": "985e4bbb-00df-4ba4-b7d7-dcc89c873481",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -225,14 +252,14 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 8,
"id": "73955cfe-29c7-43f5-a59f-4a535d6f7f9d",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "dd226b42b3cf4db88c60fa92f656eb87",
"model_id": "98700e058d9f4e53bc43ec5208c19c90",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -246,7 +273,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c80f10ea941940eda0f604c72e6fc9d5",
"model_id": "3f38614658624f01bd0dec9996932d77",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -260,7 +287,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2be47f80564b4503bbae09c997580ea5",
"model_id": "170598014b7945849d95e000efb7154a",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -302,7 +329,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e6dbc57d0896492aa1ccb0c72a055dfc",
"model_id": "620811bb84bd47fc80263240383006f2",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -316,7 +343,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "947ac268da874774a0ecc31d3c3d102d",
"model_id": "3594f8b6b9a8446eade22b661ea41411",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -334,11 +361,11 @@
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mUnboundLocalError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ntl_r \u001b[38;5;241m=\u001b[39m bm_raster(\n\u001b[1;32m 2\u001b[0m gdf,\n\u001b[1;32m 3\u001b[0m product_id\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mVNP46A2\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 4\u001b[0m date_range\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2023-01-01\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 5\u001b[0m bearer\u001b[38;5;241m=\u001b[39mbearer,\n\u001b[1;32m 6\u001b[0m variable\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGap_Filled_DNB_BRDF-Corrected_NTL\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 7\u001b[0m )\n",
"Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ntl_r \u001b[38;5;241m=\u001b[39m bm_raster(\n\u001b[1;32m 2\u001b[0m gdf,\n\u001b[1;32m 3\u001b[0m product_id\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mVNP46A2\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 4\u001b[0m date_range\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2023-01-01\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 5\u001b[0m bearer\u001b[38;5;241m=\u001b[39mbearer,\n\u001b[1;32m 6\u001b[0m variable\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGap_Filled_DNB_BRDF-Corrected_NTL\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 7\u001b[0m )\n",
"File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/pydantic/_internal/_validate_call.py:113\u001b[0m, in \u001b[0;36mValidateCallWrapper.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m--> 113\u001b[0m res \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__pydantic_validator__\u001b[38;5;241m.\u001b[39mvalidate_python(pydantic_core\u001b[38;5;241m.\u001b[39mArgsKwargs(args, kwargs))\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__return_pydantic_validator__:\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__return_pydantic_validator__(res)\n",
"File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:285\u001b[0m, in \u001b[0;36mbm_raster\u001b[0;34m(gdf, product_id, date_range, bearer, variable, quality_flag_rm, check_all_tiles_exist, file_directory, file_prefix, file_skip_if_exists)\u001b[0m\n\u001b[1;32m 281\u001b[0m filenames \u001b[38;5;241m=\u001b[39m _pivot_paths_by_date(pathnames)\u001b[38;5;241m.\u001b[39mget(date)\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 284\u001b[0m \u001b[38;5;66;03m# Open each GeoTIFF file as a DataArray and store in a list\u001b[39;00m\n\u001b[0;32m--> 285\u001b[0m da \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 286\u001b[0m rioxarray\u001b[38;5;241m.\u001b[39mopen_rasterio(\n\u001b[1;32m 287\u001b[0m h5_to_geotiff(\n\u001b[1;32m 288\u001b[0m f,\n\u001b[1;32m 289\u001b[0m variable\u001b[38;5;241m=\u001b[39mvariable,\n\u001b[1;32m 290\u001b[0m quality_flag_rm\u001b[38;5;241m=\u001b[39mquality_flag_rm,\n\u001b[1;32m 291\u001b[0m output_prefix\u001b[38;5;241m=\u001b[39mfile_prefix,\n\u001b[1;32m 292\u001b[0m output_directory\u001b[38;5;241m=\u001b[39md,\n\u001b[1;32m 293\u001b[0m ),\n\u001b[1;32m 294\u001b[0m )\n\u001b[1;32m 295\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m filenames\n\u001b[1;32m 296\u001b[0m ]\n\u001b[1;32m 297\u001b[0m ds \u001b[38;5;241m=\u001b[39m merge_arrays(da)\n\u001b[1;32m 298\u001b[0m ds \u001b[38;5;241m=\u001b[39m ds\u001b[38;5;241m.\u001b[39mrio\u001b[38;5;241m.\u001b[39mclip(gdf\u001b[38;5;241m.\u001b[39mgeometry\u001b[38;5;241m.\u001b[39mapply(mapping), gdf\u001b[38;5;241m.\u001b[39mcrs, drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
"File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:287\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 281\u001b[0m filenames \u001b[38;5;241m=\u001b[39m _pivot_paths_by_date(pathnames)\u001b[38;5;241m.\u001b[39mget(date)\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 284\u001b[0m \u001b[38;5;66;03m# Open each GeoTIFF file as a DataArray and store in a list\u001b[39;00m\n\u001b[1;32m 285\u001b[0m da \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 286\u001b[0m rioxarray\u001b[38;5;241m.\u001b[39mopen_rasterio(\n\u001b[0;32m--> 287\u001b[0m h5_to_geotiff(\n\u001b[1;32m 288\u001b[0m f,\n\u001b[1;32m 289\u001b[0m variable\u001b[38;5;241m=\u001b[39mvariable,\n\u001b[1;32m 290\u001b[0m quality_flag_rm\u001b[38;5;241m=\u001b[39mquality_flag_rm,\n\u001b[1;32m 291\u001b[0m output_prefix\u001b[38;5;241m=\u001b[39mfile_prefix,\n\u001b[1;32m 292\u001b[0m output_directory\u001b[38;5;241m=\u001b[39md,\n\u001b[1;32m 293\u001b[0m ),\n\u001b[1;32m 294\u001b[0m )\n\u001b[1;32m 295\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m filenames\n\u001b[1;32m 296\u001b[0m ]\n\u001b[1;32m 297\u001b[0m ds \u001b[38;5;241m=\u001b[39m merge_arrays(da)\n\u001b[1;32m 298\u001b[0m ds \u001b[38;5;241m=\u001b[39m ds\u001b[38;5;241m.\u001b[39mrio\u001b[38;5;241m.\u001b[39mclip(gdf\u001b[38;5;241m.\u001b[39mgeometry\u001b[38;5;241m.\u001b[39mapply(mapping), gdf\u001b[38;5;241m.\u001b[39mcrs, drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
"File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:114\u001b[0m, in \u001b[0;36mh5_to_geotiff\u001b[0;34m(f, variable, quality_flag_rm, output_directory, output_prefix)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;66;03m# Extract data and attributes\u001b[39;00m\n\u001b[1;32m 113\u001b[0m fill_value \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFill_Value\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m9999999\u001b[39m) \u001b[38;5;66;03m# default to large value that no dataset will have\u001b[39;00m\n\u001b[0;32m--> 114\u001b[0m data \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mwhere(data \u001b[38;5;241m==\u001b[39m fill_value, np\u001b[38;5;241m.\u001b[39mnan, data)\n\u001b[1;32m 116\u001b[0m scale_factor \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mscale_factor\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 117\u001b[0m offset \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moffset\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m0\u001b[39m)\n",
"File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:114\u001b[0m, in \u001b[0;36mh5_to_geotiff\u001b[0;34m(f, variable, quality_flag_rm, output_directory, output_prefix)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;66;03m# Extract data and attributes\u001b[39;00m\n\u001b[1;32m 113\u001b[0m fill_value \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfill_value\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m9999999\u001b[39m) \u001b[38;5;66;03m# default to large value that no dataset will have\u001b[39;00m\n\u001b[0;32m--> 114\u001b[0m data \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mwhere(data \u001b[38;5;241m==\u001b[39m fill_value, np\u001b[38;5;241m.\u001b[39mnan, data)\n\u001b[1;32m 116\u001b[0m scale_factor \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mscale_factor\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 117\u001b[0m offset \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moffset\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m0\u001b[39m)\n",
"\u001b[0;31mUnboundLocalError\u001b[0m: cannot access local variable 'data' where it is not associated with a value"
]
}
Expand Down
8 changes: 5 additions & 3 deletions src/blackmarble/raster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,13 @@ def h5_to_geotiff(
]

# Extract data and attributes
fill_value = dataset.attrs.get("fill_value", 9999999) # default to large value that no dataset will have
data = np.where(data == fill_value, np.nan, data)

fill_value = dataset.attrs.get("fill_value", 9999999) # default to large value that no dataset will have
scale_factor = dataset.attrs.get("scale_factor", 1)
offset = dataset.attrs.get("offset", 0)

data = dataset[:]

data = np.where(data == fill_value, np.nan, data)
data = scale_factor * dataset[:] + offset
qf = qf[:]

Expand Down

0 comments on commit a292518

Please sign in to comment.