Skip to content

Commit

Permalink
fix: jgrss/dependencies (#134)
Browse files Browse the repository at this point in the history 
* modified circleci config

* pinned cython and numpy versions

* pinned numpy upper

* relaxed numpy upper

* fixed duplicate test names

* cloned xarray.open_rasterio

* replaced xarray.open_rasterio with open_rasterio

* changed type to avoid deprecation warning

* minor line reformatting

* removed numpy CircleCI version

* code reformatting

* added source URL

* added missing }

* minor line formatting

* reversed try/except to use preferred EPSG:

* added test for warping

* removed +init

* added str.lower()

* line formatting

* replaced requirements.txt with setup.cfg

* updated header docstring modifications

* removed xarray from docstring example
  • Loading branch information
@jgrss
jgrss committed May 31, 2022
1 parent 8800fe8 commit 342bb2b
Show file tree
Hide file tree
Showing 11 changed files with 549 additions and 172 deletions.
16 changes: 7 additions & 9 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,26 +55,24 @@ jobs:
. gwenv/bin/activate
pip install -U pip setuptools wheel
# Install pre-geowombat libraries
# This shouldn't be necessary with pyproject.toml
pip install Cython numpy
# This shouldn't be necessary with pyproject.toml but we need numpy for GDAL
pip install -U numpy
pip install GDAL==$GDAL_VERSION
# Install GeoWombat
pip install .
# We should be able to install these as extras (i.e., .[ml])
pip install sklearn-xarray rtree
pip install .[perf,ml]
- save_cache:
key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
key: deps1-{{ .Branch }}-{{ checksum "setup.cfg" }}
paths:
- gwenv
- restore_cache:
key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
key: deps1-{{ .Branch }}-{{ checksum "setup.cfg" }}
- run:
name: Run tests
working_directory: ~/project/tests/
command: |
. /home/circleci/project/gwenv/bin/activate
pip install testfixtures
python -m unittest discover
python -m unittest
new_release:
docker:
- image: cimg/python:3.8.12
Expand All @@ -84,7 +82,7 @@ jobs:
fingerprints:
- "1c:2f:26:72:77:69:09:fe:53:14:49:0d:9c:42:0f:7c"
- restore_cache:
key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
key: deps1-{{ .Branch }}-{{ checksum "setup.cfg" }}
- run:
name: Release new version
command: |
Expand Down
2 changes: 1 addition & 1 deletion doc/source/tasks.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ Setup a task and visualize the steps
open_args = {'chunks': 512}
config_args = {'sensor': 'bgr', 'nodata': 0, 'scale_factor': 0.0001}
out_args = {
out_args = {}
# Setup the task object
task_mean = GeoTask(inputs,
Expand Down
6 changes: 3 additions & 3 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[build-system]
requires = [
'setuptools',
'setuptools>=59.5.0',
'wheel',
'Cython',
'numpy'
'Cython>=0.29.*',
'numpy>=1.19.0'
]
12 changes: 7 additions & 5 deletions src/geowombat/backends/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
from .xarray_ import warp_open
from .xarray_ import transform_crs

__all__ = ['Cluster',
'concat',
'mosaic',
'warp_open',
'transform_crs']
__all__ = [
'Cluster',
'concat',
'mosaic',
'warp_open',
'transform_crs'
]
204 changes: 102 additions & 102 deletions src/geowombat/backends/xarray_.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,15 @@
from ..core.windows import get_window_offsets
from ..core.util import parse_filename_dates
from ..config import config
from .rasterio_ import get_ref_image_meta, warp, warp_images, get_file_bounds, window_to_bounds, unpack_bounding_box, unpack_window
from .rasterio_ import get_ref_image_meta
from .rasterio_ import warp
from .rasterio_ import warp_images
from .rasterio_ import get_file_bounds
from .rasterio_ import window_to_bounds
from .rasterio_ import unpack_bounding_box
from .rasterio_ import unpack_window
from .rasterio_ import transform_crs as rio_transform_crs
from .xarray_rasterio_ import open_rasterio

import numpy as np
from rasterio import open as rio_open
Expand All @@ -25,9 +32,7 @@


def _update_kwarg(ref_obj, ref_kwargs, key):

"""
Updates keyword arguments for global config parameters
"""Updates keyword arguments for global config parameters
Args:
ref_obj (str or object)
Expand All @@ -53,19 +58,15 @@ def _update_kwarg(ref_obj, ref_kwargs, key):


def _get_raster_coords(filename):

with xr.open_rasterio(filename) as src:

with open_rasterio(filename) as src:
x = src.x.values - src.res[0] / 2.0
y = src.y.values + src.res[1] / 2.0

return x, y


def _check_config_globals(filenames, bounds_by, ref_kwargs):

"""
Checks global configuration parameters
"""Checks global configuration parameters
Args:
filenames (str or str list)
Expand Down Expand Up @@ -183,20 +184,20 @@ def _check_config_globals(filenames, bounds_by, ref_kwargs):
return ref_kwargs


def warp_open(filename,
band_names=None,
resampling='nearest',
dtype=None,
netcdf_vars=None,
nodata=0,
return_windows=False,
warp_mem_limit=512,
num_threads=1,
tap=False,
**kwargs):

"""
Warps and opens a file
def warp_open(
filename,
band_names=None,
resampling='nearest',
dtype=None,
netcdf_vars=None,
nodata=0,
return_windows=False,
warp_mem_limit=512,
num_threads=1,
tap=False,
**kwargs
):
"""Warps and opens a file
Args:
filename (str): The file to open.
Expand All @@ -210,20 +211,21 @@ def warp_open(filename,
warp_mem_limit (Optional[int]): The memory limit (in MB) for the ``rasterio.vrt.WarpedVRT`` function.
num_threads (Optional[int]): The number of warp worker threads.
tap (Optional[bool]): Whether to target align pixels.
kwargs (Optional[dict]): Keyword arguments passed to ``xarray.open_rasterio``.
kwargs (Optional[dict]): Keyword arguments passed to ``open_rasterio``.
Returns:
``xarray.DataArray``
"""

ref_kwargs = {'bounds': None,
'crs': None,
'res': None,
'nodata': nodata if config['nodata'] is None else config['nodata'],
'warp_mem_limit': warp_mem_limit,
'num_threads': num_threads,
'tap': tap,
'tac': None}
ref_kwargs = {
'bounds': None,
'crs': None,
'res': None,
'nodata': nodata if config['nodata'] is None else config['nodata'],
'warp_mem_limit': warp_mem_limit,
'num_threads': num_threads,
'tap': tap,
'tac': None
}

ref_kwargs_netcdf_stack = ref_kwargs.copy()
ref_kwargs_netcdf_stack['bounds_by'] = 'union'
Expand Down Expand Up @@ -253,21 +255,26 @@ def warp_open(filename,

@contextlib.contextmanager
def warp_netcdf_vars():

# Warp all images to the same grid.
warped_objects = warp_images(filenames,
resampling=resampling,
**ref_kwargs_netcdf_stack)

yield xr.concat((xr.open_rasterio(wobj, **kwargs)\
.assign_coords(band=[band_names[wi]] if band_names else [netcdf_vars[wi]])
for wi, wobj in enumerate(warped_objects)), dim='band')

with xr.open_rasterio(warp(filename,
resampling=resampling,
**ref_kwargs),
**kwargs) if not filenames else warp_netcdf_vars() as src:

yield xr.concat(
(
open_rasterio(wobj, **kwargs).assign_coords(band=[band_names[wi]]
if band_names else [netcdf_vars[wi]])
for wi, wobj in enumerate(warped_objects)
), dim='band'
)

with open_rasterio(
warp(
filename,
resampling=resampling,
**ref_kwargs
), **kwargs
) if not filenames else warp_netcdf_vars() as src:
if band_names:
if len(band_names) > src.gw.nbands:
src.coords['band'] = band_names[:src.gw.nbands]
Expand Down Expand Up @@ -321,20 +328,19 @@ def warp_netcdf_vars():
else:
return src


def mosaic(filenames,
overlap='max',
bounds_by='reference',
resampling='nearest',
band_names=None,
nodata=0,
dtype=None,
warp_mem_limit=512,
num_threads=1,
**kwargs):

"""
Mosaics a list of images
def mosaic(
filenames,
overlap='max',
bounds_by='reference',
resampling='nearest',
band_names=None,
nodata=0,
dtype=None,
warp_mem_limit=512,
num_threads=1,
**kwargs
):
"""Mosaics a list of images
Args:
filenames (list): A list of file names to mosaic.
Expand All @@ -353,12 +359,11 @@ def mosaic(filenames,
from the file.
warp_mem_limit (Optional[int]): The memory limit (in MB) for the ``rasterio.vrt.WarpedVRT`` function.
num_threads (Optional[int]): The number of warp worker threads.
kwargs (Optional[dict]): Keyword arguments passed to ``xarray.open_rasterio``.
kwargs (Optional[dict]): Keyword arguments passed to ``open_rasterio``.
Returns:
``xarray.DataArray``
"""

if overlap not in ['min', 'max', 'mean']:
logger.exception(" The overlap argument must be one of ['min', 'max', 'mean'].")

Expand All @@ -384,19 +389,16 @@ def mosaic(filenames,
tags = src_.tags()

# Combine the data
with xr.open_rasterio(warped_objects[0], **kwargs) as darray:

with open_rasterio(warped_objects[0], **kwargs) as darray:
attrs = darray.attrs.copy()

# Get the original bounds, unsampled
with xr.open_rasterio(filenames[0], **kwargs) as src_:
with open_rasterio(filenames[0], **kwargs) as src_:
footprints.append(src_.gw.geometry)

for fidx, fn in enumerate(warped_objects[1:]):

with xr.open_rasterio(fn, **kwargs) as darrayb:

with xr.open_rasterio(filenames[fidx+1], **kwargs) as src_:
with open_rasterio(fn, **kwargs) as darrayb:
with open_rasterio(filenames[fidx+1], **kwargs) as src_:
footprints.append(src_.gw.geometry)
src_ = None

Expand Down Expand Up @@ -485,24 +487,23 @@ def mosaic(filenames,
else:
return darray


def concat(filenames,
stack_dim='time',
bounds_by='reference',
resampling='nearest',
time_names=None,
band_names=None,
nodata=0,
dtype=None,
netcdf_vars=None,
overlap='max',
warp_mem_limit=512,
num_threads=1,
tap=False,
**kwargs):

"""
Concatenates a list of images
def concat(
filenames,
stack_dim='time',
bounds_by='reference',
resampling='nearest',
time_names=None,
band_names=None,
nodata=0,
dtype=None,
netcdf_vars=None,
overlap='max',
warp_mem_limit=512,
num_threads=1,
tap=False,
**kwargs
):
"""Concatenates a list of images
Args:
filenames (list): A list of file names to concatenate.
Expand All @@ -525,12 +526,11 @@ def concat(filenames,
warp_mem_limit (Optional[int]): The memory limit (in MB) for the ``rasterio.vrt.WarpedVRT`` function.
num_threads (Optional[int]): The number of warp worker threads.
tap (Optional[bool]): Whether to target align pixels.
kwargs (Optional[dict]): Keyword arguments passed to ``xarray.open_rasterio``.
kwargs (Optional[dict]): Keyword arguments passed to ``open_rasterio``.
Returns:
``xarray.DataArray``
"""

if stack_dim.lower() not in ['band', 'time']:
logger.exception(" The stack dimension should be 'band' or 'time'.")

Expand Down Expand Up @@ -699,21 +699,21 @@ def concat(filenames,
return src


def transform_crs(data_src,
dst_crs=None,
dst_res=None,
dst_width=None,
dst_height=None,
dst_bounds=None,
src_nodata=None,
dst_nodata=None,
coords_only=False,
resampling='nearest',
warp_mem_limit=512,
num_threads=1):

"""
Transforms a DataArray to a new coordinate reference system
def transform_crs(
data_src,
dst_crs=None,
dst_res=None,
dst_width=None,
dst_height=None,
dst_bounds=None,
src_nodata=None,
dst_nodata=None,
coords_only=False,
resampling='nearest',
warp_mem_limit=512,
num_threads=1
):
"""Transforms a DataArray to a new coordinate reference system
Args:
data_src (DataArray): The data to transform.
Expand Down
Loading

0 comments on commit 342bb2b

Please sign in to comment.