Skip to content

Commit

Permalink
add chunk status field; add strict mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@appelmar
appelmar committed Mar 4, 2024
1 parent 714682c commit b9521ab
Show file tree
Hide file tree
Showing 38 changed files with 373 additions and 2,120 deletions.
8 changes: 4 additions & 4 deletions R/RcppExports.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ gc_create_view <- function(v) {
.Call('_gdalcubes_gc_create_view', PACKAGE = 'gdalcubes', v)
}

gc_create_image_collection_cube <- function(pin, chunk_sizes, mask, v = NULL) {
.Call('_gdalcubes_gc_create_image_collection_cube', PACKAGE = 'gdalcubes', pin, chunk_sizes, mask, v)
gc_create_image_collection_cube <- function(pin, chunk_sizes, mask, strict = TRUE, v = NULL) {
.Call('_gdalcubes_gc_create_image_collection_cube', PACKAGE = 'gdalcubes', pin, chunk_sizes, mask, strict, v)
}

gc_create_ncdf_cube <- function(path, chunk_sizes, auto_unpack) {
Expand Down Expand Up @@ -209,8 +209,8 @@ gc_create_stream_cube <- function(pin, cmd) {
.Call('_gdalcubes_gc_create_stream_cube', PACKAGE = 'gdalcubes', pin, cmd)
}

gc_create_simple_cube <- function(files, datetime_values, bands, band_names, dx, dy, chunk_sizes) {
.Call('_gdalcubes_gc_create_simple_cube', PACKAGE = 'gdalcubes', files, datetime_values, bands, band_names, dx, dy, chunk_sizes)
gc_create_simple_cube <- function(files, datetime_values, bands, band_names, dx, dy, chunk_sizes, strict = TRUE) {
.Call('_gdalcubes_gc_create_simple_cube', PACKAGE = 'gdalcubes', files, datetime_values, bands, band_names, dx, dy, chunk_sizes, strict)
}

gc_create_fill_time_cube <- function(pin, method) {
Expand Down
12 changes: 7 additions & 5 deletions R/cube.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#' @param view A data cube view defining the shape (spatiotemporal extent, resolution, and spatial reference), if missing, a default overview is used
#' @param mask mask pixels of images based on band values, see \code{\link{image_mask}}
#' @param chunking length-3 vector or a function returning a vector of length 3, defining the size of data cube chunks in the order time, y, x.
#' @param incomplete_ok logical, if TRUE (the default), chunks will ignore IO failures and simply use as much images as possible, otherwise the result will contain empty chunks if IO errors or similar occur.
#' @return A proxy data cube object
#' @details
#' The following steps will be performed when the data cube is requested to read data of a chunk:
Expand Down Expand Up @@ -38,7 +39,7 @@
#'
#' @note This function returns a proxy object, i.e., it will not start any computations besides deriving the shape of the result.
#' @export
raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$default_chunksize) {
raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$default_chunksize, incomplete_ok = TRUE) {

stopifnot(is.image_collection(image_collection))
if (is.function(chunking)) {
Expand All @@ -60,10 +61,10 @@ raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$defa
x = NULL
if (!missing(view)) {
stopifnot(is.cube_view(view))
x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask, view)
x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask, !incomplete_ok, view)
}
else {
x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask)
x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask, !incomplete_ok)
}
class(x) <- c("image_collection_cube", "cube", "xptr")
return(x)
Expand Down Expand Up @@ -100,6 +101,7 @@ raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$defa
#' @param chunking vector of length 3 defining the size of data cube chunks in the order time, y, x.
#' @param dx optional target pixel size in x direction, by default (NULL) the original or highest resolution of images is used
#' @param dy optional target pixel size in y direction, by default (NULL) the original or highest resolution of images is used
#' @param incomplete_ok logical, if TRUE (the default), chunks will ignore IO failures and simply use as much images as possible, otherwise the result will contain empty chunks if IO errors or similar occur.
#' @return A proxy data cube object
#' @examples
#' # toy example, repeating the same image as a daily time series
Expand All @@ -121,7 +123,7 @@ raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$defa
#'
#' @note This function returns a proxy object, i.e., it will not start any computations besides deriving the shape of the result.
#' @export
stack_cube <- function(x, datetime_values, bands = NULL, band_names = NULL, chunking = c(1, 256, 256), dx=NULL, dy=NULL) {
stack_cube <- function(x, datetime_values, bands = NULL, band_names = NULL, chunking = c(1, 256, 256), dx=NULL, dy=NULL, incomplete_ok = TRUE) {


if (length(datetime_values) != length(x)) {
Expand Down Expand Up @@ -157,7 +159,7 @@ stack_cube <- function(x, datetime_values, bands = NULL, band_names = NULL, chun
dy = -1.0
}

x = gc_create_simple_cube(x, datetime_values, bands, band_names, dx, dy, as.integer(chunking))
x = gc_create_simple_cube(x, datetime_values, bands, band_names, dx, dy, as.integer(chunking), !incomplete_ok)
class(x) <- c("simple_cube", "cube", "xptr")
return(x)
}
Expand Down
18 changes: 10 additions & 8 deletions src/RcppExports.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,16 +265,17 @@ BEGIN_RCPP
END_RCPP
}
// gc_create_image_collection_cube
SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask, SEXP v);
RcppExport SEXP _gdalcubes_gc_create_image_collection_cube(SEXP pinSEXP, SEXP chunk_sizesSEXP, SEXP maskSEXP, SEXP vSEXP) {
SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask, bool strict, SEXP v);
RcppExport SEXP _gdalcubes_gc_create_image_collection_cube(SEXP pinSEXP, SEXP chunk_sizesSEXP, SEXP maskSEXP, SEXP strictSEXP, SEXP vSEXP) {
BEGIN_RCPP
Rcpp::RObject rcpp_result_gen;
Rcpp::RNGScope rcpp_rngScope_gen;
Rcpp::traits::input_parameter< SEXP >::type pin(pinSEXP);
Rcpp::traits::input_parameter< Rcpp::IntegerVector >::type chunk_sizes(chunk_sizesSEXP);
Rcpp::traits::input_parameter< SEXP >::type mask(maskSEXP);
Rcpp::traits::input_parameter< bool >::type strict(strictSEXP);
Rcpp::traits::input_parameter< SEXP >::type v(vSEXP);
rcpp_result_gen = Rcpp::wrap(gc_create_image_collection_cube(pin, chunk_sizes, mask, v));
rcpp_result_gen = Rcpp::wrap(gc_create_image_collection_cube(pin, chunk_sizes, mask, strict, v));
return rcpp_result_gen;
END_RCPP
}
Expand Down Expand Up @@ -656,8 +657,8 @@ BEGIN_RCPP
END_RCPP
}
// gc_create_simple_cube
SEXP gc_create_simple_cube(std::vector<std::string> files, std::vector<std::string> datetime_values, std::vector<std::string> bands, std::vector<std::string> band_names, double dx, double dy, Rcpp::IntegerVector chunk_sizes);
RcppExport SEXP _gdalcubes_gc_create_simple_cube(SEXP filesSEXP, SEXP datetime_valuesSEXP, SEXP bandsSEXP, SEXP band_namesSEXP, SEXP dxSEXP, SEXP dySEXP, SEXP chunk_sizesSEXP) {
SEXP gc_create_simple_cube(std::vector<std::string> files, std::vector<std::string> datetime_values, std::vector<std::string> bands, std::vector<std::string> band_names, double dx, double dy, Rcpp::IntegerVector chunk_sizes, bool strict);
RcppExport SEXP _gdalcubes_gc_create_simple_cube(SEXP filesSEXP, SEXP datetime_valuesSEXP, SEXP bandsSEXP, SEXP band_namesSEXP, SEXP dxSEXP, SEXP dySEXP, SEXP chunk_sizesSEXP, SEXP strictSEXP) {
BEGIN_RCPP
Rcpp::RObject rcpp_result_gen;
Rcpp::RNGScope rcpp_rngScope_gen;
Expand All @@ -668,7 +669,8 @@ BEGIN_RCPP
Rcpp::traits::input_parameter< double >::type dx(dxSEXP);
Rcpp::traits::input_parameter< double >::type dy(dySEXP);
Rcpp::traits::input_parameter< Rcpp::IntegerVector >::type chunk_sizes(chunk_sizesSEXP);
rcpp_result_gen = Rcpp::wrap(gc_create_simple_cube(files, datetime_values, bands, band_names, dx, dy, chunk_sizes));
Rcpp::traits::input_parameter< bool >::type strict(strictSEXP);
rcpp_result_gen = Rcpp::wrap(gc_create_simple_cube(files, datetime_values, bands, band_names, dx, dy, chunk_sizes, strict));
return rcpp_result_gen;
END_RCPP
}
Expand Down Expand Up @@ -875,7 +877,7 @@ static const R_CallMethodDef CallEntries[] = {
{"_gdalcubes_gc_add_images", (DL_FUNC) &_gdalcubes_gc_add_images, 4},
{"_gdalcubes_gc_list_collection_formats", (DL_FUNC) &_gdalcubes_gc_list_collection_formats, 0},
{"_gdalcubes_gc_create_view", (DL_FUNC) &_gdalcubes_gc_create_view, 1},
{"_gdalcubes_gc_create_image_collection_cube", (DL_FUNC) &_gdalcubes_gc_create_image_collection_cube, 4},
{"_gdalcubes_gc_create_image_collection_cube", (DL_FUNC) &_gdalcubes_gc_create_image_collection_cube, 5},
{"_gdalcubes_gc_create_ncdf_cube", (DL_FUNC) &_gdalcubes_gc_create_ncdf_cube, 3},
{"_gdalcubes_gc_create_dummy_cube", (DL_FUNC) &_gdalcubes_gc_create_dummy_cube, 4},
{"_gdalcubes_gc_create_empty_cube", (DL_FUNC) &_gdalcubes_gc_create_empty_cube, 3},
Expand Down Expand Up @@ -904,7 +906,7 @@ static const R_CallMethodDef CallEntries[] = {
{"_gdalcubes_gc_write_chunks_ncdf", (DL_FUNC) &_gdalcubes_gc_write_chunks_ncdf, 4},
{"_gdalcubes_gc_write_tif", (DL_FUNC) &_gdalcubes_gc_write_tif, 8},
{"_gdalcubes_gc_create_stream_cube", (DL_FUNC) &_gdalcubes_gc_create_stream_cube, 2},
{"_gdalcubes_gc_create_simple_cube", (DL_FUNC) &_gdalcubes_gc_create_simple_cube, 7},
{"_gdalcubes_gc_create_simple_cube", (DL_FUNC) &_gdalcubes_gc_create_simple_cube, 8},
{"_gdalcubes_gc_create_fill_time_cube", (DL_FUNC) &_gdalcubes_gc_create_fill_time_cube, 2},
{"_gdalcubes_gc_create_aggregate_time_cube", (DL_FUNC) &_gdalcubes_gc_create_aggregate_time_cube, 4},
{"_gdalcubes_gc_create_aggregate_space_cube", (DL_FUNC) &_gdalcubes_gc_create_aggregate_space_cube, 5},
Expand Down
6 changes: 4 additions & 2 deletions src/gdalcubes.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -844,7 +844,7 @@ SEXP gc_create_view(SEXP v) {


// [[Rcpp::export]]
SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask, SEXP v = R_NilValue) {
SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask,bool strict = true, SEXP v = R_NilValue) {

try {
Rcpp::XPtr<std::shared_ptr<image_collection>> aa = Rcpp::as<Rcpp::XPtr<std::shared_ptr<image_collection>>>(pin);
Expand All @@ -858,6 +858,7 @@ SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes,
x = new std::shared_ptr<image_collection_cube>( image_collection_cube::create(*aa, cv));
}
(*x)->set_chunk_size(chunk_sizes[0], chunk_sizes[1], chunk_sizes[2]);
(*x)->set_strict(strict);


if (mask != R_NilValue) {
Expand Down Expand Up @@ -1464,12 +1465,13 @@ SEXP gc_create_stream_cube(SEXP pin, std::string cmd) {
// [[Rcpp::export]]
SEXP gc_create_simple_cube(std::vector<std::string> files,std::vector<std::string> datetime_values,
std::vector<std::string> bands, std::vector<std::string> band_names,
double dx, double dy, Rcpp::IntegerVector chunk_sizes) {
double dx, double dy, Rcpp::IntegerVector chunk_sizes, bool strict = true) {
try {
std::shared_ptr<simple_cube>* x = new std::shared_ptr<simple_cube>( simple_cube::create(files, datetime_values,
bands, band_names,
dx, dy));
(*x)->set_chunk_size(chunk_sizes[0], chunk_sizes[1], chunk_sizes[2]);
(*x)->set_strict(strict);
Rcpp::XPtr< std::shared_ptr<simple_cube> > p(x, true) ;
return p;
}
Expand Down
9 changes: 9 additions & 0 deletions src/gdalcubes/src/aggregate_space.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,15 @@ std::shared_ptr<chunk_data> aggregate_space_cube::read_chunk(chunkid_t id) {
chunkid_t input_chunk_id = _in_cube->chunk_id_from_coords({ccoords[0], ch_y, ch_x});
std::shared_ptr<chunk_data> in_chunk = _in_cube->read_chunk(input_chunk_id);

// propagate chunk status
if (in_chunk->status() == chunk_data::chunk_status::ERROR) {
out->set_status(chunk_data::chunk_status::ERROR);
}
else if (in_chunk->status() == chunk_data::chunk_status::INCOMPLETE && out->status() != chunk_data::chunk_status::ERROR) {
out->set_status(chunk_data::chunk_status::INCOMPLETE);
}


if (in_chunk->empty()) {
continue;
}
Expand Down
9 changes: 9 additions & 0 deletions src/gdalcubes/src/aggregate_time.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,15 @@ std::shared_ptr<chunk_data> aggregate_time_cube::read_chunk(chunkid_t id) {
}

std::shared_ptr<chunk_data> in_chunk = chunk_cache[cur_in_chunk];

// propagate chunk status
if (in_chunk->status() == chunk_data::chunk_status::ERROR) {
out->set_status(chunk_data::chunk_status::ERROR);
}
else if (in_chunk->status() == chunk_data::chunk_status::INCOMPLETE && out->status() != chunk_data::chunk_status::ERROR) {
out->set_status(chunk_data::chunk_status::INCOMPLETE);
}

if (in_chunk->empty()) {
continue;
}
Expand Down
3 changes: 2 additions & 1 deletion src/gdalcubes/src/apply_pixel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ std::shared_ptr<chunk_data> apply_pixel_cube::read_chunk(chunkid_t id) {
return std::make_shared<chunk_data>(); // chunk is outside of the view, we don't need to read anything.

std::shared_ptr<chunk_data> out = std::make_shared<chunk_data>();

std::shared_ptr<chunk_data> in = _in_cube->read_chunk(id);

out->set_status(in->status()); // propagate chunk status
if (in->empty()) {
return out;
}
Expand Down
8 changes: 8 additions & 0 deletions src/gdalcubes/src/crop.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,14 @@ std::shared_ptr<chunk_data> crop_cube::read_chunk(chunkid_t id) {
chunkid_t input_chunk_id = _in_cube->chunk_id_from_coords({ch_t, ch_y, ch_x});
std::shared_ptr<chunk_data> in_chunk = _in_cube->read_chunk(input_chunk_id);


// propagate chunk status
if (in_chunk->status() == chunk_data::chunk_status::ERROR) {
out->set_status(chunk_data::chunk_status::ERROR);
}
else if (in_chunk->status() == chunk_data::chunk_status::INCOMPLETE && out->status() != chunk_data::chunk_status::ERROR) {
out->set_status(chunk_data::chunk_status::INCOMPLETE);
}
if (in_chunk->empty()) {
continue;
}
Expand Down
Loading

0 comments on commit b9521ab

Please sign in to comment.