From 2b5a868067ffc5689b7b96bcf41294ecaa907dec Mon Sep 17 00:00:00 2001 From: Marius Appel Date: Thu, 15 Jul 2021 14:01:23 +0200 Subject: [PATCH] v0.4.0 CRAN release --- .Rbuildignore | 8 ++- DESCRIPTION | 8 +-- NAMESPACE | 3 +- NEWS.md | 8 +-- R/gdalcubes.R | 2 +- R/image_collection_ops.R | 77 -------------------------- R/zzz.R | 2 +- README.md | 68 +++++++++++------------ configure | 18 +++--- configure.ac | 2 +- man/translate_cog.Rd | 52 ----------------- man/translate_gtiff.Rd | 53 ------------------ src/gdalcubes.cpp | 117 +++++++++++++++++++++++++++------------ 13 files changed, 140 insertions(+), 278 deletions(-) delete mode 100644 R/image_collection_ops.R delete mode 100644 man/translate_cog.Rd delete mode 100644 man/translate_gtiff.Rd diff --git a/.Rbuildignore b/.Rbuildignore index cfe759c..99ea7a9 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -9,6 +9,7 @@ ^src/gdalcubes/\.clang-format$ ^src/gdalcubes/\.dockerignore$ ^src/gdalcubes/\.travis.yml$ +^src/gdalcubes/\.github$ ^src/gdalcubes/\.git$ ^vignettes/MOD11A2$ ^vignettes/MOD11A2\.db$ @@ -70,6 +71,7 @@ ^Dockerfile$ ^.*Dockerfile$ ^\.dockerignore$ -^.github$ -^.pkgdown.R$ -^.test$ +^\.docker$ +^\.github$ +^\.pkgdown.R$ +^\.test.*$ diff --git a/DESCRIPTION b/DESCRIPTION index acf14c2..4d487d8 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: gdalcubes Title: Earth Observation Data Cubes from Satellite Image Collections -Version: 0.3.9999 -Date: 2021-02-08 +Version: 0.4.0 +Date: 2021-07-08 Authors@R: c( person(given = "Marius", family = "Appel", @@ -46,13 +46,13 @@ Description: Processing collections of Earth observation images as on-demand mul multithreading. All computational parts are implemented in C++, linking to the 'GDAL', 'netCDF', 'CURL', and 'SQLite' libraries. See Appel and Pebesma (2019) for further details. Depends: R (>= 3.4) -Imports: Rcpp, jsonlite, ncdf4 +Imports: Rcpp, RcppProgress, jsonlite, ncdf4 License: MIT + file LICENSE URL: https://github.com/appelmar/gdalcubes_R BugReports: https://github.com/appelmar/gdalcubes_R/issues/ Encoding: UTF-8 RoxygenNote: 7.1.1 -LinkingTo: Rcpp, RcppThread +LinkingTo: Rcpp, RcppProgress Suggests: knitr, magrittr, diff --git a/NAMESPACE b/NAMESPACE index 227532d..f640b36 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -64,13 +64,12 @@ export(size) export(srs) export(st_as_stars.cube) export(stac_image_collection) -export(translate_cog) -export(translate_gtiff) export(window_time) export(write_chunk_from_array) export(write_ncdf) export(write_tif) export(zonal_statistics) +import(RcppProgress) import(jsonlite) import(ncdf4) importFrom(Rcpp,sourceCpp) diff --git a/NEWS.md b/NEWS.md index cf31fa0..7e3c2ed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,18 +1,16 @@ -# gdalcubes 0.3.2 (under development) +# gdalcubes 0.4.0 (2021-07-08) * new operator `ncdf_cube()` to read data cubes from (intermediate) results * new operator `rename_bands()` to change band names * image collection creation from STAC API queries with `stac_image_collection()` * progress bar can now be disabled with `gdalcubes_options()` -* new progress bar -* removed `RcppProgress` package dependency -* C++ multithreading now uses `RcppThread` * removed `gdalcubes_set_threads()` in favor of `gdalcubes_options()` * removed `gdalcubes_debug_output()` in favor of `gdalcubes_options()` * removed `gdalcubes_set_ncdf_compression()` in favor of `gdalcubes_options()` * removed `gdalcubes_use_cache()` in favor of `gdalcubes_options()` - +* removed image collection operations `translate_COG()` and `translate_gtiff()` +* fix installation issues on MacOS and GCC11 warnings # gdalcubes 0.3.1 (2020-08-25) diff --git a/R/gdalcubes.R b/R/gdalcubes.R index 60143f4..4b82734 100644 --- a/R/gdalcubes.R +++ b/R/gdalcubes.R @@ -17,7 +17,7 @@ #' @importFrom graphics axis box image.default layout lcm par plot rasterImage rect title legend lines #' @importFrom stats quantile rnorm #' @importFrom utils head download.file -#' @import jsonlite ncdf4 +#' @import RcppProgress jsonlite ncdf4 #' #' #' diff --git a/R/image_collection_ops.R b/R/image_collection_ops.R deleted file mode 100644 index 572839e..0000000 --- a/R/image_collection_ops.R +++ /dev/null @@ -1,77 +0,0 @@ -#' Convert complete image collections to cloud-optimized GeoTIFFs -#' -#' This function translates all images of a gdalcubes image collection to cloud-optimized GeoTIFF files. -#' The output contains converted imagery as an additional copy (original files are not deleted) and -#' a new image collection file. -#' -#' @param collection path to an existing image collection file -#' @param target_dir directory where the output will be stored, will be created if necessary -#' @param overwrite logical; if TRUE existing files will be overwritten -#' @param creation_options further settings of the GDAL COG driver; see \url{https://gdal.org/drivers/raster/cog.html} -#' @return path to the new image collection file for use as argument to \code{\link{image_collection}} -#' @examples -#' # create image collection from example Landsat data only -#' # if not already done in other examples -#' if (!file.exists(file.path(tempdir(), "L8.db"))) { -#' L8_files <- list.files(system.file("L8NY18", package = "gdalcubes"), -#' ".TIF", recursive = TRUE, full.names = TRUE) -#' create_image_collection(L8_files, "L8_L1TP", file.path(tempdir(), "L8.db")) -#' } -#' -#' L8.col = image_collection(file.path(tempdir(), "L8.db")) -#' L8.col -#' \donttest{ -#' if ("COG" %in% gdalcubes_gdalformats()) { -#' L8.cog.col = translate_cog(L8.col) -#' L8.cog.col -#' } -#' } -#' @note This function requires the GDAL COG driver, which was added in GDAL version 3.1. -#' @export -translate_cog <- function(collection, target_dir = tempfile(pattern = "image_collection_"), overwrite = TRUE, creation_options = c("BLOCKSIZE=256","COMPRESS=DEFLTE","LEVEL=1","RESAMPLING=CUBIC")) { - stopifnot(is.image_collection(collection)) - out = libgdalcubes_translate_cog(collection, target_dir, .pkgenv$threads, overwrite, creation_options) - return(out) -} - - - - -#' Convert complete image collections to cloud-optimized GeoTIFFs -#' -#' This function translates all images of a gdalcubes image collection to GeoTIFF files. -#' The output contains converted imagery as an additional copy (original files are not deleted) and -#' a new image collection file. -#' -#' @param collection path to an existing image collection file -#' @param target_dir directory where the output will be stored, will be created if necessary -#' @param overwrite logical; if TRUE existing files will be overwritten -#' @param creation_options further settings of the GDAL GTiff driver; see \url{https://gdal.org/drivers/raster/gtiff.html} -#' @return path to the new image collection file for use as argument to \code{\link{image_collection}} -#' @examples -#' # create image collection from example Landsat data only -#' # if not already done in other examples -#' if (!file.exists(file.path(tempdir(), "L8.db"))) { -#' L8_files <- list.files(system.file("L8NY18", package = "gdalcubes"), -#' ".TIF", recursive = TRUE, full.names = TRUE) -#' create_image_collection(L8_files, "L8_L1TP", file.path(tempdir(), "L8.db")) -#' } -#' -#' L8.col = image_collection(file.path(tempdir(), "L8.db")) -#' L8.col -#' -#' \donttest{ -#' L8.tif.col = translate_gtiff(L8.col) -#' L8.tif.col -#' } -#' -#' @details -#' The functions \code{\link{translate_gtiff}} and \code{\link{translate_cog}} have the same purpose to convert imagery to optimized GeoTIFF files. The latter uses the recent COG GDAL driver, -#' whereas the former uses the normal GTiff driver. Depending on additional creation options and the input images, files creted with \code{\link{translate_gtiff}} may or may not contain overview images. -#' @export -translate_gtiff <- function(collection, target_dir = tempfile(pattern = "image_collection_"), overwrite = TRUE, creation_options = c("TILED=YES","COMPRESS=DEFLATE","ZLEVEL=1","COPY_SRC_OVERVIEWS=TRUE")) { - stopifnot(is.image_collection(collection)) - out = libgdalcubes_translate_gtiff(collection, target_dir, .pkgenv$threads, overwrite, creation_options) - return(out) -} - diff --git a/R/zzz.R b/R/zzz.R index 62644af..23fc295 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -16,7 +16,7 @@ .pkgenv$ncdf_write_bounds = TRUE .pkgenv$use_overview_images = TRUE if (interactive()) { - .pkgenv$show_progress = FALSE + .pkgenv$show_progress = TRUE } else { .pkgenv$show_progress = FALSE diff --git a/README.md b/README.md index 3eaa0a3..83f4bae 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ x ## resolution : 1000, 1000 (x, y) ## extent : -6582280, -5799280, -764014.4, -205014.4 (xmin, xmax, ymin, ymax) ## crs : +proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +nadgrids=@null +wktext +no_defs - ## names : cube_4a86d5076049c2013, cube_4a86d5076049c2014, cube_4a86d5076049c2015, cube_4a86d5076049c2016, cube_4a86d5076049c2017, cube_4a86d5076049c2018, cube_4a86d5076049c2019 + ## names : cube_2a3ac18d60def2013, cube_2a3ac18d60def2014, cube_2a3ac18d60def2015, cube_2a3ac18d60def2016, cube_2a3ac18d60def2017, cube_2a3ac18d60def2018, cube_2a3ac18d60def2019 ``` r suppressPackageStartupMessages(library(stars)) @@ -351,17 +351,17 @@ raster_cube(L8.col, v.overview) %>% query_points(x,y,t, v.overview$space$srs) ``` - ## B04 B05 - ## 1 431.5464 455.1081 - ## 2 205.4124 2851.5982 - ## 3 258.9569 2527.4188 - ## 4 249.7894 3129.4676 - ## 5 259.1570 3277.0765 - ## 6 NaN NaN - ## 7 NaN NaN - ## 8 167.8571 2888.4166 - ## 9 NaN NaN - ## 10 NaN NaN + ## B04 B05 + ## 1 176.5942 3032.5324 + ## 2 NaN NaN + ## 3 3878.2453 4828.9001 + ## 4 309.5338 221.7426 + ## 5 NaN NaN + ## 6 213.2422 3180.3651 + ## 7 725.4656 3244.4685 + ## 8 NaN NaN + ## 9 731.5507 2186.4171 + ## 10 NaN NaN ``` r raster_cube(L8.col, v.overview) %>% @@ -370,30 +370,30 @@ raster_cube(L8.col, v.overview) %>% ``` ## $B04 - ## 2013 2014 2015 2016 2017 2018 2019 - ## 1 221.9795 431.5464 315.9992 388.2862 490.8458 NaN 188.7578 - ## 2 214.8850 192.7086 262.5035 205.4124 231.8854 232.1024 918.7678 - ## 3 282.3089 230.5423 264.7602 263.5589 225.6577 258.9569 196.1888 - ## 4 NaN 247.1129 287.4146 249.7894 423.3914 257.3268 NaN - ## 5 NaN 259.1570 NaN 4162.9515 188.5636 293.4895 216.6074 - ## 6 NaN NaN NaN NaN NaN NaN NaN - ## 7 NaN NaN NaN NaN NaN NaN NaN - ## 8 168.4943 203.5209 249.7479 674.5761 169.0721 227.1009 167.8571 - ## 9 NaN NaN NaN NaN NaN NaN NaN - ## 10 NaN NaN NaN NaN NaN NaN NaN + ## 2013 2014 2015 2016 2017 2018 2019 + ## 1 168.1739 166.4379 199.2356 218.3613 176.5942 164.1710 152.0536 + ## 2 NaN NaN NaN NaN NaN NaN NaN + ## 3 342.9068 198.0663 188.4386 171.4423 234.2039 176.9131 3878.2453 + ## 4 NaN 309.5338 653.0165 490.7659 334.7592 662.8106 NaN + ## 5 NaN NaN NaN NaN NaN NaN NaN + ## 6 158.2323 213.2422 288.2853 228.0108 222.6970 175.8526 152.9250 + ## 7 NaN 233.6421 286.0872 725.4656 234.8174 1012.2984 NaN + ## 8 NaN NaN NaN NaN NaN NaN NaN + ## 9 570.6068 1094.3404 698.8447 286.1469 731.5507 NaN 193.9442 + ## 10 NaN 242.1999 295.2309 283.9409 256.7757 270.4900 NaN ## ## $B05 - ## 2013 2014 2015 2016 2017 2018 2019 - ## 1 181.1607 455.1081 355.2088 404.1128 385.7219 NaN 126.5602 - ## 2 2971.5674 2993.9556 3041.8152 2851.5982 3162.7466 3070.888 3082.5338 - ## 3 2842.5738 2752.5699 2840.5063 2970.3880 2998.9544 2527.419 2808.5265 - ## 4 NaN 3155.3942 3341.5062 3129.4676 3349.6015 3404.804 NaN - ## 5 NaN 3277.0765 NaN 5617.8526 3158.9640 3422.652 3237.7933 - ## 6 NaN NaN NaN NaN NaN NaN NaN - ## 7 NaN NaN NaN NaN NaN NaN NaN - ## 8 2841.4007 2891.8610 2888.1330 3083.6199 2823.1811 2904.574 2888.4166 - ## 9 NaN NaN NaN NaN NaN NaN NaN - ## 10 NaN NaN NaN NaN NaN NaN NaN + ## 2013 2014 2015 2016 2017 2018 2019 + ## 1 2881.747 3006.7054 2988.5880 3004.6927 3032.5324 2856.9048 2918.1521 + ## 2 NaN NaN NaN NaN NaN NaN NaN + ## 3 3117.646 3057.9624 2891.6479 2843.5118 3050.4765 2869.4212 4828.9001 + ## 4 NaN 221.7426 337.6181 220.5666 227.3253 435.6817 NaN + ## 5 NaN NaN NaN NaN NaN NaN NaN + ## 6 2991.214 3180.3651 3131.6213 3199.9639 3090.0413 3103.3886 3027.6535 + ## 7 NaN 2994.1378 3212.3224 3244.4685 2994.0125 2884.8660 NaN + ## 8 NaN NaN NaN NaN NaN NaN NaN + ## 9 663.193 1116.9384 650.0309 848.6397 2186.4171 NaN 394.6935 + ## 10 NaN 3167.8636 3493.6315 3106.9801 3229.5042 3434.0334 NaN To compute time series of summary statistics over spatial polygons, we need to specify polygon geometries (e.g., as an `sf` object) and specify diff --git a/configure b/configure index 113dba5..a76ba08 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for gdalcubes 0.3.2. +# Generated by GNU Autoconf 2.69 for gdalcubes 0.4.0. # # Report bugs to . # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='gdalcubes' PACKAGE_TARNAME='gdalcubes' -PACKAGE_VERSION='0.3.2' -PACKAGE_STRING='gdalcubes 0.3.2' +PACKAGE_VERSION='0.4.0' +PACKAGE_STRING='gdalcubes 0.4.0' PACKAGE_BUGREPORT='marius.appel@uni-muenster.de' PACKAGE_URL='' @@ -1257,7 +1257,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures gdalcubes 0.3.2 to adapt to many kinds of systems. +\`configure' configures gdalcubes 0.4.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1319,7 +1319,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of gdalcubes 0.3.2:";; + short | recursive ) echo "Configuration of gdalcubes 0.4.0:";; esac cat <<\_ACEOF @@ -1415,7 +1415,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -gdalcubes configure 0.3.2 +gdalcubes configure 0.4.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1717,7 +1717,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by gdalcubes $as_me 0.3.2, which was +It was created by gdalcubes $as_me 0.4.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -4503,7 +4503,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by gdalcubes $as_me 0.3.2, which was +This file was extended by gdalcubes $as_me 0.4.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -4556,7 +4556,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -gdalcubes config.status 0.3.2 +gdalcubes config.status 0.4.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 59e30e5..32403c1 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ # Process this file with autoconf to produce a configure script. -AC_INIT(gdalcubes, 0.3.2, marius.appel@uni-muenster.de) +AC_INIT(gdalcubes, 0.4.0, marius.appel@uni-muenster.de) AC_LANG(C++) # find R home and set CC/CFLAGS diff --git a/man/translate_cog.Rd b/man/translate_cog.Rd deleted file mode 100644 index 68618d4..0000000 --- a/man/translate_cog.Rd +++ /dev/null @@ -1,52 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/image_collection_ops.R -\name{translate_cog} -\alias{translate_cog} -\title{Convert complete image collections to cloud-optimized GeoTIFFs} -\usage{ -translate_cog( - collection, - target_dir = tempfile(pattern = "image_collection_"), - overwrite = TRUE, - creation_options = c("BLOCKSIZE=256", "COMPRESS=DEFLTE", "LEVEL=1", - "RESAMPLING=CUBIC") -) -} -\arguments{ -\item{collection}{path to an existing image collection file} - -\item{target_dir}{directory where the output will be stored, will be created if necessary} - -\item{overwrite}{logical; if TRUE existing files will be overwritten} - -\item{creation_options}{further settings of the GDAL COG driver; see \url{https://gdal.org/drivers/raster/cog.html}} -} -\value{ -path to the new image collection file for use as argument to \code{\link{image_collection}} -} -\description{ -This function translates all images of a gdalcubes image collection to cloud-optimized GeoTIFF files. -The output contains converted imagery as an additional copy (original files are not deleted) and -a new image collection file. -} -\note{ -This function requires the GDAL COG driver, which was added in GDAL version 3.1. -} -\examples{ -# create image collection from example Landsat data only -# if not already done in other examples -if (!file.exists(file.path(tempdir(), "L8.db"))) { - L8_files <- list.files(system.file("L8NY18", package = "gdalcubes"), - ".TIF", recursive = TRUE, full.names = TRUE) - create_image_collection(L8_files, "L8_L1TP", file.path(tempdir(), "L8.db")) -} - -L8.col = image_collection(file.path(tempdir(), "L8.db")) -L8.col -\donttest{ -if ("COG" \%in\% gdalcubes_gdalformats()) { - L8.cog.col = translate_cog(L8.col) - L8.cog.col -} -} -} diff --git a/man/translate_gtiff.Rd b/man/translate_gtiff.Rd deleted file mode 100644 index 8ccc0ba..0000000 --- a/man/translate_gtiff.Rd +++ /dev/null @@ -1,53 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/image_collection_ops.R -\name{translate_gtiff} -\alias{translate_gtiff} -\title{Convert complete image collections to cloud-optimized GeoTIFFs} -\usage{ -translate_gtiff( - collection, - target_dir = tempfile(pattern = "image_collection_"), - overwrite = TRUE, - creation_options = c("TILED=YES", "COMPRESS=DEFLATE", "ZLEVEL=1", - "COPY_SRC_OVERVIEWS=TRUE") -) -} -\arguments{ -\item{collection}{path to an existing image collection file} - -\item{target_dir}{directory where the output will be stored, will be created if necessary} - -\item{overwrite}{logical; if TRUE existing files will be overwritten} - -\item{creation_options}{further settings of the GDAL GTiff driver; see \url{https://gdal.org/drivers/raster/gtiff.html}} -} -\value{ -path to the new image collection file for use as argument to \code{\link{image_collection}} -} -\description{ -This function translates all images of a gdalcubes image collection to GeoTIFF files. -The output contains converted imagery as an additional copy (original files are not deleted) and -a new image collection file. -} -\details{ -The functions \code{\link{translate_gtiff}} and \code{\link{translate_cog}} have the same purpose to convert imagery to optimized GeoTIFF files. The latter uses the recent COG GDAL driver, -whereas the former uses the normal GTiff driver. Depending on additional creation options and the input images, files creted with \code{\link{translate_gtiff}} may or may not contain overview images. -} -\examples{ -# create image collection from example Landsat data only -# if not already done in other examples -if (!file.exists(file.path(tempdir(), "L8.db"))) { - L8_files <- list.files(system.file("L8NY18", package = "gdalcubes"), - ".TIF", recursive = TRUE, full.names = TRUE) - create_image_collection(L8_files, "L8_L1TP", file.path(tempdir(), "L8.db")) -} - -L8.col = image_collection(file.path(tempdir(), "L8.db")) -L8.col - -\donttest{ -L8.tif.col = translate_gtiff(L8.col) -L8.tif.col -} - -} diff --git a/src/gdalcubes.cpp b/src/gdalcubes.cpp index 5c8ec49..65550b6 100644 --- a/src/gdalcubes.cpp +++ b/src/gdalcubes.cpp @@ -2,10 +2,12 @@ #include "gdalcubes/src/gdalcubes.h" // [[Rcpp::plugins("cpp11")]] -// [[Rcpp::depends(RcppThread)]] +// [[Rcpp::depends(RcppProgress)]] #include -#include +#include +#include #include +#include #include @@ -51,13 +53,17 @@ class chunk_processor_multithread_interruptible : public chunk_processor { void chunk_processor_multithread_interruptible::apply(std::shared_ptr c, std::function, std::mutex &)> f) { std::mutex mutex; - std::vector workers; + bool interrupted = false; + std::vector workers; + std::vector finished(_nthreads, false); for (uint16_t it = 0; it < _nthreads; ++it) { - workers.push_back(RcppThread::Thread([this, &c, f, it, &mutex](void) { + workers.push_back(std::thread([this, &c, f, it, &mutex, &finished, &interrupted](void) { for (uint32_t i = it; i < c->count_chunks(); i += _nthreads) { try { - std::shared_ptr dat = c->read_chunk(i); - f(i, dat, mutex); + if (!interrupted) { + std::shared_ptr dat = c->read_chunk(i); + f(i, dat, mutex); + } } catch (std::string s) { GCBS_ERROR(s); continue; @@ -65,13 +71,33 @@ void chunk_processor_multithread_interruptible::apply(std::shared_ptr c, GCBS_ERROR("unexpected exception while processing chunk " + std::to_string(i)); continue; } - RcppThread::checkUserInterrupt(); } + finished[it] = true; })); } + + bool done = false; + int i=0; + while (!done) { + done = true; + for (uint16_t it = 0; it < _nthreads; ++it) { + done = done && finished[it]; + } + if (!done) { + if (Progress::check_abort()) { + interrupted = true; // still need to wait for threads to finish current chunk + } + uint32_t cur_ms = std::min(500, 50 + i*50); + std::this_thread::sleep_for(std::chrono::milliseconds(cur_ms)); + } + ++i; + } for (uint16_t it = 0; it < _nthreads; ++it) { workers[it].join(); } + if (interrupted) { + throw std::string("computations have been interrupted by the user"); + } } @@ -90,8 +116,7 @@ struct error_handling_r { static void do_output() { _m_errhandl.lock(); _defer = false; - //Rcpp::Rcerr << _err_stream.str() << std::endl; - RcppThread::Rcout << _err_stream.str() << std::endl; + Rcpp::Rcerr << _err_stream.str() << std::endl; _err_stream.str(""); _m_errhandl.unlock(); } @@ -110,8 +135,7 @@ struct error_handling_r { _err_stream << "Debug message: " << msg << where_str << std::endl; } if (!_defer) { - //Rcpp::Rcerr << _err_stream.str() ; - RcppThread::Rcout << _err_stream.str(); + Rcpp::Rcerr << _err_stream.str() ; _err_stream.str(""); } _m_errhandl.unlock(); @@ -128,8 +152,7 @@ struct error_handling_r { _err_stream << "## " << msg << std::endl; } if (!_defer) { - //Rcpp::Rcerr << _err_stream.str(); - RcppThread::Rcout << _err_stream.str(); + Rcpp::Rcerr << _err_stream.str(); _err_stream.str(""); } @@ -158,50 +181,72 @@ struct progress_simple_R : public progress { } virtual void finalize() override { _m.lock(); - RcppThread::Rcout << std::endl; + _rp->update(100); error_handling_r::do_output(); _m.unlock(); } - progress_simple_R() : _p(0) {} - - ~progress_simple_R(){} + progress_simple_R() : _p(0), _rp(nullptr) {} + + ~progress_simple_R(){ + if (_rp) { + delete _rp; + } + } private: std::mutex _m; double _p; + Progress *_rp; void _set(double p) { // call this function only with a lock on _m - //RcppThread::checkUserInterrupt(); - error_handling_r::defer_output(); - _p = p; - RcppThread::Rcout << "["; - int pp = 50 * p; - int i = 0; - while (i < pp) { - if (i < pp) RcppThread::Rcout << "="; - ++i; + //Rcpp::checkUserInterrupt(); + // if (Progress::check_abort()) { + // throw std::string("Operation has been interrupted by user"); + // } + if (!_rp) { + error_handling_r::defer_output(); + _rp = new Progress(100,true); } - RcppThread::Rcout << ">"; - ++i; - while (i < 50) { - RcppThread::Rcout << " "; - ++i; - } - RcppThread::Rcout << "] " << int(p * 100.0) << " %\r" << std::flush; + + _p = p; + _rp->update((int)(_p*100)); } }; struct progress_none_R : public progress { std::shared_ptr get() override { return std::make_shared(); } - void set(double p) override {}; + + void set(double p) override {} + void increment(double dp) override {} - virtual void finalize() override {} - progress_none_R() {} + virtual void finalize() override { + _rp->update(100); + } + progress_none_R() : _rp(nullptr) { + // instance is needed for Progress::check_abort() + _rp = new Progress(100,false); + } + ~progress_none_R(){ + if (_rp) { + delete _rp; + } + } + +private: + Progress *_rp; }; +// +// struct progress_none_R : public progress { +// std::shared_ptr get() override { return std::make_shared(); } +// void set(double p) override {}; +// void increment(double dp) override {} +// virtual void finalize() override {} +// progress_none_R() {} +// };