writeRaster to an S3 destination through GDAL fails

[goergen95]

I prepared a reproducible set-up using the minio and rocker images that you can find in this Gist.

I am trying to use terra to directly write to an S3 bucket through the GDAL Virtual Filesystem driver. It does not seem to work with rasters. Using stars, in contrast currently works seamlessly.
Considering vector data, both using terra and sf seem to write the data though some (to me) cryptic error messages are thrown.

Would you consider adding support to write rasters to virtual file systems in terra?

Sys.setenv(AWS_ACCESS_KEY_ID = "miniouser",
           AWS_SECRET_ACCESS_KEY = "miniopass",
           AWS_DEFAULT_REGION = "",
           AWS_SESSION_TOKEN = "",
           AWS_S3_ENDPOINT = "minio:9000",
           AWS_VIRTUAL_HOSTING = "FALSE", # for GDAL to resolve endpoint correctly
           AWS_HTTPS = "FALSE", # for GDAL to not use https
           CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE = "YES")  # for GDAL to allow writing GTiff

if(!requireNamespace("aws.s3")) install.packages("aws.s3")
#> Loading required namespace: aws.s3
library(aws.s3)
library(terra)
#> terra 1.7.29
library(stars)
#> Loading required package: abind
#> Loading required package: sf
#> Linking to GEOS 3.10.2, GDAL 3.4.1, PROJ 8.2.1; sf_use_s2() is TRUE
library(sf)

# create a bucket
put_bucket(bucket = "my-bucket", use_https = FALSE, region = "")
#> [1] TRUE

# showcase behavior for raster
file <- system.file("ex/elev.tif", package="terra")
raster_terra <- rast(file)
raster_stars <- read_stars(file)

write_stars(raster_stars, dsn = "/vsis3/my-bucket/raster_stars.tif")
(raster_stars <- read_stars("/vsis3/my-bucket/raster_stars.tif"))
#> stars object with 2 dimensions and 1 attribute
#> attribute(s):
#>                   Min. 1st Qu. Median     Mean 3rd Qu. Max. NA's
#> raster_stars.tif   141     291    333 348.3366     406  547 3942
#> dimension(s):
#>   from to  offset       delta refsys point x/y
#> x    1 95 5.74167  0.00833333 WGS 84 FALSE [x]
#> y    1 90 50.1917 -0.00833333 WGS 84 FALSE [y]

try(writeRaster(raster_terra, filename = "/vsis3/my-bucket/raster_terra.tif"))
#> Error : [writeRaster] path does not exist

# showcase behavior for vector
file <- system.file("gpkg/nc.gpkg", package="sf")

vector_sf <- read_sf(file)
vector_terra <- vect(file)

st_write(vector_sf, dsn = "/vsis3/my-bucket/vector_sf.gpkg", driver = "GPKG")
#> Warning in CPL_write_ogr(obj, dsn, layer, driver,
#> as.character(dataset_options), : GDAL Error 1:
#> sqlite3_exec(x-amz-content-sha256:
#> e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) failed: near
#> "c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855": syntax error
#> Writing layer `vector_sf' to data source 
#>   `/vsis3/my-bucket/vector_sf.gpkg' using driver `GPKG'
#> Writing 100 features with 14 fields and geometry type Multi Polygon.
writeVector(vector_terra, filename = "/vsis3/my-bucket/vector_terra.gpkg")
#> Warning in x@ptr$write(filename, layer, filetype, insert[1], overwrite[1], :
#> GDAL Error 1: sqlite3_exec(x-amz-content-sha256:
#> e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) failed: near
#> "c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855": syntax error
#> Warning in x@ptr$write(filename, layer, filetype, insert[1], overwrite[1], :
#> GDAL Message 6: dataset /vsis3/my-bucket/vector_terra.gpkg does not support
#> layer creation option ENCODING

(vector_sf <- read_sf("/vsis3/my-bucket/vector_sf.gpkg"))
#> Warning in CPL_read_ogr(dsn, layer, query, as.character(options), quiet, : GDAL
#> Message 1: GPKG: unrecognized user_version=0x00000000 (0) on
#> '/vsis3/my-bucket/vector_sf.gpkg'
#> Simple feature collection with 100 features and 14 fields
#> Geometry type: MULTIPOLYGON
#> Dimension:     XY
#> Bounding box:  xmin: -84.32385 ymin: 33.88199 xmax: -75.45698 ymax: 36.58965
#> Geodetic CRS:  NAD27
#> # A tibble: 100 × 15
#>     AREA PERIMETER CNTY_ CNTY_ID NAME  FIPS  FIPSNO CRESS_ID BIR74 SID74 NWBIR74
#>    <dbl>     <dbl> <dbl>   <dbl> <chr> <chr>  <dbl>    <int> <dbl> <dbl>   <dbl>
#>  1 0.114      1.44  1825    1825 Ashe  37009  37009        5  1091     1      10
#>  2 0.061      1.23  1827    1827 Alle… 37005  37005        3   487     0      10
#>  3 0.143      1.63  1828    1828 Surry 37171  37171       86  3188     5     208
#>  4 0.07       2.97  1831    1831 Curr… 37053  37053       27   508     1     123
#>  5 0.153      2.21  1832    1832 Nort… 37131  37131       66  1421     9    1066
#>  6 0.097      1.67  1833    1833 Hert… 37091  37091       46  1452     7     954
#>  7 0.062      1.55  1834    1834 Camd… 37029  37029       15   286     0     115
#>  8 0.091      1.28  1835    1835 Gates 37073  37073       37   420     0     254
#>  9 0.118      1.42  1836    1836 Warr… 37185  37185       93   968     4     748
#> 10 0.124      1.43  1837    1837 Stok… 37169  37169       85  1612     1     160
#> # ℹ 90 more rows
#> # ℹ 4 more variables: BIR79 <dbl>, SID79 <dbl>, NWBIR79 <dbl>,
#> #   geom <MULTIPOLYGON [°]>
(vector_terra - vect("/vsis3/my-bucket/vector_terra.gpkg"))
#> Warning in p@ptr$read(x, layer, query, extent, filter, proxy, what): GDAL
#> Message 1: GPKG: unrecognized user_version=0x00000000 (0) on
#> '/vsis3/my-bucket/vector_terra.gpkg'
#>  class       : SpatVector 
#>  geometry    : none 
#>  dimensions  : 0, 14  (geometries, attributes)
#>  extent      : 0, 0, 0, 0  (xmin, xmax, ymin, ymax)
#>  coord. ref. : lon/lat NAD27 (EPSG:4267) 
#>  names       :  AREA PERIMETER CNTY_ CNTY_ID  NAME  FIPS FIPSNO CRESS_ID BIR74
#>  type        : <num>     <num> <num>   <num> <chr> <chr>  <num>    <int> <num>
#>  SID74 (and 4 more)
#>  <num>

get_bucket("my-bucket", use_https = FALSE, region = "")
#> Bucket: my-bucket 
#> 
#> $Contents
#> Key:            raster_stars.tif 
#> LastModified:   2023-06-29T08:38:50.018Z 
#> ETag:           "5ffc92c4dddf42b95be60b9682732925" 
#> Size (B):       34596 
#> Owner:          minio 
#> Storage class:  STANDARD 
#> 
#> $Contents
#> Key:            vector_sf.gpkg 
#> LastModified:   2023-06-29T08:38:50.132Z 
#> ETag:           "ddf91f915837b0e1b23a2fe1c7b2ae55" 
#> Size (B):       167936 
#> Owner:          minio 
#> Storage class:  STANDARD 
#> 
#> $Contents
#> Key:            vector_terra.gpkg 
#> LastModified:   2023-06-29T08:38:50.194Z 
#> ETag:           "85363a10d6291944385462d420fcd8a6" 
#> Size (B):       167936 
#> Owner:          minio 
#> Storage class:  STANDARD

^{Created on 2023-06-29 with reprex v2.0.2}

Session info

sessioninfo::session_info()
#> ─ Session info ───────────────────────────────────────────────────────────────
#>  setting  value
#>  version  R version 4.3.0 (2023-04-21)
#>  os       Ubuntu 22.04.2 LTS
#>  system   x86_64, linux-gnu
#>  ui       X11
#>  language (EN)
#>  collate  en_US.UTF-8
#>  ctype    en_US.UTF-8
#>  tz       Etc/UTC
#>  date     2023-06-29
#>  pandoc   2.19.2 @ /usr/lib/rstudio-server/bin/quarto/bin/tools/ (via rmarkdown)
#> 
#> ─ Packages ───────────────────────────────────────────────────────────────────
#>  package       * version date (UTC) lib source
#>  abind         * 1.4-5   2016-07-21 [1] RSPM (R 4.3.0)
#>  aws.s3        * 0.3.21  2020-04-07 [1] RSPM (R 4.3.0)
#>  aws.signature   0.6.0   2020-06-01 [1] RSPM (R 4.3.0)
#>  base64enc       0.1-3   2015-07-28 [1] RSPM (R 4.3.0)
#>  class           7.3-21  2023-01-23 [2] CRAN (R 4.3.0)
#>  classInt        0.4-9   2023-02-28 [1] RSPM (R 4.3.0)
#>  cli             3.6.1   2023-03-23 [1] RSPM (R 4.3.0)
#>  codetools       0.2-19  2023-02-01 [2] CRAN (R 4.3.0)
#>  curl            5.0.0   2023-01-12 [1] RSPM (R 4.3.0)
#>  DBI             1.1.3   2022-06-18 [1] RSPM (R 4.3.0)
#>  digest          0.6.31  2022-12-11 [1] RSPM (R 4.3.0)
#>  dplyr           1.1.2   2023-04-20 [1] RSPM (R 4.3.0)
#>  e1071           1.7-13  2023-02-01 [1] RSPM (R 4.3.0)
#>  evaluate        0.21    2023-05-05 [1] RSPM (R 4.3.0)
#>  fansi           1.0.4   2023-01-22 [1] RSPM (R 4.3.0)
#>  fastmap         1.1.1   2023-02-24 [1] RSPM (R 4.3.0)
#>  fs              1.6.2   2023-04-25 [1] RSPM (R 4.3.0)
#>  generics        0.1.3   2022-07-05 [1] RSPM (R 4.3.0)
#>  glue            1.6.2   2022-02-24 [1] RSPM (R 4.3.0)
#>  htmltools       0.5.5   2023-03-23 [1] RSPM (R 4.3.0)
#>  httr            1.4.6   2023-05-08 [1] RSPM (R 4.3.0)
#>  KernSmooth      2.23-20 2021-05-03 [2] CRAN (R 4.3.0)
#>  knitr           1.42    2023-01-25 [1] RSPM (R 4.3.0)
#>  lifecycle       1.0.3   2022-10-07 [1] RSPM (R 4.3.0)
#>  lwgeom          0.2-11  2023-01-14 [1] RSPM (R 4.3.0)
#>  magrittr        2.0.3   2022-03-30 [1] RSPM (R 4.3.0)
#>  mime            0.12    2021-09-28 [1] RSPM (R 4.3.0)
#>  pillar          1.9.0   2023-03-22 [1] RSPM (R 4.3.0)
#>  pkgconfig       2.0.3   2019-09-22 [1] RSPM (R 4.3.0)
#>  proxy           0.4-27  2022-06-09 [1] RSPM (R 4.3.0)
#>  R6              2.5.1   2021-08-19 [1] RSPM (R 4.3.0)
#>  Rcpp            1.0.10  2023-01-22 [1] RSPM (R 4.3.0)
#>  reprex          2.0.2   2022-08-17 [1] RSPM (R 4.3.0)
#>  rlang           1.1.1   2023-04-28 [1] RSPM (R 4.3.0)
#>  rmarkdown       2.21    2023-03-26 [1] RSPM (R 4.3.0)
#>  rstudioapi      0.14    2022-08-22 [1] RSPM (R 4.3.0)
#>  sessioninfo     1.2.2   2021-12-06 [1] RSPM (R 4.3.0)
#>  sf            * 1.0-12  2023-03-19 [1] RSPM (R 4.3.0)
#>  stars         * 0.6-1   2023-04-06 [1] RSPM (R 4.3.0)
#>  terra         * 1.7-29  2023-04-22 [1] RSPM (R 4.3.0)
#>  tibble          3.2.1   2023-03-20 [1] RSPM (R 4.3.0)
#>  tidyselect      1.2.0   2022-10-10 [1] RSPM (R 4.3.0)
#>  units           0.8-2   2023-04-27 [1] RSPM (R 4.3.0)
#>  utf8            1.2.3   2023-01-31 [1] RSPM (R 4.3.0)
#>  vctrs           0.6.2   2023-04-19 [1] RSPM (R 4.3.0)
#>  withr           2.5.0   2022-03-03 [1] RSPM (R 4.3.0)
#>  xfun            0.39    2023-04-20 [1] RSPM (R 4.3.0)
#>  xml2            1.3.4   2023-04-27 [1] RSPM (R 4.3.0)
#>  yaml            2.3.7   2023-01-23 [1] RSPM (R 4.3.0)
#> 
#>  [1] /usr/local/lib/R/site-library
#>  [2] /usr/local/lib/R/library
#> 
#> ──────────────────────────────────────────────────────────────────────────────

[goergen95]

Sorry for the noise! I just found #1159 and with updating terra to v1.7.39 writing rasters also works! Thanks!