Merge branch 'development'

.Rbuildignore

@@ -7,15 +7,15 @@
 ^\.travis\.yml$
 ^_pkgdown\.yml$
 ^appveyor\.yml$
-cheatsheets/
+^cheatsheets$
 ^CONTRIBUTING\.md$
 ^cran-comments\.md$
+^CRAN-RELEASE$
 ^docs$
 ^LICENSE$
 ^revdep$
 ^travis_wait_.*\.log$
-stickers/
+^stickers$
 tests/testthat/\.httr-oauth$
 vignettes/.*_cache$
 vignettes/.*\.log$
-^CRAN-RELEASE$

.travis.yml

@@ -101,8 +101,8 @@ matrix:
     latex: false
     r_build_args: '--no-build-vignettes'
     r_check_args: '--as-cran --ignore-vignettes --no-examples'
-    before_script:
-      - Rscript -e "install.packages(c('rgdal', 'rgeos'), type = 'source')"
+#    before_script:
+#      - Rscript -e "install.packages(c('rgdal', 'rgeos'), type = 'source')"
 r_build_args: " "
 r_check_args: "--as-cran"
 warnings_are_errors: true

DESCRIPTION

@@ -12,8 +12,8 @@ SystemRequirements: 'unrar' (Linux/macOS) or '7-Zip' (Windows) to work with '.ra
 URL: 
     https://reproducible.predictiveecology.org,
     https://github.com/PredictiveEcology/reproducible
-Date: 2019-11-17
-Version: 0.2.11
+Date: 2020-02-13
+Version: 1.0.0
 Authors@R: c(
     person("Eliot J B", "McIntire", email = "eliot.mcintire@canada.ca",
     role = c("aut", "cre"), comment = c(ORCID = "0000-0002-6914-8316")),
@@ -31,30 +31,27 @@ Authors@R: c(
 Depends:
     R (>= 3.5)
 Imports:
-    archivist (>= 2.1.2),
     backports,
     crayon,
     data.table (>= 1.10.4),
+    DBI,
     digest,
-    dplyr,
     fastdigest,
-    fasterize,
     fpCompare,
     gdalUtils,
-    git2r (>= 0.18),
+    glue,
     googledrive,
     httr,
     magrittr,
     memoise,
     methods,
-    parallel,
+    qs,
     quickPlot,
     R.utils,
     raster,
-    RCurl (>= 1.95-4.8),
+    RSQLite,
     remotes,
     rgdal,
-    rgeos,
     rlang,
     sf,
     sp,
@@ -63,9 +60,16 @@ Imports:
     utils,
     versions
 Suggests:
+    archivist (>= 2.1.2),
     covr,
+    dplyr,
+    fasterize,
     future,
+    git2r (>= 0.18),
     knitr,
+    parallel,
+    RCurl (>= 1.95-4.8),
+    rgeos,
     rmarkdown,
     TimeWarp
 Encoding: UTF-8
@@ -74,8 +78,9 @@ License: GPL-3
 VignetteBuilder: knitr, rmarkdown
 BugReports: https://github.com/PredictiveEcology/reproducible/issues
 ByteCompile: yes
-RoxygenNote: 7.0.0
+RoxygenNote: 7.0.2
 Collate: 
+    'DBI.R'
     'cache-helpers.R'
     'cache-internals.R'
     'cache-tools.R'

NAMESPACE

@@ -37,7 +37,6 @@ export(.cacheMessage)
 export(.checkCacheRepo)
 export(.formalsNotInCurrentDots)
 export(.grepSysCalls)
-export(.objSizeInclEnviros)
 export(.orderDotsUnderscoreFirst)
 export(.preDigestByClass)
 export(.prefix)
@@ -49,7 +48,12 @@ export(.sortDotsUnderscoreFirst)
 export(.suffix)
 export(.tagsByClass)
 export(Cache)
+export(CacheDBFile)
+export(CacheDBTableName)
 export(CacheDigest)
+export(CacheIsACache)
+export(CacheStorageDir)
+export(CacheStoredFile)
 export(Checksums)
 export(Copy)
 export(Filenames)
@@ -73,6 +77,7 @@ export(compareNA)
 export(convertPaths)
 export(convertRasterPaths)
 export(copyFile)
+export(createCache)
 export(cropInputs)
 export(downloadFile)
 export(fastMask)
@@ -84,6 +89,7 @@ export(getUserTags)
 export(installVersions)
 export(installedVersions)
 export(linkOrCopy)
+export(loadFromCache)
 export(makeMemoisable)
 export(maskInputs)
 export(newLibPaths)
@@ -99,6 +105,7 @@ export(prepInputs)
 export(projectInputs)
 export(reproducibleOptions)
 export(retry)
+export(rmFromCache)
 export(searchFull)
 export(searchFullEx)
 export(showCache)
@@ -111,7 +118,6 @@ exportMethods(.addChangedAttr)
 exportMethods(.addTagsToOutput)
 exportMethods(.cacheMessage)
 exportMethods(.checkCacheRepo)
-exportMethods(.objSizeInclEnviros)
 exportMethods(.preDigestByClass)
 exportMethods(.prepareOutput)
 exportMethods(.robustDigest)
@@ -126,7 +132,6 @@ exportMethods(mergeCache)
 exportMethods(normPath)
 exportMethods(showCache)
 import(methods)
-import(parallel)
 importClassesFrom(quickPlot,spatialObjects)
 importClassesFrom(raster,RasterBrick)
 importClassesFrom(raster,RasterLayer)
@@ -141,18 +146,20 @@ importClassesFrom(sp,SpatialPoints)
 importClassesFrom(sp,SpatialPointsDataFrame)
 importClassesFrom(sp,SpatialPolygons)
 importClassesFrom(sp,SpatialPolygonsDataFrame)
+importFrom(DBI,SQL)
+importFrom(DBI,dbAppendTable)
+importFrom(DBI,dbBind)
+importFrom(DBI,dbClearResult)
+importFrom(DBI,dbConnect)
+importFrom(DBI,dbDisconnect)
+importFrom(DBI,dbFetch)
+importFrom(DBI,dbListTables)
+importFrom(DBI,dbSendQuery)
+importFrom(DBI,dbSendStatement)
+importFrom(DBI,dbWriteTable)
 importFrom(R.utils,isAbsolutePath)
 importFrom(R.utils,isFile)
-importFrom(RCurl,url.exists)
-importFrom(archivist,addTagsRepo)
-importFrom(archivist,cache)
-importFrom(archivist,createLocalRepo)
-importFrom(archivist,loadFromLocalRepo)
-importFrom(archivist,rmFromLocalRepo)
-importFrom(archivist,saveToLocalRepo)
-importFrom(archivist,searchInLocalRepo)
-importFrom(archivist,showLocalRepo)
-importFrom(archivist,splitTagsLocal)
+importFrom(RSQLite,SQLite)
 importFrom(crayon,blue)
 importFrom(crayon,magenta)
 importFrom(data.table,":=")
@@ -165,40 +172,19 @@ importFrom(data.table,fread)
 importFrom(data.table,melt)
 importFrom(data.table,rbindlist)
 importFrom(data.table,set)
+importFrom(data.table,setDF)
 importFrom(data.table,setDT)
 importFrom(data.table,setattr)
 importFrom(data.table,setindex)
 importFrom(data.table,setkeyv)
 importFrom(data.table,setnames)
 importFrom(digest,digest)
-importFrom(dplyr,arrange)
-importFrom(dplyr,desc)
-importFrom(dplyr,filter)
-importFrom(dplyr,group_by)
-importFrom(dplyr,left_join)
-importFrom(dplyr,mutate)
-importFrom(dplyr,rename)
-importFrom(dplyr,row_number)
-importFrom(dplyr,select)
 importFrom(fastdigest,fastdigest)
-importFrom(fasterize,fasterize)
 importFrom(fpCompare,"%==%")
 importFrom(gdalUtils,gdal_setInstallation)
 importFrom(gdalUtils,gdalwarp)
-importFrom(git2r,checkout)
-importFrom(git2r,clone)
-importFrom(git2r,commit)
-importFrom(git2r,config)
-importFrom(git2r,cred_ssh_key)
-importFrom(git2r,cred_token)
-importFrom(git2r,head)
-importFrom(git2r,init)
-importFrom(git2r,lookup)
-importFrom(git2r,remote_add)
-importFrom(git2r,remote_set_url)
-importFrom(git2r,remote_url)
-importFrom(git2r,repository)
-importFrom(git2r,status)
+importFrom(glue,double_quote)
+importFrom(glue,glue_sql)
 importFrom(googledrive,as_id)
 importFrom(googledrive,drive_auth)
 importFrom(googledrive,drive_download)
@@ -231,7 +217,8 @@ importFrom(methods,setMethod)
 importFrom(methods,showMethods)
 importFrom(methods,slot)
 importFrom(methods,slotNames)
-importFrom(parallel,detectCores)
+importFrom(qs,qread)
+importFrom(qs,qsave)
 importFrom(quickPlot,isRstudioServer)
 importFrom(raster,"crs<-")
 importFrom(raster,"dataType<-")
@@ -249,13 +236,15 @@ importFrom(raster,getValues)
 importFrom(raster,hasValues)
 importFrom(raster,inMemory)
 importFrom(raster,mask)
+importFrom(raster,maxValue)
 importFrom(raster,ncell)
 importFrom(raster,nlayers)
 importFrom(raster,projectExtent)
 importFrom(raster,projectRaster)
 importFrom(raster,raster)
 importFrom(raster,res)
 importFrom(raster,sampleRandom)
+importFrom(raster,setMinMax)
 importFrom(raster,shapefile)
 importFrom(raster,stack)
 importFrom(raster,tmpDir)
@@ -266,8 +255,8 @@ importFrom(raster,ymax)
 importFrom(raster,ymin)
 importFrom(remotes,install_github)
 importFrom(rgdal,getGDALVersionInfo)
-importFrom(rgeos,gIsValid)
 importFrom(rlang,eval_tidy)
+importFrom(rlang,inherits_only)
 importFrom(rlang,quo)
 importFrom(sf,st_as_sf)
 importFrom(sf,st_buffer)
@@ -299,6 +288,7 @@ importFrom(utils,available.packages)
 importFrom(utils,capture.output)
 importFrom(utils,chooseCRANmirror)
 importFrom(utils,getFromNamespace)
+importFrom(utils,head)
 importFrom(utils,install.packages)
 importFrom(utils,installed.packages)
 importFrom(utils,methods)

NEWS.md

@@ -1,5 +1,30 @@
 Known issues: https://github.com/PredictiveEcology/reproducible/issues
 
+version 1.0.0
+==============
+
+## New features
+
+* This version is not backwards-compatible out of the box. To maintain backwards compatibility, set: `options("reproducible.useDBI" = FALSE)`
+* A new backend was introduced that uses `DBI` package directly, without `archivist`. This has much improved speed. 
+* New option: `options("reproducible.cacheSaveFormat")`. This can be either `rds` (default) or `qs`. All cached objects will be saved with this format. Previously it was `rda`. 
+* Cache objects can now be saved with with `qs::qsave`. In many cases, this has much improved speed and file sizes compared to `rds`; however, testing across a wide range of conditions will occur before it becomes the default.
+* Changed default behaviour for memoising `...` because `Cache` is now much faster, the default is to turn memoising off, via `options("reproducible.useMemoise" = FALSE)`.
+  In cases of large objects, memoising should still be faster, so user can still activate it, setting the option to `TRUE`.
+* Much better SQLite database handling for concurrent write attempts.
+  Tested with dozens of write attempts per second by 3 cores with abundant locked database occurrences.
+* `postProcess` arg `useGDAL` can now take `"force"` as the default behaviour is to not use GDAL if the problem can fit into RAM and `sf` or `raster` tools will be faster than `GDAL` tools
+
+## Dependency changes
+
+* deprecating `archivist`; moved to Suggests.
+* removed imports for `bitops`, `dplyr`, `fasterize`, `flock`, `git2r`, `lubridate`, `RcppArmadillo`, `RCurl` and `tidyselect`. Some of these went to Suggests.
+
+## bug fixes
+
+* `postProcess` calls that use GDAL made more robust (including #93).
+* Several minor, edge cases were detected and fixed.
+
 version 0.2.11
 ==============