Permalink
Browse files

Merge branch 'development'

  • Loading branch information...
achubaty committed Jul 5, 2018
2 parents 3b22cf9 + 0b636c0 commit 518453c51ddf494824d1a2ace7a36e15cea0ba9c
Showing with 3,486 additions and 1,660 deletions.
  1. +3 −2 DESCRIPTION
  2. +6 −0 NAMESPACE
  3. +20 −0 NEWS.md
  4. +9 −6 R/cache-helpers.R
  5. +39 −43 R/cache-tools.R
  6. +147 −93 R/cache.R
  7. +61 −0 R/convertPaths.R
  8. +86 −80 R/download.R
  9. +4 −4 R/gis.R
  10. +28 −21 R/postProcess.R
  11. +68 −4 R/preProcess.R
  12. +90 −34 R/prepInputs.R
  13. +14 −11 cran-comments.md
  14. +1 −1 docs/CONTRIBUTING.html
  15. +860 −860 docs/articles/Intro-to-Cache.html
  16. BIN docs/articles/Intro-to-Cache_files/figure-html/manual-cache-1.png
  17. +1 −1 docs/articles/index.html
  18. +1 −1 docs/authors.html
  19. +1 −1 docs/index.html
  20. +41 −3 docs/news/index.html
  21. +1 −1 docs/reference/Checksums.html
  22. +1 −1 docs/reference/Copy.html
  23. +1 −1 docs/reference/Path-class.html
  24. +1 −1 docs/reference/Require.html
  25. +1 −1 docs/reference/addTagsToOutput.html
  26. +1 −1 docs/reference/available.packagesMem.html
  27. +1 −1 docs/reference/cache.html
  28. +1 −1 docs/reference/cacheHelper.html
  29. +1 −1 docs/reference/cacheMessage.html
  30. +1 −1 docs/reference/checkCacheRepo.html
  31. +1 −1 docs/reference/checkForAuxiliaryFiles.html
  32. +1 −1 docs/reference/checkGDALVersion.html
  33. +1 −1 docs/reference/checkGitConfig.html
  34. +2 −2 docs/reference/checkPath.html
  35. +1 −1 docs/reference/checkoutVersion.html
  36. +86 −86 docs/reference/clearStubArtifacts.html
  37. +1 −1 docs/reference/compareNA.html
  38. +219 −0 docs/reference/convertPaths.html
  39. +1 −1 docs/reference/copyFile.html
  40. +4 −6 docs/reference/cropInputs.html
  41. +1 −1 docs/reference/debugCache.html
  42. +2 −2 docs/reference/determineFilename.html
  43. +1 −1 docs/reference/digest.html
  44. +1 −1 docs/reference/dlGeneric.html
  45. +1 −1 docs/reference/dlGoogle.html
  46. +10 −10 docs/reference/downloadFile.html
  47. +1 −1 docs/reference/extractFromArchive.html
  48. +1 −1 docs/reference/fastMask.html
  49. +1 −1 docs/reference/fixErrors.SpatialPolygons.html
  50. +2 −2 docs/reference/fixErrors.html
  51. +1 −1 docs/reference/getGDALVersion.html
  52. +1 −1 docs/reference/guessAtTarget.html
  53. +13 −1 docs/reference/index.html
  54. +1 −1 docs/reference/installPackages.html
  55. +1 −1 docs/reference/installVersions.html
  56. +2 −2 docs/reference/installedVersions.html
  57. +180 −0 docs/reference/listFilesInArchive.html
  58. +193 −0 docs/reference/makeMemoiseable.html
  59. +2 −2 docs/reference/maskInputs.html
  60. +1 −1 docs/reference/mergeCache.html
  61. +1 −1 docs/reference/newLibPaths.html
  62. +2 −2 docs/reference/normPath.html
  63. +1 −1 docs/reference/objSize.html
  64. +1 −1 docs/reference/objSizeInclEnviros.html
  65. +1 −1 docs/reference/package_dependenciesMem.html
  66. +2 −2 docs/reference/pipe.html
  67. +1 −1 docs/reference/pipe2.html
  68. +1 −1 docs/reference/pkgDep.html
  69. +1 −1 docs/reference/pkgSnapshot.html
  70. +2 −2 docs/reference/postProcess.html
  71. +1 −1 docs/reference/postProcessHelpers.html
  72. +1 −1 docs/reference/preDigestByClass.html
  73. +53 −11 docs/reference/preProcess.html
  74. +1 −1 docs/reference/prefix.html
  75. +18 −13 docs/reference/prepInputs.html
  76. +3 −3 docs/reference/prepareFileBackedRaster.html
  77. +2 −2 docs/reference/prepareOutput.html
  78. +2 −2 docs/reference/projectInputs.html
  79. +10 −10 docs/reference/purge.html
  80. +17 −16 docs/reference/readLinesRcpp.html
  81. +17 −16 docs/reference/readLinesRcppInternal.html
  82. +1 −1 docs/reference/reproducible-deprecated.html
  83. +1 −1 docs/reference/reproducible-package.html
  84. +4 −4 docs/reference/robustDigest.html
  85. +25 −34 docs/reference/search.html
  86. +1 −1 docs/reference/sortDotsUnderscoreFirst.html
  87. +1 −1 docs/reference/tagsByClass.html
  88. +98 −96 docs/reference/viewCache.html
  89. +2 −2 docs/reference/writeOutputs.html
  90. +9 −0 docs/sitemap.xml
  91. +47 −0 man/convertPaths.Rd
  92. +9 −9 man/downloadFile.Rd
  93. +18 −0 man/listFilesInArchive.Rd
  94. +30 −0 man/makeMemoiseable.Rd
  95. +50 −10 man/preProcess.Rd
  96. +17 −11 man/prepInputs.Rd
  97. +9 −9 man/purge.Rd
  98. +26 −24 man/viewCache.Rd
  99. +51 −22 revdep/README.md
  100. +14 −7 revdep/check.R
  101. BIN revdep/checks.rds
  102. +59 −0 revdep/problems.md
  103. +3 −3 revdep/timing.md
  104. +0 −4 tests/testthat/test-cache.R
  105. +0 −18 tests/testthat/test-examples.R
  106. +650 −8 tests/testthat/test-prepInputs.R
View
@@ -11,8 +11,8 @@ Description: Collection of high-level, robust, machine- and OS-independent tools
URL:
http://reproducible.predictiveecology.org,
https://github.com/PredictiveEcology/reproducible
Date: 2018-06-15
Version: 0.2.0
Date: 2018-07-05
Version: 0.2.1
Authors@R: c(
person("Eliot J B", "McIntire", email = "eliot.mcintire@canada.ca",
role = c("aut", "cre")),
@@ -72,6 +72,7 @@ Collate:
'cache.R'
'checksums.R'
'consistentPaths.R'
'convertPaths.R'
'download.R'
'gis.R'
'git.R'
View
@@ -5,6 +5,7 @@ S3method(cropInputs,default)
S3method(cropInputs,spatialObjects)
S3method(fixErrors,SpatialPolygons)
S3method(fixErrors,default)
S3method(makeMemoiseable,default)
S3method(maskInputs,Raster)
S3method(maskInputs,Spatial)
S3method(objSize,Path)
@@ -16,6 +17,7 @@ S3method(postProcess,spatialObjects)
S3method(projectInputs,Raster)
S3method(projectInputs,Spatial)
S3method(projectInputs,sf)
S3method(unmakeMemoiseable,default)
export("%<%")
export("%>%")
export("%C%")
@@ -44,6 +46,8 @@ export(checkoutVersion)
export(clearCache)
export(clearStubArtifacts)
export(compareNA)
export(convertPaths)
export(convertRasterPaths)
export(copyFile)
export(cropInputs)
export(downloadFile)
@@ -52,6 +56,7 @@ export(fixErrors)
export(getGDALVersion)
export(installVersions)
export(installedVersions)
export(makeMemoiseable)
export(maskInputs)
export(newLibPaths)
export(normPath)
@@ -66,6 +71,7 @@ export(readLinesRcpp)
export(searchFull)
export(searchFullEx)
export(showCache)
export(unmakeMemoiseable)
export(writeOutputs)
exportClasses(Path)
exportMethods(.addTagsToOutput)
View
20 NEWS.md
@@ -1,5 +1,25 @@
Known issues: https://github.com/PredictiveEcology/reproducible/issues
version 0.2.1
=============
## New features
* new functions:
- `convertPaths` and `convertRasterPaths` to assist with renaming moved files.
* `prepInputs` -- new features
- `alsoExtract` now has more options (`NULL`, `NA`, `"similar"`) and defaults to extracting all files in an archive (`NULL`).
- skips `postProcess` altogether if no `studyArea` or `rasterToMatch`. Previously, this would invoke Cache even if there was nothing to `postProcess`.
## Bug fixes
* `copyFile` correctly handles directory names containing spaces.
* `makeMemoisable` fixed to handle additonal edge cases.
* other minor bug fixes.
version 0.2.0
=============
View
@@ -725,14 +725,14 @@ copyFile <- function(from = NULL, to = NULL, useRobocopy = TRUE,
robocopyBin <- tryCatch(Sys.which("robocopy"), warning = function(w) NA_character_)
robocopy <- if (silent) {
paste0(robocopyBin, " /purge"[delDestination], " /ETA /XJ /XO /NDL /NFL /NJH /NJS ", # nolint
normalizePath(dirname(from), mustWork = TRUE, winslash = "\\"), " ",
normalizePath(to, mustWork = FALSE, winslash = "\\"), " ",
paste0(robocopyBin, " /purge"[delDestination], " /ETA /XJ /XO /NDL /NFL /NJH /NJS \"", # nolint
normalizePath(dirname(from), mustWork = TRUE, winslash = "\\"), "\" \"",
normalizePath(to, mustWork = FALSE, winslash = "\\"), "\" ",
basename(from))
} else {
paste0(robocopyBin, " /purge"[delDestination], " /ETA /XJ /XO ", # nolint
normalizePath(dirname(from), mustWork = TRUE, winslash = "\\"), " ",
normalizePath(to, mustWork = FALSE, winslash = "\\"), " ",
paste0(robocopyBin, " /purge"[delDestination], " /ETA /XJ /XO \"", # nolint
normalizePath(dirname(from), mustWork = TRUE, winslash = "\\"), "\" \"",
normalizePath(to, mustWork = FALSE, winslash = "\\"), "\" ",
basename(from))
}
@@ -741,6 +741,9 @@ copyFile <- function(from = NULL, to = NULL, useRobocopy = TRUE,
} else {
TRUE
}
if (isTRUE(any(grepl("ERROR", useFileCopy)))) {
useFileCopy <- TRUE
}
} else if ( (os == "linux") || (os == "darwin") ) { # nolint
if (!dir.exists(to)) to <- dirname(to) # extract just the directory part
rsyncBin <- tryCatch(Sys.which("rsync"), warning = function(w) NA_character_)
View
@@ -4,25 +4,6 @@
#' @param before A time (POSIX, character understandable by data.table).
#' Objects cached before this time will be shown or deleted.
#' @param ... Other arguments. Currently unused.
#'
#' If neither \code{after} or \code{before} are provided, nor \code{userTags},
#' then all objects will be removed.
#' If both \code{after} and \code{before} are specified, then all objects between \code{after} and
#' \code{before} will be deleted.
#' If \code{userTags} is used, this will override \code{after} or \code{before}.
#'
#' @return Will clear all (or that match \code{userTags}, or between \code{after} or \code{before})
#' objects from the repository located at \code{cachePath} of the sim object,
#' if \code{sim} is provided, or located in \code{cacheRepo}. Also returns a data.table invisibly
#' of the removed items.
#'
#' @note If the cache is larger than 10MB, and clearCache is used, there will be a message
#' and a pause, if interactive, to prevent accidentally deleting of a large cache repository.
#'
#'
#' @export
#' @importFrom archivist rmFromLocalRepo searchInLocalRepo
#' @importFrom methods setGeneric setMethod
#' @param userTags Character vector. If used, this will be used in place of the
#' \code{after} and \code{before}.
#' Specifying one or more \code{userTag} here will clear all
@@ -33,8 +14,28 @@
#' Matching will be against any of the 3 columns returned by \code{showCache()},
#' i.e., \code{artifact}, \code{tagValue} or \code{tagName}.
#' Also, length \code{userTags} > 1, then matching is by `and`.
#' For `or` matching, use | in a single character string. See examples.
#' For `or` matching, use \code{|} in a single character string.
#' See examples.
#'
#' If neither \code{after} or \code{before} are provided, nor \code{userTags},
#' then all objects will be removed.
#' If both \code{after} and \code{before} are specified, then all objects between
#' \code{after} and \code{before} will be deleted.
#' If \code{userTags} is used, this will override \code{after} or \code{before}.
#'
#' @return Will clear all objects (or those that match \code{userTags}, or those
#' between \code{after} or \code{before}) from the repository located at
#' \code{cachePath} of the sim object, if \code{sim} is provided, or located in
#' \code{cacheRepo}.
#' Invisibly returns a \code{data.table} of the removed items.
#'
#' @note If the cache is larger than 10MB, and clearCache is used, there will be
#' a message and a pause, if interactive, to prevent accidentally deleting of a
#' large cache repository.
#'
#' @export
#' @importFrom archivist rmFromLocalRepo searchInLocalRepo
#' @importFrom methods setGeneric setMethod
#' @rdname viewCache
#'
#' @examples
@@ -70,7 +71,6 @@
#' clearCache(tmpDir, userTags = toRemove)
#' cacheAfter <- showCache(tmpDir, userTags = c("runif")) # Only the small one is left
#'
#'
setGeneric("clearCache", function(x, userTags = character(), after, before, ...) {
standardGeneric("clearCache")
})
@@ -95,7 +95,7 @@ setMethod(
formattedCacheSize <- format(cacheSize, "auto")
if (cacheSize > 1e7) {
message("Your current cache size is ", formattedCacheSize,
message("Your current cache size is ", formattedCacheSize, ".\n",
" Are you sure you would like to delete it all? Y or N")
rl <- readline()
if (!identical(toupper(rl), "Y")) {
@@ -109,7 +109,7 @@ setMethod(
unlink(file.path(x, "backpack.db"))
checkPath(x, create = TRUE)
createLocalRepo(x)
memoise::forget(loadFromLocalRepoMem)
memoise::forget(.loadFromLocalRepoMem)
return(invisible())
}
@@ -127,8 +127,9 @@ setMethod(
}
if (NROW(objsDT)) {
rastersInRepo <- objsDT[grepl(pattern = "class", tagKey) & grepl(pattern = "Raster", tagValue)] # only Rasters* class
if (all(!is.na(rastersInRepo$artifact)) && NROW(rastersInRepo)>0) {
rastersInRepo <- objsDT[grepl(pattern = "class", tagKey) &
grepl(pattern = "Raster", tagValue)] # only Rasters* class
if (all(!is.na(rastersInRepo$artifact)) && NROW(rastersInRepo) > 0) {
suppressWarnings(rasters <- lapply(rastersInRepo$artifact, function(ras) {
loadFromLocalRepo(ras, repoDir = x, value = TRUE)
}))
@@ -142,30 +143,29 @@ setMethod(
cacheSize <- sum(cacheSize, file.size(dirLs))
}
}
}
if (interactive()) {
class(cacheSize) <- "object_size"
formattedCacheSize <- format(cacheSize, "auto")
if (cacheSize > 1e7) {
message("Your current cache size is ", formattedCacheSize,
message("Your current cache size is ", formattedCacheSize, ".\n",
" Are you sure you would like to delete it all? Y or N")
rl <- readline()
if (!identical(rl, "Y")) {
if (!identical(toupper(rl), "Y")) {
message("Aborting clearCache")
return(invisible())
}
}
}
if (all(!is.na(rastersInRepo$artifact)) && NROW(rastersInRepo)>0) {
if (all(!is.na(rastersInRepo$artifact)) && NROW(rastersInRepo) > 0) {
unlink(filesToRemove)
}
suppressWarnings(rmFromLocalRepo(unique(objsDT$artifact), x, many = TRUE))
}
memoise::forget(loadFromLocalRepoMem)
memoise::forget(.loadFromLocalRepoMem)
return(invisible(objsDT))
})
@@ -218,7 +218,7 @@ setMethod(
if (length(userTags) > 0) {
for (ut in userTags) {
objsDT2 <- objsDT[
grepl(tagValue, pattern = ut) |
grepl(tagValue, pattern = ut) |
grepl(tagKey, pattern = ut) |
grepl(artifact, pattern = ut)]
setkeyv(objsDT2, "artifact")
@@ -252,21 +252,18 @@ setMethod(
args <- append(list(x = x, after = after, before = before, userTags = userTags),
list(...))
objsDTAll <- showCache(x)
objsDTAll <- suppressMessages(showCache(x))
objsDT <- do.call(showCache, args = args)
keep <- unique(objsDT$artifact)
eliminate <- unique(objsDTAll$artifact[!(objsDTAll$artifact %in% keep)])
if (length(eliminate)) {
eliminate <- paste(eliminate, collapse = "|")
eliminate <- paste(eliminate, collapse = "|") ## TODO: remove
clearCache(x, eliminate)
}
return(objsDT)
})
##############################################################
##############################################################
#' Merge two cache repositories together
#'
#' All the \code{cacheFrom} artifacts will be put into \code{cacheTo}
@@ -343,24 +340,23 @@ setMethod(
.messageCacheSize(cacheTo)
return(invisible(cacheTo))
})
})
#' @keywords internal
.messageCacheSize <- function(x, artifacts = NULL) {
fsTotal <- sum(file.size(dir(x, full.names = TRUE, recursive = TRUE)))
class(fsTotal) <- "object_size"
preMessage1 <- " Total (including Rasters): "
fs <- sum(file.size(dir(file.path(x, "gallery"),
pattern = paste(collapse = "|",
unique(artifacts)), full.names = TRUE)))
allFiles <- dir(file.path(x, "gallery"), full.names = TRUE)
matchingIDs <- which(basename(allFiles) %in% paste0(unique(artifacts), ".rda"))
matchingFiles <- allFiles[matchingIDs]
fs <- sum(file.size(matchingFiles))
class(fs) <- "object_size"
preMessage <- " Selected objects (not including Rasters): "
message("Cache size: ")
message(preMessage1, format(fsTotal, "auto"))
message(preMessage, format(fs, "auto"))
}
Oops, something went wrong.

0 comments on commit 518453c

Please sign in to comment.