Skip to content

Commit

Permalink
#31 no more default parallel handler, download examples
Browse files Browse the repository at this point in the history
  • Loading branch information
eblondel committed Sep 1, 2020
1 parent 7fb7c9a commit f1c3f19
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 23 deletions.
36 changes: 19 additions & 17 deletions R/ZenodoRecord.R
Original file line number Diff line number Diff line change
Expand Up @@ -332,13 +332,13 @@
#' download directory (by default it will be the current working directory).
#'
#' The argument \code{parallel} (default is \code{FALSE}) can be used to parallelize
#' the files download. If set to \code{TRUE}, files will be downloaded in parallel
#' using default \code{mclapply} interface from \pkg{parallel} package. To use a different
#' parallel handler (such as eg \code{parLapply} or \code{parSapply}), specify its function
#' the files download. If set to \code{TRUE}, files will be downloaded in parallel.
#' using the chosen \code{parallel_handler}, eg \code{mclapply} interface from \pkg{parallel} package.
#' To use a different parallel handler (such as eg \code{parLapply} or \code{parSapply}), specify its function
#' in \code{parallel_handler} argument. For cluster-based parallel download, this is the
#' way to proceed. In that case, the cluster should be created earlier by the user with
#' \code{makeCluster} and passed as \code{cl} argument. After downloading all files, the cluster
#' will be stopped automatically.
#' will be stopped automatically. See examples in \code{download_zenodo} utility function.
#'
#' The logical argument \code{quiet} (default is \code{FALSE}) can be set to
#' suppress informative messages (not warnings).
Expand Down Expand Up @@ -1187,20 +1187,22 @@ ZenodoRecord <- R6Class("ZenodoRecord",

if(parallel){
if (!quiet) self$INFO("Download in parallel mode")
if(is.null(parallel_handler)){
if (!quiet) self$INFO("Using default parallel 'mclapply' handler")
if (!quiet) self$INFO(files_summary)
invisible(mclapply(self$files, download_file, ...))
}else{
if (!quiet) self$INFO("Using cluster-based parallel handler")
if(is.null(cl)){
errMsg <- "No cluster object defined as 'cl' argument. Aborting file download..."
self$ERROR(errMsg)
stop(errMsg)
if (is.null(parallel_handler)) {
errMsg <- "No 'parallel_handler' specified"
self$ERROR(errMsg)
stop(errMsg)
}
if(!is.null(parallel_handler)){
if(!is.null(cl)){
if (!quiet) self$INFO("Using cluster-based parallel handler (cluster 'cl' argument specified)")
if (!quiet) self$INFO(files_summary)
invisible(parallel_handler(cl, self$files, download_file, ...))
try(stopCluster(cl))
}else{
if (!quiet) self$INFO("Using non cluster-based (no cluster 'cl' argument specified)")
if (!quiet) self$INFO(files_summary)
invisible(parallel_handler(self$files, download_file, ...))
}
if (!quiet) self$INFO(files_summary)
invisible(parallel_handler(cl, self$files, download_file, ...))
try(stopCluster(cl))
}
}else{
if (!quiet) self$INFO("Download in sequential mode")
Expand Down
9 changes: 8 additions & 1 deletion R/zen4R_downloader.R
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,14 @@
#'
#' @examples
#' \dontrun{
#' download_zenodo("10.5281/zenodo.2547036")
#' #simple download (sequential)
#' download_zenodo("10.5281/zenodo.2547036")
#'
#' #download files as parallel using a cluster approach (for both Unix/Win systems)
#' download_zenodo("10.5281/zenodo.2547036", parallel = TRUE, parallel_handler = parLapply, cl = makeCluster(2))
#'
#' #download files as parallel using mclapply (for Unix systems)
#' download_zenodo("10.5281/zenodo.2547036", parallel = TRUE, parallel_handler = mclapply, mc.cores = 2)
#' }
#'
#' @param doi a Zenodo DOI or concept DOI
Expand Down
8 changes: 4 additions & 4 deletions man/ZenodoRecord.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 8 additions & 1 deletion man/download_zenodo.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit f1c3f19

Please sign in to comment.