diff --git a/.Rbuildignore b/.Rbuildignore index fab1586f..23e90cf2 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -27,3 +27,4 @@ ^data-raw$ ^NEWS\.txt$ ^\.github$ +^\.lintr$ diff --git a/.gitignore b/.gitignore index 84eaaed3..7ea7405e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ _targets.yaml tests/testthat/targets tests/testthat/_targets tests/testthat/_targets_cloudbuild +cloudbuild_targets.yaml diff --git a/NAMESPACE b/NAMESPACE index 471a6960..0119f27e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -71,6 +71,9 @@ export(cr_buildstep_r) export(cr_buildstep_run) export(cr_buildstep_secret) export(cr_buildstep_slack) +export(cr_buildstep_targets) +export(cr_buildstep_targets_setup) +export(cr_buildstep_targets_teardown) export(cr_buildtrigger) export(cr_buildtrigger_copy) export(cr_buildtrigger_delete) diff --git a/R/build_targets.R b/R/build_targets.R index 31fa0ffb..918c0d7d 100644 --- a/R/build_targets.R +++ b/R/build_targets.R @@ -10,22 +10,22 @@ #' #' \itemize{ #' \item Create your `targets` workflow. -#' \item Create a Dockerfile that holds the R and system dependencies for your workflow. You can test the image using \link{cr_deploy_docker()}. Include \code{library(targets)} dependencies - a Docker image with \code{targets} installed is available at \code{gcr.io/gcer-public/targets}. -#' \item Run \code{cr_build_targets()} to create the cloudbuild yaml file. +#' \item Create a Dockerfile that holds the R and system dependencies for your workflow. You can test the image using \link{cr_deploy_docker}. Include \code{library(targets)} dependencies - a Docker image with \code{targets} installed is available at \code{gcr.io/gcer-public/targets}. +#' \item Run \code{cr_build_targets} to create the cloudbuild yaml file. #' \item Run the build via \link{cr_build} or similar. Each build should only recompute outdated targets. #' \item Optionally create a build trigger via \link{cr_buildtrigger}. #' \item Trigger a build. The first trigger will run the targets pipeline, subsequent runs will only recompute the outdated targets. #' } #' #' @return A Yaml object as generated by \link{cr_build_yaml} -#' @param path File path to write the Google Cloud Build yaml workflow file. Set to NULL to write no file and just return the \link{Yaml} object. +#' @param path File path to write the Google Cloud Build yaml workflow file. Set to NULL to write no file and just return the \code{Yaml} object. #' @param task_image An existing Docker image that will be used to run your targets workflow after the targets meta has been downloaded from Google Cloud Storage -#' @param target_folder Where target metadata will sit within the Google Cloud Storage bucket as a folder. Defaults to RStudio project name. +#' @param target_folder Where target metadata will sit within the Google Cloud Storage bucket as a folder. If NULL defaults to RStudio project name or "targets_cloudbuild" if no RStudio project found. #' @param bucket The Google Cloud Storage bucket the target metadata will be saved to in folder `target_folder` -#' @param ... Other arguments passed to \link{cr_build_yaml()} +#' @param ... Other arguments passed to \link{cr_build_yaml} #' @inheritDotParams cr_build_yaml -#' @param task_args A named list of additional arguments to send to \link{cr_buildstep_r()} when its executing the \link[targets]{tar_make()} command (such as environment arguments) -#' @param tar_make The R script that will run in the tar_make() step. Modify to include custom settings such as "script" +#' @param task_args A named list of additional arguments to send to \link{cr_buildstep_r} when its executing the \link[targets]{tar_make} command (such as environment arguments) +#' @param tar_make The R script that will run in the \code{tar_make()} step. Modify to include custom settings #' #' @examples #' @@ -34,22 +34,33 @@ #' # adding custom environment args and secrets to the build #' cr_build_targets( #' task_image = "gcr.io/my-project/my-targets-pipeline", +#' target_folder = "my_targets_project", #' options = list(env = c( #' "ENV1=1234", #' "ENV_USER=Dave" #' )), #' availableSecrets = cr_build_yaml_secrets("MY_PW", "my-pw"), #' task_args = list(secretEnv = "MY_PW") -#' ) +#' @seealso \link{cr_buildstep_targets} if you want to customise the build cr_build_targets <- function(task_image = "gcr.io/gcer-public/targets", - target_folder = basename(rstudioapi::getActiveProject()), + target_folder = NULL, path = "cloudbuild_targets.yaml", bucket = cr_bucket_get(), task_args = list(), tar_make = "targets::tar_make()", ...) { - if (is.null(target_folder)) { - target_folder <- "targets_cloudbuild" + + + if(is.null(target_folder)) { + target_folder <- tryCatch( + basename(rstudioapi::getActiveProject()), + error = function(err){ + NULL + } + ) + if(is.null(target_folder)){ + target_folder <- "targets_cloudbuild" + } } myMessage(sprintf("targets cloud location: gs://%s/%s", bucket, target_folder), @@ -57,41 +68,11 @@ cr_build_targets <- function(task_image = "gcr.io/gcer-public/targets", ) bs <- c( - cr_buildstep_bash( - "mkdir /workspace/_targets && mkdir /workspace/_targets/meta && gsutil -m cp -r ${_TARGET_BUCKET}/_targets/meta /workspace/_targets || exit 0", - name = "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", - entrypoint = "bash", - escape_dollar = FALSE, - id = "get previous _targets metadata" - ), - do.call( - cr_buildstep_r, - args = c( - task_args, - list( - r = tar_make, - name = task_image, - id = "target pipeline" - ) - ) - ), - cr_buildstep_bash( - "date > buildtime.txt && gsutil cp buildtime.txt ${_TARGET_BUCKET}/_targets/buildtime.txt", - name = "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", - entrypoint = "bash", - escape_dollar = FALSE, - id = "Ensure ${_TARGET_BUCKET}/_targets/ always exists" - ), - cr_buildstep_gcloud( - "gsutil", - args = c("-m", "cp", "-r", "/workspace/_targets", "${_TARGET_BUCKET}"), - id = "Upload Artifacts this way as artifacts doesn't support folders" - ), - cr_buildstep_gcloud( - "gsutil", - args = c("ls", "-r", "${_TARGET_BUCKET}"), - id = "Artifacts location" - ) + cr_buildstep_targets_setup("${_TARGET_BUCKET}"), + cr_buildstep_targets(task_args = task_args, + tar_make = tar_make, + task_image = task_image), + cr_buildstep_targets_teardown("${_TARGET_BUCKET}") ) # gs://bucket-name/target-folder @@ -174,7 +155,7 @@ cr_build_targets_artifacts <- function(build, download_folder, { lapply(arts$name, function(x) { - gcs_get_object(x, + googleCloudStorageR::gcs_get_object(x, bucket = bb, saveToDisk = x, overwrite = overwrite @@ -185,3 +166,80 @@ cr_build_targets_artifacts <- function(build, normalizePath(file.path(download_folder, build_folder)) } + + +#' Buildstep to run a targets pipeline on Cloud Build +#' +#' This is a buildstep to help upload a targets pipeline, see \link{cr_build_targets} for examples and suggested workflow +#' @export +#' @param task_args A named list of additional arguments to send to \link{cr_buildstep_r} when its executing the \link[targets]{tar_make} command (such as environment arguments) +#' @param tar_make The R script that will run in the \code{tar_make()} step. Modify to include custom settings +#' @param task_image An existing Docker image that will be used to run your targets workflow after the targets meta has been downloaded from Google Cloud Storage +#' @family Cloud Buildsteps +cr_buildstep_targets <- function( + task_args = list(), + tar_make = "targets::tar_make()", + task_image = "gcr.io/gcer-public/targets"){ + + do.call( + cr_buildstep_r, + args = c( + task_args, + list( + r = tar_make, + name = task_image, + id = "target pipeline" + ) + ) + ) + +} + +#' @export +#' @rdname cr_buildstep_targets +#' @param bucket The Google Cloud Storage bucket and folder the target metadata will be saved to, e.g. \code{gs://my-bucket/my_target_project} You can also pass in build substitution variables such as \code{"${_MY_BUCKET}"}. +cr_buildstep_targets_setup <- function(bucket){ + cr_buildstep_bash( + bash_script = paste( + c("mkdir /workspace/_targets &&", + "mkdir /workspace/_targets/meta &&", + "gsutil -m cp -r", + sprintf("%s/_targets/meta",bucket), + "/workspace/_targets", + "|| exit 0"), collapse = " "), + name = "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", + entrypoint = "bash", + escape_dollar = FALSE, + id = "get previous _targets metadata" + ) +} + +#' @export +#' @rdname cr_buildstep_targets +cr_buildstep_targets_teardown <- function(bucket = cr_bucket_get()){ + c( + cr_buildstep_bash( + bash_script = paste( + c( + "date > buildtime.txt &&", + "gsutil cp buildtime.txt", + sprintf("%s/_targets/buildtime.txt", bucket) + ), collapse = " "), + name = "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", + entrypoint = "bash", + escape_dollar = FALSE, + id = "Ensure bucket/_targets/ always exists" + ), + cr_buildstep_bash( + bash_script = paste( + c("gsutil -m cp -r", "/workspace/_targets", bucket, + "&& gsutil ls -r", bucket), + collapse = " "), + name = "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", + entrypoint = "bash", + escape_dollar = FALSE, + id = "Upload Artifacts" + ) + ) +} + diff --git a/R/buildstep_templates_git.R b/R/buildstep_templates_git.R index 67e2f869..a9a61eef 100644 --- a/R/buildstep_templates_git.R +++ b/R/buildstep_templates_git.R @@ -92,7 +92,6 @@ cr_buildstep_git <- function(git_args = c( #' #' Use \code{git_volume} to add the git credentials folder to other buildsteps #' -#' @examples git_volume <- function() { list(list( name = "ssh", diff --git a/R/buildsteps_templates.R b/R/buildsteps_templates.R index b012c5e2..816bb102 100644 --- a/R/buildsteps_templates.R +++ b/R/buildsteps_templates.R @@ -28,7 +28,7 @@ #' @export #' #' @seealso \url{https://docs.codecov.io/reference#upload} -#' +#' @family Cloud Buildsteps #' @examples #' #' cr_buildstep_packagetests() diff --git a/man/cr_build_targets.Rd b/man/cr_build_targets.Rd index 0685947b..aef84481 100644 --- a/man/cr_build_targets.Rd +++ b/man/cr_build_targets.Rd @@ -7,7 +7,7 @@ \usage{ cr_build_targets( task_image = "gcr.io/gcer-public/targets", - target_folder = basename(rstudioapi::getActiveProject()), + target_folder = NULL, path = "cloudbuild_targets.yaml", bucket = cr_bucket_get(), task_args = list(), @@ -25,15 +25,15 @@ cr_build_targets_artifacts( \arguments{ \item{task_image}{An existing Docker image that will be used to run your targets workflow after the targets meta has been downloaded from Google Cloud Storage} -\item{target_folder}{Where target metadata will sit within the Google Cloud Storage bucket as a folder. Defaults to RStudio project name.} +\item{target_folder}{Where target metadata will sit within the Google Cloud Storage bucket as a folder. If NULL defaults to RStudio project name or "targets_cloudbuild" if no RStudio project found.} -\item{path}{File path to write the Google Cloud Build yaml workflow file. Set to NULL to write no file and just return the \link{Yaml} object.} +\item{path}{File path to write the Google Cloud Build yaml workflow file. Set to NULL to write no file and just return the \code{Yaml} object.} \item{bucket}{The Google Cloud Storage bucket the target metadata will be saved to in folder `target_folder`} -\item{task_args}{A named list of additional arguments to send to \link{cr_buildstep_r()} when its executing the \link[targets]{tar_make()} command (such as environment arguments)} +\item{task_args}{A named list of additional arguments to send to \link{cr_buildstep_r} when its executing the \link[targets]{tar_make} command (such as environment arguments)} -\item{tar_make}{The R script that will run in the tar_make() step. Modify to include custom settings such as "script"} +\item{tar_make}{The R script that will run in the \code{tar_make()} step. Modify to include custom settings} \item{...}{ Arguments passed on to \code{\link[=cr_build_yaml]{cr_build_yaml}} @@ -76,8 +76,8 @@ Steps to set up your target task in Cloud Build: \itemize{ \item Create your `targets` workflow. - \item Create a Dockerfile that holds the R and system dependencies for your workflow. You can test the image using \link{cr_deploy_docker()}. Include \code{library(targets)} dependencies - a Docker image with \code{targets} installed is available at \code{gcr.io/gcer-public/targets}. - \item Run \code{cr_build_targets()} to create the cloudbuild yaml file. + \item Create a Dockerfile that holds the R and system dependencies for your workflow. You can test the image using \link{cr_deploy_docker}. Include \code{library(targets)} dependencies - a Docker image with \code{targets} installed is available at \code{gcr.io/gcer-public/targets}. + \item Run \code{cr_build_targets} to create the cloudbuild yaml file. \item Run the build via \link{cr_build} or similar. Each build should only recompute outdated targets. \item Optionally create a build trigger via \link{cr_buildtrigger}. \item Trigger a build. The first trigger will run the targets pipeline, subsequent runs will only recompute the outdated targets. @@ -93,15 +93,17 @@ cr_build_targets(path = tempfile()) # adding custom environment args and secrets to the build cr_build_targets( task_image = "gcr.io/my-project/my-targets-pipeline", + target_folder = "my_targets_project", options = list(env = c( "ENV1=1234", "ENV_USER=Dave" )), availableSecrets = cr_build_yaml_secrets("MY_PW", "my-pw"), task_args = list(secretEnv = "MY_PW") -) } \seealso{ +\link{cr_buildstep_targets} if you want to customise the build + Other Cloud Build functions: \code{\link{Build}()}, \code{\link{RepoSource}()}, diff --git a/man/cr_buildstep.Rd b/man/cr_buildstep.Rd index 30115f53..63ef955d 100644 --- a/man/cr_buildstep.Rd +++ b/man/cr_buildstep.Rd @@ -135,10 +135,12 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, -\code{\link{cr_buildstep_slack}()} +\code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_bash.Rd b/man/cr_buildstep_bash.Rd index 9326faf2..5e9bc3db 100644 --- a/man/cr_buildstep_bash.Rd +++ b/man/cr_buildstep_bash.Rd @@ -49,11 +49,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_decrypt.Rd b/man/cr_buildstep_decrypt.Rd index 553bf468..85c47666 100644 --- a/man/cr_buildstep_decrypt.Rd +++ b/man/cr_buildstep_decrypt.Rd @@ -52,11 +52,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_df.Rd b/man/cr_buildstep_df.Rd index e2d9f0ea..d3b6d516 100644 --- a/man/cr_buildstep_df.Rd +++ b/man/cr_buildstep_df.Rd @@ -38,11 +38,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_docker.Rd b/man/cr_buildstep_docker.Rd index cc4d489c..e72b100a 100644 --- a/man/cr_buildstep_docker.Rd +++ b/man/cr_buildstep_docker.Rd @@ -93,11 +93,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_edit.Rd b/man/cr_buildstep_edit.Rd index 269eaa92..e705ba28 100644 --- a/man/cr_buildstep_edit.Rd +++ b/man/cr_buildstep_edit.Rd @@ -52,11 +52,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_extract.Rd b/man/cr_buildstep_extract.Rd index 52ff87fc..41fb0c17 100644 --- a/man/cr_buildstep_extract.Rd +++ b/man/cr_buildstep_extract.Rd @@ -34,11 +34,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_gcloud.Rd b/man/cr_buildstep_gcloud.Rd index 02f642d4..f793b509 100644 --- a/man/cr_buildstep_gcloud.Rd +++ b/man/cr_buildstep_gcloud.Rd @@ -40,11 +40,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_git.Rd b/man/cr_buildstep_git.Rd index fef7ef18..8108efaf 100644 --- a/man/cr_buildstep_git.Rd +++ b/man/cr_buildstep_git.Rd @@ -64,11 +64,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gcloud}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_mailgun.Rd b/man/cr_buildstep_mailgun.Rd index 5a7a525a..7af36659 100644 --- a/man/cr_buildstep_mailgun.Rd +++ b/man/cr_buildstep_mailgun.Rd @@ -70,11 +70,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gcloud}()}, \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_nginx_setup.Rd b/man/cr_buildstep_nginx_setup.Rd index 297b7fef..7f91bd38 100644 --- a/man/cr_buildstep_nginx_setup.Rd +++ b/man/cr_buildstep_nginx_setup.Rd @@ -46,11 +46,13 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gcloud}()}, \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_packagetests.Rd b/man/cr_buildstep_packagetests.Rd index 9d1b5540..4ca78f74 100644 --- a/man/cr_buildstep_packagetests.Rd +++ b/man/cr_buildstep_packagetests.Rd @@ -49,4 +49,24 @@ cr_buildstep_packagetests() } \seealso{ \url{https://docs.codecov.io/reference#upload} + +Other Cloud Buildsteps: +\code{\link{cr_buildstep_bash}()}, +\code{\link{cr_buildstep_decrypt}()}, +\code{\link{cr_buildstep_df}()}, +\code{\link{cr_buildstep_docker}()}, +\code{\link{cr_buildstep_edit}()}, +\code{\link{cr_buildstep_extract}()}, +\code{\link{cr_buildstep_gcloud}()}, +\code{\link{cr_buildstep_gitsetup}()}, +\code{\link{cr_buildstep_mailgun}()}, +\code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_pkgdown}()}, +\code{\link{cr_buildstep_run}()}, +\code{\link{cr_buildstep_r}()}, +\code{\link{cr_buildstep_secret}()}, +\code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, +\code{\link{cr_buildstep}()} } +\concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_pkgdown.Rd b/man/cr_buildstep_pkgdown.Rd index 87424fca..25dc3e85 100644 --- a/man/cr_buildstep_pkgdown.Rd +++ b/man/cr_buildstep_pkgdown.Rd @@ -76,10 +76,12 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_r.Rd b/man/cr_buildstep_r.Rd index b2e63a32..19c4819e 100644 --- a/man/cr_buildstep_r.Rd +++ b/man/cr_buildstep_r.Rd @@ -103,10 +103,12 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_run.Rd b/man/cr_buildstep_run.Rd index 92c09099..5e8719c6 100644 --- a/man/cr_buildstep_run.Rd +++ b/man/cr_buildstep_run.Rd @@ -59,10 +59,12 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_secret.Rd b/man/cr_buildstep_secret.Rd index d799f598..8d6df966 100644 --- a/man/cr_buildstep_secret.Rd +++ b/man/cr_buildstep_secret.Rd @@ -53,10 +53,12 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_slack.Rd b/man/cr_buildstep_slack.Rd index 3dda6ffc..30c4794a 100644 --- a/man/cr_buildstep_slack.Rd +++ b/man/cr_buildstep_slack.Rd @@ -62,10 +62,12 @@ Other Cloud Buildsteps: \code{\link{cr_buildstep_gitsetup}()}, \code{\link{cr_buildstep_mailgun}()}, \code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, \code{\link{cr_buildstep_pkgdown}()}, \code{\link{cr_buildstep_run}()}, \code{\link{cr_buildstep_r}()}, \code{\link{cr_buildstep_secret}()}, +\code{\link{cr_buildstep_targets}()}, \code{\link{cr_buildstep}()} } \concept{Cloud Buildsteps} diff --git a/man/cr_buildstep_targets.Rd b/man/cr_buildstep_targets.Rd new file mode 100644 index 00000000..dde1d91a --- /dev/null +++ b/man/cr_buildstep_targets.Rd @@ -0,0 +1,51 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/build_targets.R +\name{cr_buildstep_targets} +\alias{cr_buildstep_targets} +\alias{cr_buildstep_targets_setup} +\alias{cr_buildstep_targets_teardown} +\title{Buildstep to run a targets pipeline on Cloud build} +\usage{ +cr_buildstep_targets( + task_args = list(), + tar_make = "targets::tar_make()", + task_image = "gcr.io/gcer-public/targets" +) + +cr_buildstep_targets_setup(bucket) + +cr_buildstep_targets_teardown(bucket = cr_bucket_get()) +} +\arguments{ +\item{task_args}{A named list of additional arguments to send to \link{cr_buildstep_r} when its executing the \link[targets]{tar_make} command (such as environment arguments)} + +\item{tar_make}{The R script that will run in the \code{tar_make()} step. Modify to include custom settings} + +\item{task_image}{An existing Docker image that will be used to run your targets workflow after the targets meta has been downloaded from Google Cloud Storage} + +\item{bucket}{The Google Cloud Storage bucket and folder the target metadata will be saved to, e.g. \code{gs://my-bucket/my_target_project} You can also pass in build substitution variables such as \code{"${_MY_BUCKET}"}.} +} +\description{ +This is a buildstep to help upload a targets pipeline, see \link{cr_build_targets} for examples and suggested workflow +} +\seealso{ +Other Cloud Buildsteps: +\code{\link{cr_buildstep_bash}()}, +\code{\link{cr_buildstep_decrypt}()}, +\code{\link{cr_buildstep_df}()}, +\code{\link{cr_buildstep_docker}()}, +\code{\link{cr_buildstep_edit}()}, +\code{\link{cr_buildstep_extract}()}, +\code{\link{cr_buildstep_gcloud}()}, +\code{\link{cr_buildstep_gitsetup}()}, +\code{\link{cr_buildstep_mailgun}()}, +\code{\link{cr_buildstep_nginx_setup}()}, +\code{\link{cr_buildstep_packagetests}()}, +\code{\link{cr_buildstep_pkgdown}()}, +\code{\link{cr_buildstep_run}()}, +\code{\link{cr_buildstep_r}()}, +\code{\link{cr_buildstep_secret}()}, +\code{\link{cr_buildstep_slack}()}, +\code{\link{cr_buildstep}()} +} +\concept{Cloud Buildsteps} diff --git a/tests/testthat/_snaps/targets.md b/tests/testthat/_snaps/targets.md index b5bd122c..2ace6e99 100644 --- a/tests/testthat/_snaps/targets.md +++ b/tests/testthat/_snaps/targets.md @@ -25,23 +25,13 @@ args: - -c - date > buildtime.txt && gsutil cp buildtime.txt ${_TARGET_BUCKET}/_targets/buildtime.txt - id: Ensure ${_TARGET_BUCKET}/_targets/ always exists + id: Ensure bucket/_targets/ always exists - name: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine - entrypoint: gsutil - args: - - -m - - cp - - -r - - /workspace/_targets - - ${_TARGET_BUCKET} - id: Upload Artifacts this way as artifacts doesn't support folders - - name: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine - entrypoint: gsutil + entrypoint: bash args: - - ls - - -r - - ${_TARGET_BUCKET} - id: Artifacts location + - -c + - gsutil -m cp -r /workspace/_targets ${_TARGET_BUCKET} && gsutil ls -r ${_TARGET_BUCKET} + id: Upload Artifacts timeout: 3600s substitutions: _TARGET_BUCKET: gs://mark-edmondson-public-files/cr_build_target_tests