Merge pull request #748 from MarkEdmondson1234/gcp-integrations

GCP integrations

NAMESPACE

@@ -19,6 +19,7 @@ S3method(print,tar_resources_clustermq)
 S3method(print,tar_resources_feather)
 S3method(print,tar_resources_fst)
 S3method(print,tar_resources_future)
+S3method(print,tar_resources_gcp)
 S3method(print,tar_resources_parquet)
 S3method(print,tar_resources_qs)
 S3method(print,tar_resources_url)
@@ -29,6 +30,7 @@ S3method(resources_validate,tar_resources_clustermq)
 S3method(resources_validate,tar_resources_feather)
 S3method(resources_validate,tar_resources_fst)
 S3method(resources_validate,tar_resources_future)
+S3method(resources_validate,tar_resources_gcp)
 S3method(resources_validate,tar_resources_parquet)
 S3method(resources_validate,tar_resources_qs)
 S3method(resources_validate,tar_resources_url)
@@ -409,6 +411,7 @@ export(tar_resources_clustermq)
 export(tar_resources_feather)
 export(tar_resources_fst)
 export(tar_resources_future)
+export(tar_resources_gcp)
 export(tar_resources_parquet)
 export(tar_resources_qs)
 export(tar_resources_url)

R/class_resources_gcp.R

@@ -0,0 +1,42 @@
+resources_gcp_init <- function(
+  bucket = NULL,
+  prefix = path_objects_dir_cloud(),
+  verbose = FALSE
+) {
+  resources_gcp_new(
+    bucket = bucket,
+    prefix = prefix,
+    verbose = verbose
+  )
+}
+
+resources_gcp_new <- function(
+  bucket = NULL,
+  prefix = NULL,
+  verbose = FALSE
+) {
+  force(bucket)
+  force(prefix)
+  force(verbose)
+  enclass(environment(), c("tar_resources_gcp", "tar_resources"))
+}
+
+#' @export
+resources_validate.tar_resources_gcp <- function(resources) {
+  tar_assert_scalar(resources$bucket %|||% "bucket")
+  tar_assert_chr(resources$bucket %|||% "bucket")
+  tar_assert_nzchar(resources$bucket %|||% "bucket")
+  tar_assert_scalar(resources$prefix)
+  tar_assert_chr(resources$prefix)
+  tar_assert_nzchar(resources$prefix)
+  tar_assert_lgl(resources$verbose)
+  tar_assert_scalar(resources$verbose)
+}
+
+#' @export
+print.tar_resources_gcp <- function(x, ...) {
+  cat(
+    "<tar_resources_gcp>\n ",
+    paste0(paste_list(as.list(x)), collapse = "\n  ")
+  )
+}

R/tar_resources.R

@@ -46,6 +46,11 @@
 #'   argument of `future::future()` for `targets`.
 #'   Resources supplied through
 #'   `future::plan()` and `future::tweak()` are completely ignored.
+#' @param gcp Output of function `tar_resources_gcp()`.
+#'   Google Cloud Platform bucket storage settings for GCP backed storage
+#'   formats such as `"gcp_qs"` and `"gcp_parquet`. Applies to all formats
+#'   beginning with the `"gcp_"` prefix. For details on formats,
+#'   see the `format` argument of [tar_target()].
 #' @param parquet Output of function `tar_resources_parquet()`.
 #'   Non-default arguments to `arrow::read_parquet()` and
 #'   `arrow::write_parquet()` for `arrow`/parquet-based storage formats.
@@ -78,6 +83,7 @@ tar_resources <- function(
   feather = NULL,
   fst = NULL,
   future = NULL,
+  gcp = NULL,
   parquet = NULL,
   qs = NULL,
   url = NULL

R/tar_resources_gcp.R

@@ -0,0 +1,41 @@
+#' @title Target resources: Google Cloud Platform storage formats
+#' @export
+#' @family resources
+#' @description Create the `gcp` argument of `tar_resources()`
+#'   to specify optional settings to gcp storage formats.
+#'   See the `format` argument of [tar_target()] for details.
+#' @inheritSection tar_resources Resources
+#' @return Object of class `"tar_resources_gcp"`, to be supplied
+#'   to the `gcp` argument of `tar_resources()`.
+#' @param bucket Character of length 1, name of an existing
+#'   gcp bucket to upload and download the return values
+#'   of the affected targets during the pipeline.
+#' @param prefix Character of length 1, "directory path"
+#'   in the S3 bucket where the target return values are stored.
+#' @param verbose Whether to have feedback on the GCS upload process
+#' @examples
+#' # Somewhere in you target script file (usually _targets.R):
+#' if (identical(Sys.getenv("TAR_EXAMPLES"), "true")) {
+#' tar_target(
+#'   name,
+#'   command(),
+#'   format = "gcp_qs",
+#'   resources = tar_resources(
+#'     gcp = tar_resources_gcp(bucket = "yourbucketname"),
+#'     qs = tar_resources_qs(preset = "fast")
+#'   )
+#' )
+#' }
+tar_resources_gcp <- function(
+  bucket,
+  prefix = targets::path_objects_dir_cloud(),
+  verbose = FALSE
+) {
+  out <- resources_gcp_init(
+    bucket = bucket,
+    prefix = prefix,
+    verbose = verbose
+  )
+  resources_validate(out)
+  out
+}

targets.Rproj

@@ -15,3 +15,4 @@ LaTeX: pdfLaTeX
 BuildType: Package
 PackageUseDevtools: Yes
 PackageInstallArgs: --no-multiarch --with-keep.source
+PackageRoxygenize: rd,collate,namespace

tests/testthat/test-class_resources_gcp.R

@@ -0,0 +1,15 @@
+tar_test("create tar_resources_gcp object", {
+  x <- resources_gcp_init(bucket = "bucket_name")
+  expect_silent(resources_validate(x))
+})
+
+tar_test("prohibit empty tar_resources_gcp object", {
+  x <- resources_gcp_init(bucket = "", prefix = NULL)
+  expect_error(resources_validate(x), class = "tar_condition_validate")
+})
+
+tar_test("print tar_resources_aws object", {
+  x <- resources_gcp_init(bucket = "bucket_name")
+  out <- utils::capture.output(print(x))
+  expect_true(any(grepl("tar_resources_gcp", out)))
+})

tests/testthat/test-tar_resources.R

@@ -5,13 +5,15 @@ tar_test("empty resources", {
 tar_test("populated resources", {
   out <- tar_resources(
     aws = resources_aws_init(),
+    gcp = resources_gcp_init(),
     clustermq = resources_clustermq_init(
       template = list(a = 1, n_cores = 123)
     )
   )
   expect_true(is.list(out))
-  expect_equal(sort(names(out)), sort(c("aws", "clustermq")))
+  expect_equal(sort(names(out)), sort(c("aws", "clustermq", "gcp")))
   expect_true(inherits(out$aws, "tar_resources_aws"))
+  expect_true(inherits(out$gcp, "tar_resources_gcp"))
   expect_true(inherits(out$clustermq, "tar_resources_clustermq"))
 })
 

tests/testthat/test-tar_resources_gcp.R

@@ -0,0 +1,6 @@
+tar_test("tar_resources_gcp()", {
+  out <- tar_resources_gcp(bucket = "bucket123")
+  expect_equal(out$bucket, "bucket123")
+  expect_null(out$region)
+  expect_silent(resources_validate(out))
+})