From 6e0059f0f6212cdb7c4557d62eb67b043b991b24 Mon Sep 17 00:00:00 2001 From: Mark Edmondson Date: Sun, 19 Apr 2020 18:13:51 +0200 Subject: [PATCH] updates for #54 --- DESCRIPTION | 3 +- R/buildsteps_templates.R | 84 +++++++++++++++++-------------------- R/cloudbuild.R | 9 +--- R/deploy.R | 37 +++++++++------- inst/ssh/github_setup.sh | 9 ++++ man/cr_build_make.Rd | 7 +--- man/cr_buildstep_git.Rd | 23 +++------- man/cr_buildstep_pkgdown.Rd | 34 ++++++++------- man/cr_deploy_pkgdown.Rd | 12 ++---- tests/testthat/test-build.R | 8 ++-- vignettes/git.Rmd | 51 ++++++++++++++++++++++ 11 files changed, 161 insertions(+), 116 deletions(-) create mode 100644 inst/ssh/github_setup.sh create mode 100644 vignettes/git.Rmd diff --git a/DESCRIPTION b/DESCRIPTION index 67af4191..3e1363df 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -23,6 +23,7 @@ Imports: httr (>= 1.4.1), jsonlite (>= 1.5), openssl (>= 1.4.1), + usethis (>= 1.6.0), utils, yaml (>= 2.2.0) Suggests: @@ -35,4 +36,4 @@ Suggests: License: MIT + file LICENSE Encoding: UTF-8 LazyData: true -RoxygenNote: 7.0.2 +RoxygenNote: 7.1.0 diff --git a/R/buildsteps_templates.R b/R/buildsteps_templates.R index 4f11d41d..c2547f63 100644 --- a/R/buildsteps_templates.R +++ b/R/buildsteps_templates.R @@ -414,7 +414,7 @@ cr_buildstep_r <- function(r, ) # ability to call R scripts from Cloud Storage - if(grepl("^gs://", r)){ + if(grepl("^gs://", r[[1]])){ r_here <- paste0("/workspace/", basename(r)) myMessage(paste0("Buildstep will download R script from ", r), level = 3) @@ -655,14 +655,10 @@ cr_buildstep_docker <- function(image, #' #' This creates steps to configure git to use an ssh created key. #' -#' @param keyring The Key Management Store keyring containing the git ssh key -#' @param key The Key Management Store key containing the gitssh key -#' @param cipher The filename of the encrypted git ssh key that has been checked into the repository +#' @param secret The name of the secret on Google Secret Manager for the git ssh private key #' @details #' -#' The key should be encrypted offline using \code{gcloud kms} or similar first. See \link{cr_buildstep_decrypt} for details. -#' -#' By default the encrypted key should then be at the root of your \link{Source} object called "id_rsa.enc" +#' The ssh private key should be uploaded to Google Secret Manager first #' #' @seealso \href{https://cloud.google.com/cloud-build/docs/access-private-github-repos}{Accessing private GitHub repositories using Cloud Build (google article)} #' @@ -672,39 +668,27 @@ cr_buildstep_docker <- function(image, #' cr_project_set("my-project") #' cr_bucket_set("my-bucket") #' -#' # assumes you have previously saved git ssh key via KMS called "git_key" +#' # assumes you have previously saved git ssh key called "github-ssh" #' cr_build_yaml( #' steps = c( -#' cr_buildstep_gitsetup("my_keyring", "git_key"), +#' cr_buildstep_gitsetup("github-ssh"), #' cr_buildstep_git(c("clone", #' "git@github.com:github_name/repo_name")) #' ) #' ) #' -cr_buildstep_gitsetup <- function(keyring = "my-keyring", - key = "github-key", - cipher = "id_rsa.enc", ...){ - # don't allow dot names that would break things - dots <- list(...) - assert_that( - is.null(dots$name), - is.null(dots$args), - is.null(dots$prefix), - is.null(dots$entrypoint) - ) - - cb <- system.file("cloudbuild/cloudbuild_git.yml", - package = "googleCloudRunner") - bs <- cr_build_make(cb) - +cr_buildstep_gitsetup <- function(secret){ + github_setup <- system.file("ssh", "github_setup.sh", + package = "googleCloudRunner") c( - cr_buildstep_decrypt(cipher = cipher, - plain = "/root/.ssh/id_rsa", - keyring = keyring, - key = key, - volumes = git_volume()), - cr_buildstep_extract(bs, 2) + cr_buildstep_secret(secret = secret, + decrypted = "/root/.ssh/id_rsa", + volumes = git_volume()), + cr_buildstep_bash(github_setup, + name = "gcr.io/cloud-builders/git", + entrypoint = "bash", + volumes = git_volume()) ) } @@ -720,6 +704,7 @@ cr_buildstep_gitsetup <- function(keyring = "my-keyring", #' \code{cr_buildstep} must come after \code{cr_buildstep_gitsetup} #' @family Cloud Buildsteps #' @export +#' @import assertthat cr_buildstep_git <- function( git_args = c("clone", "git@github.com:[GIT-USERNAME]/[REPOSITORY]", @@ -754,39 +739,48 @@ cr_buildstep_git <- function( #' #' Its convenient to set some of the above via \link{Build} macros, such as \code{github_repo=$_GITHUB_REPO} and \code{git_email=$_BUILD_EMAIL} in the Build Trigger web UI #' +#' To commit the website to git, \link{cr_buildstep_gitsetup} is used for which +#' you will need to add your git ssh private key to Google Secret Manager +#' +#' The R package is installed via \link[devtools]{install} before +#' running \link[pkgdown]{build_site} +#' #' @export #' @family Cloud Buildsteps #' @examples #' cr_project_set("my-project") #' cr_bucket_set("my-bucket") +#' +#' # set github repo directly to write it out via cr_build_write() +#' cr_buildstep_pkgdown("MarkEdmondson1234/googleCloudRunner", +#' git_email = "cloudbuild@google.com", +#' secret = "github-ssh") +#' #' # github repo set via build trigger macro _GITHUB_REPO #' cr_buildstep_pkgdown("$_GITHUB_REPO", -#' "cloudbuild@google.com") +#' git_email = "cloudbuild@google.com", +#' secret = "github-ssh") #' #' # example including environment arguments for pkgdown build step -#' steps <- cr_buildstep_pkgdown("$_GITHUB_REPO", -#' "cloudbuild@google.com", +#' cr_buildstep_pkgdown("$_GITHUB_REPO", +#' git_email = "cloudbuild@google.com", +#' secret = "github-ssh", #' env = c("MYVAR=$_MY_VAR", "PROJECT=$PROJECT_ID")) -#' build_yaml <- cr_build_yaml(steps = steps) -#' my_source <- cr_build_source(RepoSource("my_repo", branch="master")) -#' build <- cr_build_make(build_yaml, source = my_source) +#' cr_buildstep_pkgdown <- function( github_repo, git_email, - keyring = "my-keyring", - key = "github-key", + secret, env = NULL, - cipher = "id_rsa.enc", - build_image = 'gcr.io/gcer-public/packagetools:master'){ + build_image = "gcr.io/gcer-public/packagetools:master"){ repo <- paste0("git@github.com:", github_repo) c( - cr_buildstep_gitsetup(keyring = keyring, - key = key, - cipher = cipher), + cr_buildstep_gitsetup(secret), cr_buildstep_git(c("clone",repo, "repo")), - cr_buildstep_r(c("devtools::install()", "pkgdown::build_site()"), + cr_buildstep_r(c("devtools::install()", + "pkgdown::build_site()"), name = build_image, dir = "repo", env = env), diff --git a/R/cloudbuild.R b/R/cloudbuild.R index f8d66bc1..4619c923 100644 --- a/R/cloudbuild.R +++ b/R/cloudbuild.R @@ -148,19 +148,14 @@ extract_logs <- function(o){ #' @examples #' cloudbuild <- system.file("cloudbuild/cloudbuild.yaml", #' package = "googleCloudRunner") -#' cr_build_make(cloudbuild, projectId = "test-project") +#' cr_build_make(cloudbuild) cr_build_make <- function(yaml, source = NULL, timeout=NULL, images=NULL, artifacts = NULL, options = NULL, - substitutions = NULL, - projectId = cr_project_get()){ - - assert_that( - is.string(projectId) - ) + substitutions = NULL){ stepsy <- get_cr_yaml(yaml) if(is.null(stepsy$steps)){ diff --git a/R/deploy.R b/R/deploy.R index c1d92b14..2a30f156 100644 --- a/R/deploy.R +++ b/R/deploy.R @@ -229,12 +229,10 @@ cr_deploy_docker <- function(local, #' unlink("cloudbuild-pkgdown.yml") #' cr_deploy_pkgdown <- function(steps = NULL, + secret, cloudbuild_file = "cloudbuild-pkgdown.yml", git_email = "googlecloudrunner@r.com", - keyring = "my-keyring", - key = "github-key", env = NULL, - cipher = "id_rsa.enc", build_image = 'gcr.io/gcer-public/packagetools:master'){ @@ -242,17 +240,23 @@ cr_deploy_pkgdown <- function(steps = NULL, cr_build_yaml(steps = c(steps, cr_buildstep_pkgdown("$_GIT_REPO", git_email = git_email, - env = env)) + secret = secret, + env = env, + build_image = build_image)) ) build <- cr_build_make(build_yaml) cr_build_write(build, file = cloudbuild_file) - myMessage("Now make a build trigger pointing at this file in your repo: ", cloudbuild_file, - "\nBuild Trigger settings:", - "\nSubstitution variable: _GITHUB_REPO = {your-repo-to-push-to}", - "\nIgnored files filter (glob): docs/**", - level = 3) + usethis::ui_line() + usethis::ui_info("Complete deployment of pkgdown Cloud Build yaml:") + usethis::ui_todo(c( + "Go to https://console.cloud.google.com/cloud-build/triggers and + make a build trigger pointing at this file in your repo: + {cloudbuild_file} ")) + usethis::ui_info(c("Build Trigger substitution variable settings:", + "_GITHUB_REPO = username/repo", + "Ignored files filter (glob): docs/**")) invisible(build) @@ -318,11 +322,16 @@ cr_deploy_packagetests <- function( ) cr_build_write(build_yaml, file = cloudbuild_file) - myMessage("Now make a build trigger pointing at this file in your repo: ", cloudbuild_file, - "\nBuild Trigger settings:", - "\nSubstitution variable: _CODECOV_TOKEN = {your-codecov-token}", - "\nIgnored files filter (glob): docs/** and vignettes/**", - level = 3) + + usethis::ui_line() + usethis::ui_info("Complete deployment of tests Cloud Build yaml:") + usethis::ui_todo(c( + "Go to https://console.cloud.google.com/cloud-build/triggers and + make a build trigger pointing at this file in your repo: + {cloudbuild_file} ")) + usethis::ui_info(c("Build Trigger substitution variable settings:", + "_CODECOV_TOKEN = your-codecov-token", + "Ignored files filter (glob): docs/** and vignettes/**")) invisible(build_yaml) diff --git a/inst/ssh/github_setup.sh b/inst/ssh/github_setup.sh new file mode 100644 index 00000000..7a654166 --- /dev/null +++ b/inst/ssh/github_setup.sh @@ -0,0 +1,9 @@ +chmod 600 /root/.ssh/id_rsa +cat <known_hosts +github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== +EOF +cat </root/.ssh/config +Hostname github.com +IdentityFile /root/.ssh/id_rsa +EOF +mv known_hosts /root/.ssh/known_hosts diff --git a/man/cr_build_make.Rd b/man/cr_build_make.Rd index 500ecb14..14702352 100644 --- a/man/cr_build_make.Rd +++ b/man/cr_build_make.Rd @@ -11,8 +11,7 @@ cr_build_make( images = NULL, artifacts = NULL, options = NULL, - substitutions = NULL, - projectId = cr_project_get() + substitutions = NULL ) } \arguments{ @@ -29,8 +28,6 @@ cr_build_make( \item{options}{Options} \item{substitutions}{Substitutions data for `Build` resource} - -\item{projectId}{ID of the project} } \description{ This creates a \link{Build} object via the standard cloudbuild.yaml format @@ -38,7 +35,7 @@ This creates a \link{Build} object via the standard cloudbuild.yaml format \examples{ cloudbuild <- system.file("cloudbuild/cloudbuild.yaml", package = "googleCloudRunner") -cr_build_make(cloudbuild, projectId = "test-project") +cr_build_make(cloudbuild) } \seealso{ https://cloud.google.com/cloud-build/docs/build-config diff --git a/man/cr_buildstep_git.Rd b/man/cr_buildstep_git.Rd index d416b00a..7a5bf728 100644 --- a/man/cr_buildstep_git.Rd +++ b/man/cr_buildstep_git.Rd @@ -5,12 +5,7 @@ \alias{cr_buildstep_git} \title{Create a build step for authenticating with Git} \usage{ -cr_buildstep_gitsetup( - keyring = "my-keyring", - key = "github-key", - cipher = "id_rsa.enc", - ... -) +cr_buildstep_gitsetup(secret) cr_buildstep_git( git_args = c("clone", "git@github.com:[GIT-USERNAME]/[REPOSITORY]", "."), @@ -18,15 +13,11 @@ cr_buildstep_git( ) } \arguments{ -\item{keyring}{The Key Management Store keyring containing the git ssh key} +\item{secret}{The name of the secret on Google Secret Manager for the git ssh private key} -\item{key}{The Key Management Store key containing the gitssh key} - -\item{cipher}{The filename of the encrypted git ssh key that has been checked into the repository} +\item{git_args}{The arguments to send to git} \item{...}{Further arguments passed in to \link{cr_buildstep}} - -\item{git_args}{The arguments to send to git} } \description{ This creates steps to configure git to use an ssh created key. @@ -34,9 +25,7 @@ This creates steps to configure git to use an ssh created key. This creates steps to use git with an ssh created key. } \details{ -The key should be encrypted offline using \code{gcloud kms} or similar first. See \link{cr_buildstep_decrypt} for details. - -By default the encrypted key should then be at the root of your \link{Source} object called "id_rsa.enc" +The ssh private key should be uploaded to Google Secret Manager first \code{cr_buildstep} must come after \code{cr_buildstep_gitsetup} } @@ -44,10 +33,10 @@ By default the encrypted key should then be at the root of your \link{Source} ob cr_project_set("my-project") cr_bucket_set("my-bucket") -# assumes you have previously saved git ssh key via KMS called "git_key" +# assumes you have previously saved git ssh key called "github-ssh" cr_build_yaml( steps = c( - cr_buildstep_gitsetup("my_keyring", "git_key"), + cr_buildstep_gitsetup("github-ssh"), cr_buildstep_git(c("clone", "git@github.com:github_name/repo_name")) ) diff --git a/man/cr_buildstep_pkgdown.Rd b/man/cr_buildstep_pkgdown.Rd index b808fc50..1a695985 100644 --- a/man/cr_buildstep_pkgdown.Rd +++ b/man/cr_buildstep_pkgdown.Rd @@ -7,10 +7,8 @@ cr_buildstep_pkgdown( github_repo, git_email, - keyring = "my-keyring", - key = "github-key", + secret, env = NULL, - cipher = "id_rsa.enc", build_image = "gcr.io/gcer-public/packagetools:master" ) } @@ -19,14 +17,10 @@ cr_buildstep_pkgdown( \item{git_email}{The email the git commands will be identifying as} -\item{keyring}{The Key Management Store keyring containing the git ssh key} - -\item{key}{The Key Management Store key containing the gitssh key} +\item{secret}{The name of the secret on Google Secret Manager for the git ssh private key} \item{env}{A character vector of env arguments to set for all steps} -\item{cipher}{The filename of the encrypted git ssh key that has been checked into the repository} - \item{build_image}{A docker image with \code{pkgdown} installed} } \description{ @@ -34,21 +28,33 @@ Create buildsteps for deploying an R pkgdown website to GitHub } \details{ Its convenient to set some of the above via \link{Build} macros, such as \code{github_repo=$_GITHUB_REPO} and \code{git_email=$_BUILD_EMAIL} in the Build Trigger web UI + +To commit the website to git, \link{cr_buildstep_gitsetup} is used for which + you will need to add your git ssh private key to Google Secret Manager + +The R package is installed via \link[devtools]{install} before + running \link[pkgdown]{build_site} } \examples{ cr_project_set("my-project") cr_bucket_set("my-bucket") + +# set github repo directly to write it out via cr_build_write() +cr_buildstep_pkgdown("MarkEdmondson1234/googleCloudRunner", + git_email = "cloudbuild@google.com", + secret = "github-ssh") + # github repo set via build trigger macro _GITHUB_REPO cr_buildstep_pkgdown("$_GITHUB_REPO", - "cloudbuild@google.com") + git_email = "cloudbuild@google.com", + secret = "github-ssh") # example including environment arguments for pkgdown build step -steps <- cr_buildstep_pkgdown("$_GITHUB_REPO", - "cloudbuild@google.com", +cr_buildstep_pkgdown("$_GITHUB_REPO", + git_email = "cloudbuild@google.com", + secret = "github-ssh", env = c("MYVAR=$_MY_VAR", "PROJECT=$PROJECT_ID")) -build_yaml <- cr_build_yaml(steps = steps) -my_source <- cr_build_source(RepoSource("my_repo", branch="master")) -build <- cr_build_make(build_yaml, source = my_source) + } \seealso{ Other Cloud Buildsteps: diff --git a/man/cr_deploy_pkgdown.Rd b/man/cr_deploy_pkgdown.Rd index 99f1fbe8..dfabebec 100644 --- a/man/cr_deploy_pkgdown.Rd +++ b/man/cr_deploy_pkgdown.Rd @@ -6,30 +6,24 @@ \usage{ cr_deploy_pkgdown( steps = NULL, + secret, cloudbuild_file = "cloudbuild-pkgdown.yml", git_email = "googlecloudrunner@r.com", - keyring = "my-keyring", - key = "github-key", env = NULL, - cipher = "id_rsa.enc", build_image = "gcr.io/gcer-public/packagetools:master" ) } \arguments{ \item{steps}{extra steps to run before the pkgdown website steps run} +\item{secret}{The name of the secret on Google Secret Manager for the git ssh private key} + \item{cloudbuild_file}{The cloudbuild yaml file to write to} \item{git_email}{The email the git commands will be identifying as} -\item{keyring}{The Key Management Store keyring containing the git ssh key} - -\item{key}{The Key Management Store key containing the gitssh key} - \item{env}{A character vector of env arguments to set for all steps} -\item{cipher}{The filename of the encrypted git ssh key that has been checked into the repository} - \item{build_image}{A docker image with \code{pkgdown} installed} } \description{ diff --git a/tests/testthat/test-build.R b/tests/testthat/test-build.R index ec49a4d1..c16e6e9c 100644 --- a/tests/testthat/test-build.R +++ b/tests/testthat/test-build.R @@ -372,15 +372,15 @@ test_that("Render BuildStep objects", { git_yaml <- cr_build_yaml( steps = c( - cr_buildstep_gitsetup("my_keyring", "git_key"), + cr_buildstep_gitsetup("github-ssh"), cr_buildstep_git(c("clone", "git@github.com:github_name/repo_name")) ) ) expect_equal(git_yaml$steps[[1]]$name, "gcr.io/cloud-builders/gcloud") - expect_equal(git_yaml$steps[[1]]$args[[1]], "kms") - expect_equal(git_yaml$steps[[1]]$args[[4]], "id_rsa.enc") - expect_equal(git_yaml$steps[[1]]$args[[10]], "my_keyring") + expect_equal(git_yaml$steps[[1]]$args[[1]], "-c") + expect_equal(git_yaml$steps[[1]]$args[[2]], + "gcloud secrets versions access latest --secret=github-ssh > /root/.ssh/id_rsa") expect_equal(git_yaml$steps[[1]]$volumes[[1]]$name, "ssh") expect_equal(git_yaml$steps[[1]]$volumes[[1]]$path, "/root/.ssh") diff --git a/vignettes/git.Rmd b/vignettes/git.Rmd new file mode 100644 index 00000000..c23bb6e2 --- /dev/null +++ b/vignettes/git.Rmd @@ -0,0 +1,51 @@ +--- +title: "googleCloudRunner and git (GitHub, GitLab, BitBucket etc.)" +date: "`r Sys.Date()`" +--- + +A lot of the features of Cloud Build rely on connection with git workflows. + +Connecting Cloud Build and git allows you to create workflows that automate tasks upon each git commit. + +A rundown on some common workflows and connections are detailed here. + +## Cloud Build Triggers and GitHub App + +Cloud Builds can be triggered via Build Triggers, and those Build Triggers can be initiated by GitHub, Bitbucket or Cloud Repostiory events such as commits or pull requests. + +A common use case is triggering R package tests upon each commit. + +## Authorization for Cloud Build to commit to git + +This is useful if you want your Cloud Build to modify or commit to a git repo. + +Common use cases are to build a website via `pkgdown` and then commit that website version back to GitHub to auto-build R documentation websites. + +It also allows you to work with other git repos perhaps not connected to Cloud Build via Build Triggers. + +To do so, Cloud Build needs permission to commit to git, so the first step is to create an ssh key secret that it will use to work on your behalf. This is recommended to be done via `cr_buildstep_secret` as you can then reuse the secret across multiple builds, since the ssh key is kept securely within your Google Project. + +The guide is for GitHub, adapt it if you use another git provider: + +1. Create or use an SSH key for your git account. On GitHub use this guide: `https://help.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh` +2. Add the **public** key (filename ending with `.pub`) to GitHub's SSH keys +3. Upload the **private** key (filename not ending with `.pub`) to Google Secret Manager - `https://console.cloud.google.com/security/secret-manager` and call it a name such as `github-ssh` +4. Ensure you have `Secret Manager Secret Accessor` [IAM role](https://console.cloud.google.com/iam-admin/iam) assigned to the cloudbuild service email (`{project-number}@cloudbuild.gserviceaccount.com`) +5. Use in your buildsteps by calling and using the git secret via: + +```r +# assumes you have previously saved git ssh key called "github-ssh" +cr_build_yaml( + steps = c( + cr_buildstep_gitsetup("github-ssh"), + cr_buildstep_git(c("clone", + "git@github.com:github_name/repo_name")) + ) + ) +``` + +## Duplicating git repos to Cloud Source repostiories + +You can also use Google Cloud Platform's git repostory system directly, name Cloud Source Repositories. Even if you don't use them day to day, you can set them up as duplicate of your preferred Git service, which then enables you to use workflows that only work from Cloud Source Repositories. + +This enables you to create Builds that rely on Source objects under git control, an alternative to Cloud Storage buckets.