diff --git a/.Rbuildignore b/.Rbuildignore index 91114bf..03026e5 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,2 +1,8 @@ ^.*\.Rproj$ ^\.Rproj\.user$ +.github +.lintr +tests/end2end +pkgdown +docs +vignettes diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..16a9edf --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,31 @@ +### Link to the Issue + +... + +### Definition of Done + +... + +### How to test changes + +... + + +### Tasks for PR author + +- [ ] Test your change and ensure there is no regression +- [ ] Change has a corresponding issue. ***Ensure it is linked in GitHub*** +- [ ] Author of the change opened a pull request and assigned a reviewer + +### General policy: + +- If applicable - add instructions for testing +- If there’s no issue, create it. Each issue needs to be well defined and described. +- All interaction with a user, user-facing messages, plots, reports etc. are written from the perspective of the person using or receiving it. They are understandable and helpful to this person. If a user sees an error message, there is a call to action, i.e. the user knows what to do to fix it. +- README, other documentation and code comments that we have is updated with all information related to the change. +- All code has been peer-reviewed before merging into any main branch +- All changes have been merged into the main branch we use for development. +- Continuous integration checks (linter, unit tests, integration tests) are configured and pass. +- Optional: unit tests added for all new or changed logic. +- All task requirements satisfied. If not describe it here. The reviewer is responsible to verify each aspect of the task. +- Change covers only things in task. Please create new PR if you want to fix something else. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..ff8cef2 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,64 @@ +on: push + +name: R-CMD-check + +jobs: + main: + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + runs-on: ${{ matrix.config.os }} + + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + config: + - {os: macOS-latest, r: 'release'} + - {os: windows-latest, r: 'release'} + - {os: ubuntu-22.04, r: 'devel'} + - {os: ubuntu-22.04, r: 'release'} + - {os: ubuntu-22.04, r: 'oldrel'} + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install R + uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.r }} + + - name: Install R package dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: local::. # Necessary to avoid object usage linter errors. + + - name: R CMD check + if: always() + uses: r-lib/actions/check-r-package@v2 + with: + error-on: '"note"' + + - name: Lint + if: always() + shell: Rscript {0} + run: | + lints <- lintr::lint_package() + for (lint in lints) print(lint) + quit(status = length(lints) > 0) + + - name: Spell Check + if : always() + shell: Rscript {0} + run: | + spell_check <- spelling::spell_check_package(use_wordlist = TRUE) + if (nrow(spell_check) > 0) { + print(spell_check) + } + quit(status = nrow(spell_check) > 0) + + - name: Test coverage + if: matrix.config.os == 'ubuntu-22.04' && matrix.config.r == 'release' + run: | + Rscript -e 'covr::codecov(token = "${{secrets.CODECOV_TOKEN}}")' diff --git a/.github/workflows/pkgdown.yml b/.github/workflows/pkgdown.yml new file mode 100644 index 0000000..a8edc0a --- /dev/null +++ b/.github/workflows/pkgdown.yml @@ -0,0 +1,34 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: + - main + workflow_dispatch: + +name: pkgdown + +jobs: + pkgdown: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::pkgdown, local::. + needs: website + + - name: Deploy to gh-pages branch + run: | + git config --local user.name "$GITHUB_ACTOR" + git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" + Rscript -e 'pkgdown::deploy_to_branch(new_process = FALSE)' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..c34d443 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,100 @@ +on: push + +name: package-usage-tests + +defaults: + run: + working-directory: ./tests/end2end/app/ + +jobs: + main: + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + runs-on: ${{ matrix.config.os }} + + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + config: + - {os: macOS-latest, r: 'release'} + - {os: windows-latest, r: 'release'} + - {os: ubuntu-22.04, r: 'devel'} + - {os: ubuntu-22.04, r: 'release'} + - {os: ubuntu-22.04, r: 'oldrel'} + + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install R + uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.r }} + + - name: Install R package dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: local::. # Necessary to avoid object usage linter errors. + + - name: Install shiny.benchmark + run: | + Rscript -e "install.packages('remotes')" + Rscript -e "remotes::install_local('../../../', quiet = TRUE)" + + - name: Create app structure + run: | + bash ./../setting_branches.sh + + - name: Check basic functionality - Cypress + run: | + Rscript ../run_tests.R cypress master,develop tests/cypress/ use_this_one_1 FALSE 1 + + - name: Check basic functionality - shinytest2 + run: | + Rscript ../run_tests.R shinytest2 master,develop tests/ use_this_one_1 FALSE 1 + + - name: Check if it fails when renv not present - Cypress + run: | + Rscript ./../run_tests.R cypress renv_missing,renv_shiny1,renv_shiny2 tests/cypress/ use_this_one_1 FALSE 1 + + - name: Check if it fails when renv not present - shinytest2 + run: | + Rscript ./../run_tests.R shinytest2 renv_missing,renv_shiny1,renv_shiny2 tests/ use_this_one_1 FALSE 1 + + - name: Check if it can handle renv - Cypress + run: | + Rscript ./../run_tests.R cypress renv_missing,renv_shiny1,renv_shiny2 tests/cypress/ use_this_one_1 TRUE 1 + + - name: Check if it can handle renv - shinytest2 + run: | + Rscript ./../run_tests.R shinytest2 renv_missing,renv_shiny1,renv_shiny2 tests/ use_this_one_1 TRUE 1 + + - name: Check if it can handle multiple files - Cypress + run: | + Rscript ./../run_tests.R cypress renv_shiny1,renv_shiny2 tests/cypress/,fake_folder/tests/cypress/ use_this_one_1 TRUE 1 + + - name: Check if it can handle multiple files - shinytest2 + run: | + Rscript ./../run_tests.R shinytest2 renv_shiny1,renv_shiny2 tests/,fake_folder/tests/ use_this_one_1 TRUE 1 + + - name: Check if we can replicate tests - Cypress + run: | + Rscript ./../run_tests.R cypress master,develop tests/cypress/ use_this_one_1 FALSE 2 + + - name: Check if we can replicate tests - shinytest2 + run: | + Rscript ./../run_tests.R shinytest2 master,develop tests/ use_this_one_1 FALSE 2 + + - name: Check if we can run tests based on file patterns - Cypress + run: | + Rscript ./../run_tests.R cypress master,develop tests/cypress/ use_this_one_[0-9] FALSE 1 + + - name: Check if we can run tests based on file patterns - shinytest2 + run: | + Rscript ./../run_tests.R shinytest2 master,develop tests/ use_this_one_[0-9] FALSE 1 diff --git a/.gitignore b/.gitignore index fae8299..7b46709 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ vignettes/*.pdf # R Environment Variables .Renviron + +# documentation page +docs +inst/doc diff --git a/.lintr b/.lintr new file mode 100644 index 0000000..51a8fad --- /dev/null +++ b/.lintr @@ -0,0 +1,8 @@ +linters: + linters_with_defaults( + line_length_linter = line_length_linter(100) + ) +exclusions: + c( + "vignettes" + ) diff --git a/DESCRIPTION b/DESCRIPTION index f067e11..4e73149 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,21 +1,41 @@ -Package: shiny.performance -Title: Compare performance of several versions of a shiny app -Version: 0.1.0 +Package: shiny.benchmark +Title: Benchmark the Performance of Shiny Applications +Version: 0.1.1 Authors@R: c( - person(given = "Douglas", family = "Azevedo", email = "douglas@appsilon.com", role = "aut"), - person(given = "Pedro", family = "Silva", email = "pedro@appsilon.com", role = "aut"), - person("Developers", "Appsilon", email = "support+opensource@appsilon.com", role = "cre"), - person(family = "Appsilon Sp. z o.o.", role = "cph") + person(given = "Douglas", family = "Azevedo", email = "opensource+douglas@appsilon.com", role = c("aut", "cre")), + person(family = "Appsilon Sp. z o.o.", role = "cph", email = "opensource@appsilon.com") ) -Description: Compare performance of several versions of a shiny app based on commit hashs -License: LGPL-3 + file LICENSE -URL: https://github.com/Appsilon/shiny.performance -SystemRequirements: yarn 1.22.17 or higher, cypress 9.4.1 or higher, xvfb +Description: Compare performance between different versions of a Shiny application based on Git references. +License: LGPL-3 +URL: https://github.com/Appsilon/shiny.benchmark, https://github.com/Appsilon/shiny.benchmark +BugReports: https://github.com/Appsilon/shiny.benchmark/issues +SystemRequirements: yarn 1.22.17 or higher, Node 12 or higher Encoding: UTF-8 LazyData: true Roxygen: list(markdown = TRUE) -RoxygenNote: 7.1.2 +RoxygenNote: 7.2.3 VignetteBuilder: knitr Depends: R (>= 3.1.0) +Suggests: + covr, + knitr, + lintr, + rcmdcheck, + rmarkdown, + mockr, + spelling +Imports: + dplyr, + ggplot2, + glue, + jsonlite, + methods, + progress, + renv, + shinytest2, + stringr, + testthat, + fs +Language: en-US diff --git a/NAMESPACE b/NAMESPACE index aa9a7a4..a51ca51 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,8 +1,28 @@ # Generated by roxygen2: do not edit by hand -export(performance_tests) -export(run_performance_test) -importFrom(git2r,checkout) +S3method(plot,shiny_benchmark) +S3method(print,shiny_benchmark) +S3method(summary,shiny_benchmark) +export(benchmark) +export(benchmark_cypress) +export(benchmark_shinytest2) +export(load_example) +export(run_cypress_ptest) +export(run_shinytest2_ptest) +export(shiny_benchmark_class) +exportClasses(shiny_benchmark) +import(dplyr) +import(ggplot2) importFrom(glue,glue) importFrom(jsonlite,write_json) +importFrom(methods,new) +importFrom(progress,progress_bar) +importFrom(renv,activate) +importFrom(renv,restore) +importFrom(shinytest2,test_app) +importFrom(stats,median) importFrom(stringr,str_trim) +importFrom(testthat,ListReporter) +importFrom(utils,globalVariables) +importFrom(utils,menu) +importFrom(utils,read.table) diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..d253fef --- /dev/null +++ b/NEWS.md @@ -0,0 +1,3 @@ +# 0.1.1 + +Adding a minimal example using `shinytest2` diff --git a/R/benchmark.R b/R/benchmark.R new file mode 100644 index 0000000..b86fc3c --- /dev/null +++ b/R/benchmark.R @@ -0,0 +1,106 @@ +#' @title Execute performance tests for a list of commits +#' +#' @param commit_list A list of commit hash codes, branches' names or anything +#' else you can use with git checkout [...] +#' @param cypress_dir The directory with tests recorded by Cypress. +#' It can also be a vector of the same size of commit_list +#' @param shinytest2_dir The directory with tests recorded by shinytest2 +#' It can also be a vector of the same size of commit_list +#' @param tests_pattern Cypress/shinytest2 files pattern. E.g. 'performance' +#' It can also be a vector of the same size of commit_list. If it is NULL, +#' all the content in cypress_dir/shinytest2_dir will be used +#' @param app_dir The path to the application root +#' @param port Port to run the app +#' @param use_renv In case it is set as TRUE, package will try to apply +#' renv::restore() in all branches. Otherwise, the current loaded list of +#' packages will be used in all branches. +#' @param renv_prompt Prompt the user before taking any action? +#' @param n_rep Number of replications desired +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @importFrom glue glue +#' @export +benchmark <- function( + commit_list, + cypress_dir = NULL, + shinytest2_dir = NULL, + tests_pattern = NULL, + app_dir = getwd(), + port = 3333, + use_renv = TRUE, + renv_prompt = TRUE, + n_rep = 1, + debug = FALSE +) { + # Get the call parameters + call_benchmark <- match.call() + + # Number of commits to test + n_commits <- length(commit_list) + + # Test whether we have everything we need + if (is.null(cypress_dir) && is.null(shinytest2_dir)) + stop("You must provide a cypress_dir or the shinytest2_dir") + + if (!is.null(cypress_dir) && !is.null(shinytest2_dir)) { + message("Using the cypress file only") + shinytest2_dir <- NULL + } + + type <- ifelse(!is.null(cypress_dir), "cypress", "shinytest2") + obj_name <- ifelse(type == "cypress", "cypress_dir", "shinytest2_dir") + + if (length(get(obj_name)) == 1) + assign(obj_name, rep(get(obj_name), n_commits)) + if (length(get(obj_name)) != n_commits) + stop(glue("You must provide 1 or {n_commits} paths for {obj_name}")) + + if (is.null(tests_pattern)) + tests_pattern <- vector(mode = "list", length = n_commits) + if (length(tests_pattern) == 1) + tests_pattern <- as.list(rep(tests_pattern, n_commits)) + + n_rep <- as.integer(n_rep) + if (n_rep < 1) + stop("You must provide an integer greater than 1 for n_rep") + + # check if the repo is ready for running the checks + check_uncommitted_files() + + # run tests + total_time <- system.time( + if (type == "cypress") { + perf_list <- benchmark_cypress( + commit_list = commit_list, + cypress_dir = cypress_dir, + tests_pattern = tests_pattern, + app_dir = app_dir, + port = port, + use_renv = use_renv, + renv_prompt = renv_prompt, + n_rep = n_rep, + debug = debug + ) + } else { + perf_list <- benchmark_shinytest2( + commit_list, + shinytest2_dir, + tests_pattern = tests_pattern, + app_dir, + use_renv = use_renv, + renv_prompt = renv_prompt, + n_rep = n_rep, + debug = debug + ) + } + ) + + out <- list( + call = call_benchmark, + time = total_time, + performance = perf_list + ) + class(out) <- "shiny_benchmark" + + return(out) +} diff --git a/R/benchmark_cypress.R b/R/benchmark_cypress.R new file mode 100644 index 0000000..cee35c5 --- /dev/null +++ b/R/benchmark_cypress.R @@ -0,0 +1,148 @@ +#' @title Run the performance test based on multiple commits using Cypress +#' +#' @param commit_list A list of commit hash codes, branches' names or anything +#' else you can use with git checkout [...] +#' @param cypress_dir The directory with tests recorded by Cypress. +#' It can also be a vector of the same size of commit_list +#' @param tests_pattern Cypress/shinytest2 files pattern. E.g. 'shinytest2' +#' It can also be a vector of the same size of commit_list. If it is NULL, +#' all the content in cypress_dir/shinytest2_dir will be used +#' @param app_dir The path to the application root +#' @param port Port to run the app +#' @param use_renv In case it is set as TRUE, package will try to apply +#' renv::restore() in all branches. Otherwise, the current loaded list of +#' packages will be used in all branches. +#' @param renv_prompt Prompt the user before taking any action? +#' @param n_rep Number of replications desired +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @export +benchmark_cypress <- function( + commit_list, + cypress_dir, + tests_pattern, + app_dir, + port, + use_renv, + renv_prompt, + n_rep, + debug +) { + # creating the structure + project_path <- create_cypress_structure( + app_dir = app_dir, + port = port, + debug = debug + ) + + # getting the current branch + current_branch <- get_commit_hash() + + # apply the tests for each branch/commit + perf_list <- tryCatch( + expr = { + mapply( + commit_list, + cypress_dir, + tests_pattern, + FUN = run_cypress_ptest, + project_path = project_path, + use_renv = use_renv, + renv_prompt = renv_prompt, + n_rep = n_rep, + debug = debug, + SIMPLIFY = FALSE + ) + }, + error = function(e) { + message(e) + }, + finally = { + # Checkout to the main branch + checkout(branch = current_branch, debug = debug) + message(glue("Switched back to {current_branch}")) + + # Restore renv + if (use_renv) + restore_env(branch = current_branch, renv_prompt = renv_prompt) + + # Cleaning the temporary directory + fs::file_delete(fs::path(project_path, "node")) + fs::file_delete(fs::path(project_path, "tests")) + } + ) + + return(perf_list) +} + +#' @title Run the performance test based on a single commit using Cypress +#' +#' @param commit A commit hash code or a branch's name +#' @param project_path The path to the project with all needed packages +#' installed +#' @param cypress_dir The directory with tests recorded by Cypress +#' @param tests_pattern Cypress files pattern. E.g. 'performance'. If it is NULL, +#' all the content will be used +#' @param use_renv In case it is set as TRUE, package will try to apply +#' renv::restore() in all branches. Otherwise, the current loaded list of +#' packages will be used in all branches. +#' @param renv_prompt Prompt the user before taking any action? +#' @param n_rep Number of replications desired +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @importFrom utils read.table +#' @export +run_cypress_ptest <- function( + commit, + project_path, + cypress_dir, + tests_pattern, + use_renv, + renv_prompt, + n_rep, + debug +) { + # checkout to the desired commit + checkout(branch = commit, debug = debug) + date <- get_commit_date(branch = commit) + message(glue("Switched to {commit}")) + if (use_renv) restore_env(branch = commit, renv_prompt = renv_prompt) + + # get Cypress files + files <- create_cypress_tests( + project_path = project_path, + cypress_dir = cypress_dir, + tests_pattern = tests_pattern + ) + js_file <- files$js_file + txt_file <- files$txt_file + + # replicate tests + perf_file <- list() + pb <- create_progress_bar(total = n_rep) + for (i in 1:n_rep) { + # increment progress bar + pb$tick() + + # run tests there + command <- performance_test_cmd(project_path) + system(command, ignore.stdout = !debug, ignore.stderr = !debug) + + # read the file saved by cypress + perf_file[[i]] <- read.table(file = txt_file, header = FALSE, sep = ";") + perf_file[[i]] <- cbind.data.frame(date = date, rep_id = i, perf_file[[i]]) + colnames(perf_file[[i]]) <- c("date", "rep_id", "test_name", "duration_ms") + + # removing txt measures + fs::file_delete(txt_file) + } + + # removing js tests + fs::file_delete(js_file) + + # removing anything new in the github repo + checkout_files(debug = debug) + + # return times + return(perf_file) +} diff --git a/R/benchmark_shinytest2.R b/R/benchmark_shinytest2.R new file mode 100644 index 0000000..590d673 --- /dev/null +++ b/R/benchmark_shinytest2.R @@ -0,0 +1,149 @@ +#' @title Run the performance test based on a multiple commits using shinytest2 +#' +#' @param commit_list A list of commit hash codes, branches' names or anything +#' else you can use with git checkout [...] +#' @param shinytest2_dir The directory with tests recorded by shinytest2 +#' It can also be a vector of the same size of commit_list +#' @param tests_pattern shinytest2 files pattern. E.g. 'performance' +#' It can also be a vector of the same size of commit_list. If it is NULL, +#' all the content in cypress_dir/shinytest2_dir will be used +#' @param app_dir The path to the application root +#' @param use_renv In case it is set as TRUE, package will try to apply +#' renv::restore() in all branches. Otherwise, the current loaded list of +#' packages will be used in all branches. +#' @param renv_prompt Prompt the user before taking any action? +#' @param n_rep Number of replications desired +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @export +benchmark_shinytest2 <- function( + commit_list, + shinytest2_dir, + tests_pattern, + app_dir, + use_renv, + renv_prompt, + n_rep, + debug +) { + + # creating the structure + project_path <- create_shinytest2_structure(app_dir = app_dir) + + # getting the current branch + current_branch <- get_commit_hash() + + # apply the tests for each branch/commit + perf_list <- tryCatch( + expr = { + mapply( + commit_list, + shinytest2_dir, + tests_pattern, + FUN = run_shinytest2_ptest, + app_dir = app_dir, + project_path = project_path, + use_renv = use_renv, + renv_prompt = renv_prompt, + n_rep = n_rep, + debug = debug, + SIMPLIFY = FALSE + ) + }, + error = function(e) { + message(e) + }, + finally = { + # Checkout to the main branch + checkout(branch = current_branch, debug = debug) + message(glue("Switched back to {current_branch}")) + + # Restore renv + if (use_renv) + restore_env(branch = current_branch, renv_prompt = renv_prompt) + + # Cleaning the temporary directory + # couldn't use fs::file_delete / fs::directory_delete as a process + # is accessing one of the files and it fails. unlink does not + unlink(fs::path(project_path, "tests")) + } + ) + + return(perf_list) +} + +#' @title Run the performance test based on a single commit using shinytest2 +#' +#' @param commit A commit hash code or a branch's name +#' @param app_dir The path to the application root +#' @param project_path The path to the project +#' @param shinytest2_dir The directory with tests recorded by shinytest2 +#' @param tests_pattern shinytest2 files pattern. E.g. 'performance'. If it is NULL, +#' all the content will be used +#' @param use_renv In case it is set as TRUE, package will try to apply +#' renv::restore() in all branches. Otherwise, the current loaded list of +#' packages will be used in all branches. +#' @param renv_prompt Prompt the user before taking any action? +#' @param n_rep Number of replications desired +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @importFrom testthat ListReporter +#' @importFrom shinytest2 test_app +#' @export +run_shinytest2_ptest <- function( + commit, + project_path, + app_dir, + shinytest2_dir, + tests_pattern, + use_renv, + renv_prompt, + n_rep, + debug +) { + # checkout to the desired commit + checkout(branch = commit, debug = debug) + date <- get_commit_date(branch = commit) + message(glue("Switched to {commit}")) + if (use_renv) restore_env(branch = commit, renv_prompt = renv_prompt) + + # move test files to the project folder + tests_dir <- move_shinytest2_tests( + project_path = project_path, + shinytest2_dir = shinytest2_dir + ) + + perf_file <- list() + pb <- create_progress_bar(total = n_rep) + for (i in 1:n_rep) { + # increment progress bar + pb$tick() + + # run tests there + my_reporter <- ListReporter$new() + test_app( + app_dir = dirname(tests_dir), + reporter = my_reporter, + stop_on_failure = FALSE, + stop_on_warning = FALSE, + filter = tests_pattern + ) + + perf_file[[i]] <- as.data.frame(my_reporter$get_results()) + perf_file[[i]] <- perf_file[[i]][, c("test", "real")] + perf_file[[i]]$test <- gsub( + x = perf_file[[i]]$test, + pattern = "\\{shinytest2\\} recording: ", + replacement = "" + ) + + perf_file[[i]] <- cbind.data.frame(date = date, rep_id = i, perf_file[[i]]) + colnames(perf_file[[i]]) <- c("date", "rep_id", "test_name", "duration_ms") + } + + # removing anything new in the github repo + checkout_files(debug = debug) + + # return times + return(perf_file) +} diff --git a/R/globals.R b/R/globals.R new file mode 100644 index 0000000..06155fd --- /dev/null +++ b/R/globals.R @@ -0,0 +1,14 @@ +utils::globalVariables( + c( + "commit", + "date", + "duration_ms", + "max", + "mean", + "min", + "n", + "sd", + "test_name", + "total_time" + ) +) diff --git a/R/performance_tests.R b/R/performance_tests.R deleted file mode 100644 index a4506b4..0000000 --- a/R/performance_tests.R +++ /dev/null @@ -1,91 +0,0 @@ -#' @title Execute performance tests for a list of commits -#' -#' @param commit_list A list of commit hash codes, branches' names or anything else you can use with git checkout [...] -#' @param cypress_file The path to the .js file containing cypress tests to be recorded -#' @param app_dir The path to the application root -#' @param port Port to run the app -#' @param debug Logical. TRUE to display all the system messages on runtime -#' -#' @importFrom git2r checkout -#' -#' @export -performance_tests <- function(commit_list, cypress_file, app_dir = getwd(), port = 3333, debug = FALSE) { - # getting the current branch - current_branch <- get_commit_hash() - - # creating the structure - project_path <- create_tests_structure(app_dir = app_dir, port = port, debug = debug) - - # copy the cypress test file from the current location and store it - cypress_file_cp <- file.path(project_path, "cypress_tests.js") - file.copy(from = cypress_file, to = cypress_file_cp) - - # apply the tests for each branch/commit - perf_list <- tryCatch( - expr = { - lapply( - X = commit_list, - FUN = run_performance_test, - project_path = project_path, - cypress_file = cypress_file_cp, - debug = debug - ) - }, - error = function(e) { - message(e) - }, - finally = { - checkout(branch = current_branch) - message(glue("Switched back to {current_branch}")) - - # Cleaning the temporary directory - unlink( - x = c( - file.path(project_path, "node"), - file.path(project_path, "tests") - ), - recursive = TRUE - ) - } - ) - - return(perf_list) -} - -#' @title Run the performance test based on a single commit -#' -#' @param commit A commit hash code or a branch's name -#' @param project_path The path to the project with all needed packages installed -#' @param cypress_file The path to the .js file conteining cypress tests to be recorded -#' @param txt_file The path to the file where it is aimed to save the times -#' @param debug Logical. TRUE to display all the system messages on runtime -#' -#' @export -run_performance_test <- function(commit, project_path, cypress_file, txt_file, debug) { - files <- create_cypress_tests(project_path = project_path, cypress_file = cypress_file) - js_file <- files$js_file - txt_file <- files$txt_file - - # checkout to the desired commit - checkout(branch = commit) - date <- get_commit_date(branch = commit) - message(glue("Switched to {commit}")) - - # run tests there - command <- glue("cd {project_path}; set -eu; exec yarn --cwd node performance-test") - system(command, ignore.stdout = !debug, ignore.stderr = !debug) - - # read the file saved by cypress - perf_file <- read.table(file = txt_file, header = FALSE, sep = ";") - perf_file <- cbind.data.frame(date = date, perf_file) - colnames(perf_file) <- c("date", "test_name", "duration_ms") - - # removing temp files - unlink(x = c(js_file, txt_file)) - - # removing anything new in the github repo - checkout_files() - - # return times - return(perf_file) -} diff --git a/R/plot.R b/R/plot.R new file mode 100644 index 0000000..53a990d --- /dev/null +++ b/R/plot.R @@ -0,0 +1,36 @@ +#' Plot for shiny_benchmark class +#' +#' @param x shiny_benchmark object +#' @param ... Other parameters +#' +#' @method plot shiny_benchmark +#' @import dplyr +#' @import ggplot2 +#' @importFrom utils globalVariables +#' @export +plot.shiny_benchmark <- function(x, ...) { + if (!requireNamespace(package = "ggplot2", quietly = TRUE)) + stop("ggplot2 is missing. Please, consider intalling ggplot2.") + + plot_df <- lapply(X = x$performance, FUN = bind_rows) %>% + bind_rows(.id = "commit") %>% + arrange(date) %>% + mutate(commit = factor(x = commit, levels = unique(commit))) %>% + group_by(commit, test_name) %>% + summarise( + min = min(duration_ms), + mean = mean(duration_ms), + max = max(duration_ms), + .groups = "keep" + ) %>% + ungroup() + + g <- ggplot(data = plot_df, mapping = aes(x = commit, y = mean)) + + geom_pointrange(mapping = aes(ymin = min, ymax = max)) + + facet_wrap(~test_name) + + ylab("Duration (ms)") + + xlab("Commit") + + theme_bw() + + return(g) +} diff --git a/R/print.R b/R/print.R new file mode 100644 index 0000000..f42d9e8 --- /dev/null +++ b/R/print.R @@ -0,0 +1,21 @@ +#' Print for shiny_benchmark class +#' +#' @param x shiny_benchmark object +#' @param ... Other parameters +#' +#' @method print shiny_benchmark +#' @export +print.shiny_benchmark <- function(x, ...) { + cat("Shiny benchmark: \n") + cat("\n") + cat("Call:") + cat("\n") + print(x$call) + cat("\n") + cat("Total time ellapsed:") + cat("\n") + print(x$time[["elapsed"]]) + cat("\n") + cat("Fit measures: \n") + print(x$performance) +} diff --git a/R/shiny_benchmark-class.R b/R/shiny_benchmark-class.R new file mode 100644 index 0000000..34bf8ee --- /dev/null +++ b/R/shiny_benchmark-class.R @@ -0,0 +1,18 @@ +#' @title An object of 'shiny_benchmark' class +#' +#' @slot call Function call +#' @slot time Time elapsed +#' @slot performance List of measurements (one entry for each commit) +#' +#' @importFrom methods new +#' +#' @export + +shiny_benchmark_class <- setClass( + Class = "shiny_benchmark", + representation( + call = "call", + time = "proc_time", + performance = "list" + ) +) diff --git a/R/summary.R b/R/summary.R new file mode 100644 index 0000000..56b1acb --- /dev/null +++ b/R/summary.R @@ -0,0 +1,16 @@ +#' Summary for shiny_benchmark class +#' +#' @param object shiny_benchmark object +#' @param ... Other parameters +#' +#' @method summary shiny_benchmark +#' @export +summary.shiny_benchmark <- function(object, ...) { + if (!requireNamespace(package = "dplyr", quietly = TRUE)) + stop("dplyr is missing. Please, consider intalling dplyr.") + + summary_results <- lapply(X = object$performance, FUN = summarise_commit) + summary_results <- bind_rows(summary_results, .id = "commit") + + return(summary_results) +} diff --git a/R/utils.R b/R/utils.R index 72d5896..a15774c 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,173 +1,12 @@ -#' @title Create a temporary directory to store everything needed by Cypress -#' -#' @param app_dir The path to the application root -#' @param port Port to run the app -#' @param debug Logical. TRUE to display all the system messages on runtime -#' -#' @importFrom jsonlite write_json -create_tests_structure <- function(app_dir, port, debug) { - # temp dir to run the tests - dir_cypress <- tempdir() - - # node path - node_path <- file.path(dir_cypress, "node") - root_path <- file.path(node_path, "root") - - # test path - tests_path <- file.path(dir_cypress, "tests") - cypress_path <- file.path(tests_path, "cypress") - integration_path <- file.path(cypress_path, "integration") - plugins_path <- file.path(cypress_path, "plugins") - - # creating paths - dir.create(path = node_path, showWarnings = FALSE) - dir.create(path = tests_path, showWarnings = FALSE) - dir.create(path = cypress_path, showWarnings = FALSE) - dir.create(path = integration_path, showWarnings = FALSE) - dir.create(path = plugins_path, showWarnings = FALSE) - - # create a path root linked to the main directory app - symlink_cmd <- glue("cd {dir_cypress}; ln -s {app_dir} {root_path}") - system(symlink_cmd) - - # create the packages.json file - json_txt <- create_node_list(tests_path = tests_path, port = port) - json_file <- file.path(node_path, "package.json") - write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE) - - # install everything that is needed - install_deps <- glue("yarn --cwd {node_path}") - system(install_deps, ignore.stdout = !debug, ignore.stderr = !debug) - - # creating cypress plugin file - js_txt <- create_cypress_plugins() - js_file <- file.path(plugins_path, "index.js") - writeLines(text = js_txt, con = js_file) - - # creating cypress.json - json_txt <- create_cypress_list(plugins_file = js_file, port = port) - json_file <- file.path(tests_path, "cypress.json") - write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE) - - # returning the project folder - message(glue("Structure created at {dir_cypress}")) - - return(dir_cypress) -} - -#' @title Create the list of needed libraries -#' -#' @param tests_path The path to project -create_node_list <- function(tests_path, port) { - json_list <- list( - private = TRUE, - scripts = list( - "performance-test" = glue("start-server-and-test run-app http://localhost:{port} run-cypress"), - "run-app" = glue("cd root && Rscript -e 'shiny::runApp(port = {port})'"), - "run-cypress" = glue("cypress run --project {tests_path}") - ), - "devDependencies" = list( - "cypress" = "^7.6.0", - "start-server-and-test" = "^1.12.6" - ) - ) - - return(json_list) -} - -#' @title Create the cypress configuration list -#' -#' @param plugins_file The path to the Cypress plugins -create_cypress_list <- function(plugins_file, port) { - json_list <- list( - baseUrl = glue("http://localhost:{port}"), - pluginsFile = plugins_file, - supportFile = FALSE - ) - - return(json_list) -} - -#' @title Create the JS code to track execution time -create_cypress_plugins <- function() { - js_txt <- " - const fs = require('fs') - module.exports = (on, config) => { - on('task', { - performanceTimes (attributes) { - fs.writeFile(attributes.fileOut, `${ attributes.title }; ${ attributes.duration }\n`, { flag: 'a' }) - return null - } - }) - }" - - return(js_txt) -} - -#' @title Create the cypress files under project directory -#' -#' @param project_path The path to the project with all needed packages installed -#' @param cypress_file The path to the .js file conteining cypress tests to be recorded -create_cypress_tests <- function(project_path, cypress_file) { - # creating a copy to be able to edit the js file - js_file <- file.path(project_path, "tests", "cypress", "integration", "app.spec.js") - file.copy(from = cypress_file, to = js_file, overwrite = TRUE) - - # file to store the times - txt_file <- file.path(project_path, "tests", "cypress", "performance.txt") - add_sendtime2js(js_file = js_file, txt_file = txt_file) - - # returning the file location - return(list(js_file = js_file, txt_file = txt_file)) -} - -#' @title Add the sendTime function to the .js file -#' -#' @param js_file Path to the .js file to add code -#' @param txt_file Path to the file to record the execution times -add_sendtime2js <- function(js_file, txt_file) { - lines_to_add <- glue( - " - // Returning the time for each test - // https://www.cypress.io/blog/2020/05/22/where-does-the-test-spend-its-time/ - let commands = [] - let performanceAttrs - Cypress.on('test:before:run', () => { - commands.length = 0 - }) - Cypress.on('test:after:run', (attributes) => { - performanceAttrs = { - title: attributes.title, - duration: attributes.duration, - commands: Cypress._.cloneDeep(commands), - } - }) - const sendTestTimings = () => { - if (!performanceAttrs) { - return - } - const attr = performanceAttrs - attr.fileOut = '{{txt_file}}' - performanceAttrs = null - cy.task('performanceTimes', attr) - } - // Calling the sendTestTimings function - beforeEach(sendTestTimings) - after(sendTestTimings) - ", - .open = "{{", .close = "}}" - ) - - write(x = lines_to_add, file = js_file, append = TRUE) -} - #' @title Get the commit date in POSIXct format #' #' @param branch Commit hash code or branch name #' @importFrom glue glue +#' +#' @keywords internal get_commit_date <- function(branch) { date <- system( - glue("git show -s --format=%ci {branch}"), + command = glue("git show -s --format=%ci {branch}"), intern = TRUE ) date <- as.POSIXct(date[1]) @@ -176,22 +15,30 @@ get_commit_date <- function(branch) { } #' @title Find the hash code of the current commit +#' #' @importFrom glue glue #' @importFrom stringr str_trim +#' +#' @keywords internal get_commit_hash <- function() { - hash <- system("git show -s --format=%H", intern = TRUE)[1] + hash <- system(command = "git show -s --format=%H", intern = TRUE)[1] + branch <- system( - glue("git branch --contains {hash}"), + command = glue("git branch --contains {hash}"), intern = TRUE ) branch <- str_trim( - string = gsub(x = branch[length(branch)], pattern = "\\*\\s", replacement = ""), + string = gsub( + x = branch[length(branch)], + pattern = "\\*\\s", + replacement = "" + ), side = "both" ) hash_head <- system( - glue("git rev-parse {branch}"), + command = glue("git rev-parse {branch}"), intern = TRUE ) @@ -204,8 +51,181 @@ get_commit_hash <- function() { #' @title Checkout GitHub files #' -#' @description checkout anything created by the app. It prevents errors when +#' @description Checkout anything created by the app. It prevents errors when #' changing branches -checkout_files <- function() { - system("git checkout .") +#' +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @keywords internal +checkout_files <- function(debug) { + system( + command = "git checkout .", + ignore.stdout = !debug, + ignore.stderr = !debug + ) +} + +#' @title Checkout GitHub branch +#' +#' @description checkout and go to a different branch +#' +#' @param branch Commit hash code or branch name +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @keywords internal +checkout <- function(branch, debug) { + system( + command = glue("git checkout {branch}"), + ignore.stdout = !debug, + ignore.stderr = !debug + ) +} + +#' @title Running the node script "performance_test" is system-dependent +#' +#' @param project_path path to project directory (one level above node) +#' +#' @keywords internal +performance_test_cmd <- function(project_path) { + glue("yarn --cwd \"{fs::path(project_path, 'node')}\" performance-test") +} + +#' @title Check for uncommitted files +#' +#' @keywords internal +check_uncommitted_files <- function() { + changes <- system("git status --porcelain", intern = TRUE) + + if (length(changes) != 0) { + system("git status -u") + stop("You have uncommitted files. Please resolve it before running the performance checks.") + } else { + return(invisible(TRUE)) + } +} + +#' @title Check and restore renv +#' +#' @description Check whether renv is in use in the current branch. Raise error +#' if renv is not in use or apply renv:restore() in the case the package is +#' present +#' +#' @param branch Commit hash code or branch name. Useful to create an +#' informative error message +#' @param renv_prompt Prompt the user before taking any action? +#' @importFrom glue glue +#' @importFrom renv activate restore +#' +#' @keywords internal +restore_env <- function(branch, renv_prompt) { + # handling renv + tryCatch( + expr = { + activate() + restore(prompt = renv_prompt) + }, + error = function(e) { + stop(glue("Unexpected error activating renv in branch {branch}: {e}\n")) + } + ) +} + +#' @title Create a progress bar to follow the execution +#' +#' @param total Total number of replications +#' @importFrom progress progress_bar +#' +#' @keywords internal +create_progress_bar <- function(total = 100) { + pb <- progress_bar$new( + format = "Iteration :current/:total", + total = total, + clear = FALSE + ) + + return(pb) +} + +#' @title Return statistics based on the set of tests replications +#' +#' @param object A shiny_benchmark object +#' +#' @import dplyr +#' @importFrom stats median +#' +#' @keywords internal +summarise_commit <- function(object) { + out <- bind_rows(object) %>% + group_by(test_name) %>% + summarise( + n = n(), + mean = mean(duration_ms), + median = median(duration_ms), + sd = sd(duration_ms), + min = min(duration_ms), + max = max(duration_ms) + ) + + return(out) +} + +#' @title Load an application and instructions to run shiny.benchmark +#' @description This function aims to generate a template to be used +#' by shiny.benchmark. It will create the necessary structure on `path` with +#' some examples of tests using Cypress and shinytest2. Also, a simple +#' application will be added to the folder as well as instructions on how +#' to perform the performance checks. Be aware that a new git repo is need in +#' the selected `path`. +#' +#' @param path A character vector of full path name +#' @param force Create example even if directory does not exist or is not empty +#' +#' @importFrom glue glue +#' @importFrom utils menu +#' @export +#' @examples +#' load_example(file.path(tempdir(), "example_destination"), force = TRUE) +load_example <- function(path, force = FALSE) { + # see if path exists + if (!force && !fs::file_exists(path)) + stop("You must provide a valid path") + else if (!fs::file_exists(path)) { + fs::dir_create(path, recurse = TRUE) + } + + if (!force && length(fs::dir_ls(path))) { + choice <- menu( + choices = c("Yes", "No"), + title = glue("{path} seems to not be empty. Would you like to proceed?") + ) + + if (choice == 2) + stop("Process aborted by user. Consider creating a new empty path.") + } else if (length(fs::dir_ls(path))) { + message(glue( + "{path} seems to not be empty. ", + "Continuing as parameter `force = TRUE`" + )) + } + + ex_path <- system.file( + "examples", + package = "shiny.benchmark", + mustWork = TRUE + ) + files <- fs::dir_ls(path = ex_path, fun = fs::path_real) + + for (file in files) { + if (fs::is_dir(file)) { + # Due to overwrite = TRUE the destination must include the name of the + # directory to be created + fs::dir_copy(file, fs::path(path, fs::path_file(file)), overwrite = TRUE) + } else { + fs::file_copy(file, path, overwrite = TRUE) + } + print(glue("{basename(file)} created at {path}")) + } + + fpath <- fs::path(path, "run_tests.R") # nolint + message(glue("Follow instructions in {fpath}")) } diff --git a/R/utils_cypress.R b/R/utils_cypress.R new file mode 100644 index 0000000..7533d05 --- /dev/null +++ b/R/utils_cypress.R @@ -0,0 +1,235 @@ +#' @title Create a temporary directory to store everything needed by Cypress +#' +#' @param app_dir The path to the application root +#' @param port Port to run the app +#' @param debug Logical. TRUE to display all the system messages on runtime +#' +#' @importFrom jsonlite write_json +#' +#' @keywords internal +create_cypress_structure <- function(app_dir, port, debug) { + # temp dir to run the tests + dir_tests <- tempdir() + + # node path + node_path <- fs::path(dir_tests, "node") + root_path <- fs::path(node_path, "root") # nolint + + # test path + tests_path <- fs::path(dir_tests, "tests") + cypress_path <- fs::path(tests_path, "cypress") + integration_path <- fs::path(cypress_path, "integration") + plugins_path <- fs::path(cypress_path, "plugins") + + # creating paths + fs::dir_create(path = node_path) + fs::dir_create(path = tests_path) + fs::dir_create(path = cypress_path) + fs::dir_create(path = integration_path) + fs::dir_create(path = plugins_path) + + # create a path root linked to the main directory app + tryCatch( + expr = { + fs::link_create(app_dir, root_path, symbolic = TRUE) + }, + error = function(e) { + + choice <- menu( + choices = c("Yes", "No"), + title = glue( + "A symbolic link cannot be created, it is possible to clone ", + "the repository, but it can take some time and space on disk. ", + "Would you like to proceed with this operations?") + ) + + if (choice == 2) + stop("Process aborted by user.") + + # If system cannot symlink then try to clone the repository + # This may happen on some windows versions + # This can be an expensive operation on big repositories + message( + "Could not create symbolic link with fs package, ", + "trying with git clone..." + ) + system(glue::glue("git clone \"{app_dir}\" \"{root_path}\"")) + system("git submodule init") + system("git submodule update ") + }) + + # create the packages.json file + json_txt <- create_node_list(tests_path = tests_path, port = port) + json_file <- fs::path(node_path, "package.json") + write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE) + + # install everything that is needed + install_deps <- glue("yarn --cwd {node_path}") + system(install_deps, ignore.stdout = !debug, ignore.stderr = !debug) + + # creating cypress plugin file + js_txt <- create_cypress_plugins() + js_file <- fs::path(plugins_path, "index.js") + writeLines(text = js_txt, con = js_file) + + # creating cypress.json + json_txt <- create_cypress_list(plugins_file = js_file, port = port) + json_file <- fs::path(tests_path, "cypress.json") + write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE) + + # returning the project folder + message(glue("Structure created at {dir_tests}")) + return(dir_tests) +} + +#' @title Create the list of needed libraries +#' +#' @param tests_path The path to project +#' @param port Port to run the app +#' +#' @keywords internal +create_node_list <- function(tests_path, port) { + json_list <- list( + private = TRUE, + scripts = list( + "performance-test" = glue( + "start-server-and-test run-app http://localhost:{port} run-cypress" + ), + "run-app" = glue( + "cd root && ", + "Rscript -e \"shiny::runApp(port = {port})\"" + ), + "run-cypress" = glue("cypress run --project {tests_path}") + ), + "devDependencies" = list( + "cypress" = "^7.6.0", + "start-server-and-test" = "^1.12.6" + ) + ) + + return(json_list) +} + +#' @title Create the cypress configuration list +#' +#' @param plugins_file The path to the Cypress plugins +#' @param port Port to run the app +#' +#' @keywords internal +create_cypress_list <- function(plugins_file, port) { + json_list <- list( + baseUrl = glue("http://localhost:{port}"), + pluginsFile = plugins_file, + supportFile = FALSE + ) + + return(json_list) +} + +#' @title Create the JS code to track execution time +#' +#' @keywords internal +create_cypress_plugins <- function() { + js_txt <- " + const fs = require('fs') + module.exports = (on, config) => { + on('task', { + performanceTimes (attributes) { + fs.writeFile( + attributes.fileOut, + `${ attributes.title }; ${ attributes.duration }\n`, + { flag: 'a' } + ) + return null + } + }) + }" + + return(js_txt) +} + +#' @title Create the cypress files under project directory +#' +#' @param project_path The path to the project with all needed packages +#' installed +#' @param cypress_dir The directory with tests recorded by Cypress +#' @param tests_pattern Cypress files pattern. E.g. 'performance'. If it is NULL, +#' all the content will be used +#' +#' @keywords internal +create_cypress_tests <- function(project_path, cypress_dir, tests_pattern) { + # locate files + cypress_files <- list.files( + path = cypress_dir, + pattern = tests_pattern, + full.names = TRUE, + recursive = TRUE + ) + cypress_files <- grep(x = cypress_files, pattern = "\\.js$", value = TRUE) + + # creating a copy to be able to edit the js file + js_file <- fs::path( + project_path, + "tests", + "cypress", + "integration", + "app.spec.js" + ) + + # combine all files into one + for (i in seq_along(cypress_files)) { + text <- readLines(con = cypress_files[i]) + write(x = text, file = js_file, append = TRUE) + } + + # file to store the times + txt_file <- fs::path(project_path, "tests", "cypress", "performance.txt") + + add_sendtime2js(js_file = js_file, txt_file = txt_file) + + # returning the file location + return(list(js_file = js_file, txt_file = txt_file)) +} + +#' @title Add the sendTime function to the .js file +#' +#' @param js_file Path to the .js file to add code +#' @param txt_file Path to the file to record the execution times +#' +#' @keywords internal +add_sendtime2js <- function(js_file, txt_file) { + lines_to_add <- glue( + " + describe('Finalizing tests', () => {it('Ending tests', () => {})}) + + // Returning the time for each test + // https://www.cypress.io/blog/2020/05/22/where-does-the-test-spend-its-time/ + let commands = [] + let performanceAttrs + Cypress.on('test:before:run', () => { + commands.length = 0 + }) + Cypress.on('test:after:run', (attributes) => { + performanceAttrs = { + title: attributes.title, + duration: attributes.duration, + commands: Cypress._.cloneDeep(commands), + } + }) + const sendTestTimings = () => { + if (!performanceAttrs) { + return + } + const attr = performanceAttrs + attr.fileOut = '{{txt_file}}' + performanceAttrs = null + cy.task('performanceTimes', attr) + } + // Calling the sendTestTimings function + beforeEach(sendTestTimings) + ", + .open = "{{", .close = "}}" + ) + + write(x = lines_to_add, file = js_file, append = TRUE) +} diff --git a/R/utils_shinytest2.R b/R/utils_shinytest2.R new file mode 100644 index 0000000..eaeadd8 --- /dev/null +++ b/R/utils_shinytest2.R @@ -0,0 +1,36 @@ +#' @title Create a temporary directory to store everything needed by shinytest2 +#' +#' @param app_dir The path to the application root +#' +#' @importFrom glue glue +#' +#' @keywords internal +create_shinytest2_structure <- function(app_dir) { + # temp dir to run the tests + dir_tests <- tempdir() + + # shiny call + writeLines( + text = glue('shiny::runApp(appDir = "{app_dir}")'), + con = fs::path(dir_tests, "app.R") + ) + + # returning the project folder + message(glue("Structure created at {dir_tests}")) + + return(dir_tests) +} + +#' @title Move tests to a temporary folder +#' +#' @param project_path The path to the project +#' @param shinytest2_dir The directory with tests recorded by shinytest2 +#' +#' @keywords internal +move_shinytest2_tests <- function(project_path, shinytest2_dir) { + # copy everything to the temporary directory + file.copy(from = shinytest2_dir, to = project_path, recursive = TRUE) + tests_dir <- file.path(project_path, "tests") + + return(tests_dir) +} diff --git a/README.md b/README.md index 4009b6c..c976450 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,175 @@ -# shiny.performance -Tools to measure performance improvements in shiny apps +# shiny.benchmark shiny.benchmark logo + +> _Tools to measure performance improvements in Shiny apps._ + + +[![CRAN status](https://www.r-pkg.org/badges/version/shiny.benchmark)](https://cran.r-project.org/package=shiny.benchmark) +[![R-CMD-check](https://github.com/Appsilon/shiny.benchmark/workflows/R-CMD-check/badge.svg)](https://github.com/Appsilon/shiny.benchmark/actions?workflow=R-CMD-check) + + +`shiny.benchmark` is a tool aimed to measure and compare the performance of different versions of a `shiny` application. Based on a list of different application versions, accessible by a git repo by its refs (commit hash or branch name), the user can write instructions to be executed using Cypress or `shinytest2`. These instructions are then evaluated by the different versions of your `shiny` application and therefore the performance's improvement/deterioration (time elapsed) are be recorded. + +The package is flexible enough to allow different sets of tests for the different refs as well as different package versions (via `renv`). Also, the user can replicate the tests to have more accurate measures of performance. + +## How to install? + +```r +remotes::install_github("Appsilon/shiny.benchmark") +``` + +## Dependencies + +`shiny.benchmark` can use two different engines to test the change in the performance of your application: [shinytest2](https://rstudio.github.io/shinytest2/) and [Cypress](https://www.cypress.io/). +The latter requires `Node` (version 12 or higher) and `yarn` (version 1.22.17 or higher) to be available. +To install them on your computer, follow the guidelines on the documentation pages: + +- [Node](https://nodejs.org/en/download/) +- [yarn](https://yarnpkg.com/getting-started/install) + +Besides that, on Linux, it might be required to install other `Cypress` dependencies. +Check the [documentation](https://docs.cypress.io/guides/getting-started/installing-cypress#Linux-Prerequisites) to find out more. + +## How to use it? + +The best way to start using `shiny.benchmark` is through an example. If you want a start point, you can use the `load_example` function. In order to use this, create a new folder in your computer and use the following code to generate an application to serve us as example for our performance checks: + +```r +library(shiny.benchmark) + +load_example(path = "path/to/new/project") +``` + +It will create some useful files under `path/to/new/project`. The most important one is the `run_tests.R` which provides several instructions at the very top. + +As we are comparing versions of the same application, we need different app versions in different branches/commits in `git`. Start using `cd app; git init` to initiate git inside `app/` folder. + +Get familiar with `app/server.R` file in order to generate more interesting scenarios. The basic idea is to use the `Sys.sleep` function to simulate some app's functionalities. Remember that, when running the benchmark, that is the amount of time it will take to measure the performance. + +When you are ready, commit your changes in master/main using `git add .; git commit -m "your commit message"`. Make some editions and commit these new changes into a new branch or in the same branch your are testing (it will have a different commit hash). Repeat the process adding as many new modifications as you want. E.g. add renv, add more tests, change the names of the tests/test files and so on. + +Here is a complete example on how to setup your `git`: + +```git +# starting +git init +echo .Rproj.user >> .gitignore +echo *.Rproj >> .gitignore +echo .Rprofile >> .gitignore +echo renv >> .gitignore +echo .Rprofile >> .gitignore + +# master +git add . +git commit -m "first commit" + +# develop (decrease Sys.sleep times in server.R) +git checkout -b develop +git add . +git commit -m "improving performance" + +## Using renv +git branch renv_shiny1 develop +git checkout renv_shiny1 +R -e 'renv::init()' +git add . +git commit -m "renv active" + +## Downgrading shiny +git checkout -b renv_shiny2 +R -e 'renv::install("shiny@1.7.0")' +R -e 'renv::snapshot()' +git add . +git commit -m "downgrading shiny" + +## Switching back to develop +git checkout develop +``` + +Now you are ready to go. The `benchmark` function provides several arguments to make your life easier when running your performance checks. The mandatory arguments are: + +- `commit_list`: A vector with commits, branches or anything else you can use in `git checkout` +- `cypress_dir` or `shinytest2_dir`: Folder containing the tests we want to check the performance. In our case it is `tests/cypress` and `tests` respectively. + +The default behavior is to try to use `renv` in your project. If you do not have the renv structure, you can turn `renv` off using `use_renv = FALSE` + +```r +library(shiny.benchmark) + +# commits to compare +commit_list <- c("develop", "renv_shiny1", "renv_shiny2") + +# run performance check using Cypress +benchmark( + commit_list = commit_list, + cypress_dir = "tests/cypress" +) +``` + +That is all you need to run your `Cypress` tests. If you don't use `Cypress`, you may want to use `shinytest2` instead: + +```r +benchmark( + commit_list = commit_list, + shinytest2_dir = "tests" +) +``` + +To run just specific tests, you can take advantage of the `tests_pattern` argument. It will filter the test file's names based on regular expression: + +```r +benchmark( + commit_list = commit_list, + shinytest2_dir = "tests", + tests_pattern = "use_this_one_[0-9]" +) +``` + +If your project has `renv` structure, you can set `use_renv` to `TRUE` to guarantee that, for each application version your are using the correct packages. If you want to approve/reprove `renv::restore()`, you can set `renv_prompt = TRUE`. + +```r +benchmark( + commit_list = commit_list, + shinytest2_dir = "tests", + tests_pattern = "use_this_one_[0-9]", + use_renv = TRUE, # default + renv_prompt = TRUE +) +``` + +To have more accurate information about the time your application takes to perform some actions, you may need to replicate the tests. In this case, you can use the `n_rep` argument: + +```r +out <- benchmark( + commit_list = commit_list, + cypress_dir = "tests/cypress", + tests_pattern = "use_this_one_[0-9]", + use_renv = FALSE, + n_rep = 15 +) + +out +``` + +For fast information about the tests results, you can use the `summary` and also the `plot` methods: + +```r +summary(out) +plot(out) +``` + +## How to contribute? + +If you want to contribute to this project please submit a regular PR, once you're done with new feature or bug fix. + +Reporting a bug is also helpful - please use [GitHub issues](https://github.com/Appsilon/shiny.benchmark/issues) and describe your problem as detailed as possible. + +## Appsilon + + + +Appsilon is a **Posit (formerly RStudio) Full Service Certified Partner**. Learn more +at [appsilon.com](https://appsilon.com). + +Get in touch [opensource@appsilon.com](mailto:opensource@appsilon.com) + +We are hiring! diff --git a/inst/WORDLIST b/inst/WORDLIST new file mode 100644 index 0000000..6cab662 --- /dev/null +++ b/inst/WORDLIST @@ -0,0 +1,13 @@ +codecov +CMD +JS +POSIXct +Posit +RStudio +appsilon +dir +js +renv +repo +sendTime +shinytest diff --git a/inst/examples/app/global.R b/inst/examples/app/global.R new file mode 100644 index 0000000..5a0cab8 --- /dev/null +++ b/inst/examples/app/global.R @@ -0,0 +1 @@ +library(shiny) diff --git a/inst/examples/app/server.R b/inst/examples/app/server.R new file mode 100644 index 0000000..b843f12 --- /dev/null +++ b/inst/examples/app/server.R @@ -0,0 +1,39 @@ +function(input, output, session) { + # Sys.sleep + react1 <- eventReactive(input$run1, { + out <- system.time( + Sys.sleep(1 + rexp(n = 1, rate = 10)) + ) + + return(out[3]) + }) + + react2 <- eventReactive(input$run2, { + out <- system.time( + Sys.sleep(0.5 + rexp(n = 1, rate = 10)) + ) + + return(out[3]) + }) + + react3 <- eventReactive(input$run3, { + out <- system.time( + Sys.sleep(0.1 + rexp(n = 1, rate = 10)) + ) + + return(out[1]) + }) + + # outputs + output$out1 <- renderUI({ + tags$span(round(react1()), style = "font-size: 500px;") + }) + + output$out2 <- renderUI({ + tags$span(round(react2()), style = "font-size: 500px;") + }) + + output$out3 <- renderUI({ + tags$span(round(react3()), style = "font-size: 500px;") + }) +} diff --git a/inst/examples/app/tests/cypress/cypress_use_this_one_1.js b/inst/examples/app/tests/cypress/cypress_use_this_one_1.js new file mode 100644 index 0000000..015b865 --- /dev/null +++ b/inst/examples/app/tests/cypress/cypress_use_this_one_1.js @@ -0,0 +1,19 @@ +describe('Cypress test', () => { + it('Out1 time elapsed - 1', () => { + cy.visit('/'); + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed - 1', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed - 1', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); +}); diff --git a/inst/examples/app/tests/cypress/cypress_use_this_one_2.js b/inst/examples/app/tests/cypress/cypress_use_this_one_2.js new file mode 100644 index 0000000..c01f42e --- /dev/null +++ b/inst/examples/app/tests/cypress/cypress_use_this_one_2.js @@ -0,0 +1,19 @@ +describe('Cypress test', () => { + it('Out1 time elapsed - 2', () => { + cy.visit('/'); + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed - 2', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed - 2', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); +}); diff --git a/inst/examples/app/tests/testthat.R b/inst/examples/app/tests/testthat.R new file mode 100644 index 0000000..7d25b5b --- /dev/null +++ b/inst/examples/app/tests/testthat.R @@ -0,0 +1 @@ +shinytest2::test_app() diff --git a/inst/examples/app/tests/testthat/setup.R b/inst/examples/app/tests/testthat/setup.R new file mode 100644 index 0000000..be65b4f --- /dev/null +++ b/inst/examples/app/tests/testthat/setup.R @@ -0,0 +1,2 @@ +# Load application support files into testing environment +shinytest2::load_app_env() diff --git a/inst/examples/app/tests/testthat/test-use_this_one_1.R b/inst/examples/app/tests/testthat/test-use_this_one_1.R new file mode 100644 index 0000000..a530c0a --- /dev/null +++ b/inst/examples/app/tests/testthat/test-use_this_one_1.R @@ -0,0 +1,19 @@ +library(shinytest2) + +test_that("{shinytest2} recording: test1", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + +test_that("{shinytest2} recording: test2", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + +test_that("{shinytest2} recording: test3", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) diff --git a/inst/examples/app/tests/testthat/test-use_this_one_2.R b/inst/examples/app/tests/testthat/test-use_this_one_2.R new file mode 100644 index 0000000..f2fc5c6 --- /dev/null +++ b/inst/examples/app/tests/testthat/test-use_this_one_2.R @@ -0,0 +1,19 @@ +library(shinytest2) + +test_that("{shinytest2} recording: test4", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + +test_that("{shinytest2} recording: test5", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + +test_that("{shinytest2} recording: test6", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) diff --git a/inst/examples/app/ui.R b/inst/examples/app/ui.R new file mode 100644 index 0000000..8d3615a --- /dev/null +++ b/inst/examples/app/ui.R @@ -0,0 +1,20 @@ +function() { + bootstrapPage( + tags$h1("Measuring time in different commits"), + column( + width = 4, + actionButton(inputId = "run1", label = "Run 1"), + uiOutput(outputId = "out1") + ), + column( + width = 4, + actionButton(inputId = "run2", label = "Run 2"), + uiOutput(outputId = "out2") + ), + column( + width = 4, + actionButton(inputId = "run3", label = "Run 3"), + uiOutput(outputId = "out3") + ) + ) +} diff --git a/inst/examples/run_tests.R b/inst/examples/run_tests.R new file mode 100644 index 0000000..568da2e --- /dev/null +++ b/inst/examples/run_tests.R @@ -0,0 +1,65 @@ +############################################################################### +# Start a git repo under app/ folder and create some branches. It can be more # +# fun if you change the Sys.sleep time in app/server.R # +# # +# suggestion: # +# git init # +# # +# main # +# git add . # +# git commit -m "first commit" # +# # +# # develop # +# git checkout -b develop # +# git commit --allow-empty -m "dummy commit to change hash" # +# # +# # feature # +# git checkout -b feature # +# git commit --allow-empty -m "dummy commit to change hash" # +# # +# For a more complete example see: # +# https://github.com/Appsilon/shiny.benchmark # +############################################################################### + +# packages +library(shiny.benchmark) + +# commits to compare +type <- "cypress" +commit_list <- c("develop", "feature") +dir <- "tests/cypress" +pattern <- "use_this_one_[0-9]" +use_renv <- FALSE +n_rep <- 5 + +if (type == "cypress") { + # run performance check using Cypress + out <- benchmark( + commit_list = commit_list, + cypress_dir = dir, + tests_pattern = pattern, + app_dir = getwd(), + use_renv = use_renv, + renv_prompt = TRUE, + port = 3333, + n_rep = n_rep, + debug = FALSE + ) +} else { + # run performance check using shinytest2 + out <- benchmark( + commit_list = commit_list, + shinytest2_dir = dir, + tests_pattern = pattern, + app_dir = getwd(), + use_renv = use_renv, + renv_prompt = TRUE, + port = 3333, + n_rep = n_rep, + debug = FALSE + ) +} + +out +summary(out) +plot(out) diff --git a/man/add_sendtime2js.Rd b/man/add_sendtime2js.Rd index c86b595..f5ce1f8 100644 --- a/man/add_sendtime2js.Rd +++ b/man/add_sendtime2js.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R +% Please edit documentation in R/utils_cypress.R \name{add_sendtime2js} \alias{add_sendtime2js} \title{Add the sendTime function to the .js file} @@ -14,3 +14,4 @@ add_sendtime2js(js_file, txt_file) \description{ Add the sendTime function to the .js file } +\keyword{internal} diff --git a/man/benchmark.Rd b/man/benchmark.Rd new file mode 100644 index 0000000..b21b421 --- /dev/null +++ b/man/benchmark.Rd @@ -0,0 +1,50 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/benchmark.R +\name{benchmark} +\alias{benchmark} +\title{Execute performance tests for a list of commits} +\usage{ +benchmark( + commit_list, + cypress_dir = NULL, + shinytest2_dir = NULL, + tests_pattern = NULL, + app_dir = getwd(), + port = 3333, + use_renv = TRUE, + renv_prompt = TRUE, + n_rep = 1, + debug = FALSE +) +} +\arguments{ +\item{commit_list}{A list of commit hash codes, branches' names or anything +else you can use with git checkout \link{...}} + +\item{cypress_dir}{The directory with tests recorded by Cypress. +It can also be a vector of the same size of commit_list} + +\item{shinytest2_dir}{The directory with tests recorded by shinytest2 +It can also be a vector of the same size of commit_list} + +\item{tests_pattern}{Cypress/shinytest2 files pattern. E.g. 'performance' +It can also be a vector of the same size of commit_list. If it is NULL, +all the content in cypress_dir/shinytest2_dir will be used} + +\item{app_dir}{The path to the application root} + +\item{port}{Port to run the app} + +\item{use_renv}{In case it is set as TRUE, package will try to apply +renv::restore() in all branches. Otherwise, the current loaded list of +packages will be used in all branches.} + +\item{renv_prompt}{Prompt the user before taking any action?} + +\item{n_rep}{Number of replications desired} + +\item{debug}{Logical. TRUE to display all the system messages on runtime} +} +\description{ +Execute performance tests for a list of commits +} diff --git a/man/benchmark_cypress.Rd b/man/benchmark_cypress.Rd new file mode 100644 index 0000000..123e346 --- /dev/null +++ b/man/benchmark_cypress.Rd @@ -0,0 +1,46 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/benchmark_cypress.R +\name{benchmark_cypress} +\alias{benchmark_cypress} +\title{Run the performance test based on multiple commits using Cypress} +\usage{ +benchmark_cypress( + commit_list, + cypress_dir, + tests_pattern, + app_dir, + port, + use_renv, + renv_prompt, + n_rep, + debug +) +} +\arguments{ +\item{commit_list}{A list of commit hash codes, branches' names or anything +else you can use with git checkout \link{...}} + +\item{cypress_dir}{The directory with tests recorded by Cypress. +It can also be a vector of the same size of commit_list} + +\item{tests_pattern}{Cypress/shinytest2 files pattern. E.g. 'shinytest2' +It can also be a vector of the same size of commit_list. If it is NULL, +all the content in cypress_dir/shinytest2_dir will be used} + +\item{app_dir}{The path to the application root} + +\item{port}{Port to run the app} + +\item{use_renv}{In case it is set as TRUE, package will try to apply +renv::restore() in all branches. Otherwise, the current loaded list of +packages will be used in all branches.} + +\item{renv_prompt}{Prompt the user before taking any action?} + +\item{n_rep}{Number of replications desired} + +\item{debug}{Logical. TRUE to display all the system messages on runtime} +} +\description{ +Run the performance test based on multiple commits using Cypress +} diff --git a/man/benchmark_shinytest2.Rd b/man/benchmark_shinytest2.Rd new file mode 100644 index 0000000..952e632 --- /dev/null +++ b/man/benchmark_shinytest2.Rd @@ -0,0 +1,43 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/benchmark_shinytest2.R +\name{benchmark_shinytest2} +\alias{benchmark_shinytest2} +\title{Run the performance test based on a multiple commits using shinytest2} +\usage{ +benchmark_shinytest2( + commit_list, + shinytest2_dir, + tests_pattern, + app_dir, + use_renv, + renv_prompt, + n_rep, + debug +) +} +\arguments{ +\item{commit_list}{A list of commit hash codes, branches' names or anything +else you can use with git checkout \link{...}} + +\item{shinytest2_dir}{The directory with tests recorded by shinytest2 +It can also be a vector of the same size of commit_list} + +\item{tests_pattern}{shinytest2 files pattern. E.g. 'performance' +It can also be a vector of the same size of commit_list. If it is NULL, +all the content in cypress_dir/shinytest2_dir will be used} + +\item{app_dir}{The path to the application root} + +\item{use_renv}{In case it is set as TRUE, package will try to apply +renv::restore() in all branches. Otherwise, the current loaded list of +packages will be used in all branches.} + +\item{renv_prompt}{Prompt the user before taking any action?} + +\item{n_rep}{Number of replications desired} + +\item{debug}{Logical. TRUE to display all the system messages on runtime} +} +\description{ +Run the performance test based on a multiple commits using shinytest2 +} diff --git a/man/check_uncommitted_files.Rd b/man/check_uncommitted_files.Rd new file mode 100644 index 0000000..c26be49 --- /dev/null +++ b/man/check_uncommitted_files.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{check_uncommitted_files} +\alias{check_uncommitted_files} +\title{Check for uncommitted files} +\usage{ +check_uncommitted_files() +} +\description{ +Check for uncommitted files +} +\keyword{internal} diff --git a/man/checkout.Rd b/man/checkout.Rd new file mode 100644 index 0000000..090a811 --- /dev/null +++ b/man/checkout.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{checkout} +\alias{checkout} +\title{Checkout GitHub branch} +\usage{ +checkout(branch, debug) +} +\arguments{ +\item{branch}{Commit hash code or branch name} + +\item{debug}{Logical. TRUE to display all the system messages on runtime} +} +\description{ +checkout and go to a different branch +} +\keyword{internal} diff --git a/man/checkout_files.Rd b/man/checkout_files.Rd index 4827d50..b27e7f7 100644 --- a/man/checkout_files.Rd +++ b/man/checkout_files.Rd @@ -4,9 +4,13 @@ \alias{checkout_files} \title{Checkout GitHub files} \usage{ -checkout_files() +checkout_files(debug) +} +\arguments{ +\item{debug}{Logical. TRUE to display all the system messages on runtime} } \description{ -checkout anything created by the app. It prevents errors when +Checkout anything created by the app. It prevents errors when changing branches } +\keyword{internal} diff --git a/man/create_cypress_list.Rd b/man/create_cypress_list.Rd index 104eb3a..d970359 100644 --- a/man/create_cypress_list.Rd +++ b/man/create_cypress_list.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R +% Please edit documentation in R/utils_cypress.R \name{create_cypress_list} \alias{create_cypress_list} \title{Create the cypress configuration list} @@ -8,7 +8,10 @@ create_cypress_list(plugins_file, port) } \arguments{ \item{plugins_file}{The path to the Cypress plugins} + +\item{port}{Port to run the app} } \description{ Create the cypress configuration list } +\keyword{internal} diff --git a/man/create_cypress_plugins.Rd b/man/create_cypress_plugins.Rd index 208e23c..6e27410 100644 --- a/man/create_cypress_plugins.Rd +++ b/man/create_cypress_plugins.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R +% Please edit documentation in R/utils_cypress.R \name{create_cypress_plugins} \alias{create_cypress_plugins} \title{Create the JS code to track execution time} @@ -9,3 +9,4 @@ create_cypress_plugins() \description{ Create the JS code to track execution time } +\keyword{internal} diff --git a/man/create_tests_structure.Rd b/man/create_cypress_structure.Rd similarity index 68% rename from man/create_tests_structure.Rd rename to man/create_cypress_structure.Rd index 8d22fdc..292c26a 100644 --- a/man/create_tests_structure.Rd +++ b/man/create_cypress_structure.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R -\name{create_tests_structure} -\alias{create_tests_structure} +% Please edit documentation in R/utils_cypress.R +\name{create_cypress_structure} +\alias{create_cypress_structure} \title{Create a temporary directory to store everything needed by Cypress} \usage{ -create_tests_structure(app_dir, port, debug) +create_cypress_structure(app_dir, port, debug) } \arguments{ \item{app_dir}{The path to the application root} @@ -16,3 +16,4 @@ create_tests_structure(app_dir, port, debug) \description{ Create a temporary directory to store everything needed by Cypress } +\keyword{internal} diff --git a/man/create_cypress_tests.Rd b/man/create_cypress_tests.Rd index 789df5e..6baad5d 100644 --- a/man/create_cypress_tests.Rd +++ b/man/create_cypress_tests.Rd @@ -1,16 +1,21 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R +% Please edit documentation in R/utils_cypress.R \name{create_cypress_tests} \alias{create_cypress_tests} \title{Create the cypress files under project directory} \usage{ -create_cypress_tests(project_path, cypress_file) +create_cypress_tests(project_path, cypress_dir, tests_pattern) } \arguments{ -\item{project_path}{The path to the project with all needed packages installed} +\item{project_path}{The path to the project with all needed packages +installed} -\item{cypress_file}{The path to the .js file conteining cypress tests to be recorded} +\item{cypress_dir}{The directory with tests recorded by Cypress} + +\item{tests_pattern}{Cypress files pattern. E.g. 'performance'. If it is NULL, +all the content will be used} } \description{ Create the cypress files under project directory } +\keyword{internal} diff --git a/man/create_node_list.Rd b/man/create_node_list.Rd index 249eaaf..4c6dd4f 100644 --- a/man/create_node_list.Rd +++ b/man/create_node_list.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R +% Please edit documentation in R/utils_cypress.R \name{create_node_list} \alias{create_node_list} \title{Create the list of needed libraries} @@ -8,7 +8,10 @@ create_node_list(tests_path, port) } \arguments{ \item{tests_path}{The path to project} + +\item{port}{Port to run the app} } \description{ Create the list of needed libraries } +\keyword{internal} diff --git a/man/create_progress_bar.Rd b/man/create_progress_bar.Rd new file mode 100644 index 0000000..f1bcf92 --- /dev/null +++ b/man/create_progress_bar.Rd @@ -0,0 +1,15 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{create_progress_bar} +\alias{create_progress_bar} +\title{Create a progress bar to follow the execution} +\usage{ +create_progress_bar(total = 100) +} +\arguments{ +\item{total}{Total number of replications} +} +\description{ +Create a progress bar to follow the execution +} +\keyword{internal} diff --git a/man/create_shinytest2_structure.Rd b/man/create_shinytest2_structure.Rd new file mode 100644 index 0000000..ef03ad8 --- /dev/null +++ b/man/create_shinytest2_structure.Rd @@ -0,0 +1,15 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils_shinytest2.R +\name{create_shinytest2_structure} +\alias{create_shinytest2_structure} +\title{Create a temporary directory to store everything needed by shinytest2} +\usage{ +create_shinytest2_structure(app_dir) +} +\arguments{ +\item{app_dir}{The path to the application root} +} +\description{ +Create a temporary directory to store everything needed by shinytest2 +} +\keyword{internal} diff --git a/man/figures/shiny_benchmark.png b/man/figures/shiny_benchmark.png new file mode 100644 index 0000000..76f943b Binary files /dev/null and b/man/figures/shiny_benchmark.png differ diff --git a/man/get_commit_date.Rd b/man/get_commit_date.Rd index f218074..fd23414 100644 --- a/man/get_commit_date.Rd +++ b/man/get_commit_date.Rd @@ -12,3 +12,4 @@ get_commit_date(branch) \description{ Get the commit date in POSIXct format } +\keyword{internal} diff --git a/man/get_commit_hash.Rd b/man/get_commit_hash.Rd index 90f2e44..514dd25 100644 --- a/man/get_commit_hash.Rd +++ b/man/get_commit_hash.Rd @@ -9,3 +9,4 @@ get_commit_hash() \description{ Find the hash code of the current commit } +\keyword{internal} diff --git a/man/load_example.Rd b/man/load_example.Rd new file mode 100644 index 0000000..df8ada0 --- /dev/null +++ b/man/load_example.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{load_example} +\alias{load_example} +\title{Load an application and instructions to run shiny.benchmark} +\usage{ +load_example(path, force = FALSE) +} +\arguments{ +\item{path}{A character vector of full path name} + +\item{force}{Create example even if directory does not exist or is not empty} +} +\description{ +This function aims to generate a template to be used +by shiny.benchmark. It will create the necessary structure on \code{path} with +some examples of tests using Cypress and shinytest2. Also, a simple +application will be added to the folder as well as instructions on how +to perform the performance checks. Be aware that a new git repo is need in +the selected \code{path}. +} +\examples{ +load_example(file.path(tempdir(), "example_destination"), force = TRUE) +} diff --git a/man/move_shinytest2_tests.Rd b/man/move_shinytest2_tests.Rd new file mode 100644 index 0000000..a47c0f7 --- /dev/null +++ b/man/move_shinytest2_tests.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils_shinytest2.R +\name{move_shinytest2_tests} +\alias{move_shinytest2_tests} +\title{Move tests to a temporary folder} +\usage{ +move_shinytest2_tests(project_path, shinytest2_dir) +} +\arguments{ +\item{project_path}{The path to the project} + +\item{shinytest2_dir}{The directory with tests recorded by shinytest2} +} +\description{ +Move tests to a temporary folder +} +\keyword{internal} diff --git a/man/performance_test_cmd.Rd b/man/performance_test_cmd.Rd new file mode 100644 index 0000000..82a390a --- /dev/null +++ b/man/performance_test_cmd.Rd @@ -0,0 +1,15 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{performance_test_cmd} +\alias{performance_test_cmd} +\title{Running the node script "performance_test" is system-dependent} +\usage{ +performance_test_cmd(project_path) +} +\arguments{ +\item{project_path}{path to project directory (one level above node)} +} +\description{ +Running the node script "performance_test" is system-dependent +} +\keyword{internal} diff --git a/man/performance_tests.Rd b/man/performance_tests.Rd deleted file mode 100644 index 586d634..0000000 --- a/man/performance_tests.Rd +++ /dev/null @@ -1,28 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/performance_tests.R -\name{performance_tests} -\alias{performance_tests} -\title{Execute performance tests for a list of commits} -\usage{ -performance_tests( - commit_list, - cypress_file, - app_dir = getwd(), - port = 3333, - debug = FALSE -) -} -\arguments{ -\item{commit_list}{A list of commit hash codes, branches' names or anything else you can use with git checkout \link{...}} - -\item{cypress_file}{The path to the .js file containing cypress tests to be recorded} - -\item{app_dir}{The path to the application root} - -\item{port}{Port to run the app} - -\item{debug}{Logical. TRUE to display all the system messages on runtime} -} -\description{ -Execute performance tests for a list of commits -} diff --git a/man/plot.shiny_benchmark.Rd b/man/plot.shiny_benchmark.Rd new file mode 100644 index 0000000..f3d2036 --- /dev/null +++ b/man/plot.shiny_benchmark.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{plot.shiny_benchmark} +\alias{plot.shiny_benchmark} +\title{Plot for shiny_benchmark class} +\usage{ +\method{plot}{shiny_benchmark}(x, ...) +} +\arguments{ +\item{x}{shiny_benchmark object} + +\item{...}{Other parameters} +} +\description{ +Plot for shiny_benchmark class +} diff --git a/man/print.shiny_benchmark.Rd b/man/print.shiny_benchmark.Rd new file mode 100644 index 0000000..dcdea35 --- /dev/null +++ b/man/print.shiny_benchmark.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/print.R +\name{print.shiny_benchmark} +\alias{print.shiny_benchmark} +\title{Print for shiny_benchmark class} +\usage{ +\method{print}{shiny_benchmark}(x, ...) +} +\arguments{ +\item{x}{shiny_benchmark object} + +\item{...}{Other parameters} +} +\description{ +Print for shiny_benchmark class +} diff --git a/man/restore_env.Rd b/man/restore_env.Rd new file mode 100644 index 0000000..ec881e0 --- /dev/null +++ b/man/restore_env.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{restore_env} +\alias{restore_env} +\title{Check and restore renv} +\usage{ +restore_env(branch, renv_prompt) +} +\arguments{ +\item{branch}{Commit hash code or branch name. Useful to create an +informative error message} + +\item{renv_prompt}{Prompt the user before taking any action?} +} +\description{ +Check whether renv is in use in the current branch. Raise error +if renv is not in use or apply renv:restore() in the case the package is +present +} +\keyword{internal} diff --git a/man/run_cypress_ptest.Rd b/man/run_cypress_ptest.Rd new file mode 100644 index 0000000..fc978db --- /dev/null +++ b/man/run_cypress_ptest.Rd @@ -0,0 +1,41 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/benchmark_cypress.R +\name{run_cypress_ptest} +\alias{run_cypress_ptest} +\title{Run the performance test based on a single commit using Cypress} +\usage{ +run_cypress_ptest( + commit, + project_path, + cypress_dir, + tests_pattern, + use_renv, + renv_prompt, + n_rep, + debug +) +} +\arguments{ +\item{commit}{A commit hash code or a branch's name} + +\item{project_path}{The path to the project with all needed packages +installed} + +\item{cypress_dir}{The directory with tests recorded by Cypress} + +\item{tests_pattern}{Cypress files pattern. E.g. 'performance'. If it is NULL, +all the content will be used} + +\item{use_renv}{In case it is set as TRUE, package will try to apply +renv::restore() in all branches. Otherwise, the current loaded list of +packages will be used in all branches.} + +\item{renv_prompt}{Prompt the user before taking any action?} + +\item{n_rep}{Number of replications desired} + +\item{debug}{Logical. TRUE to display all the system messages on runtime} +} +\description{ +Run the performance test based on a single commit using Cypress +} diff --git a/man/run_performance_test.Rd b/man/run_performance_test.Rd deleted file mode 100644 index 2271ac9..0000000 --- a/man/run_performance_test.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/performance_tests.R -\name{run_performance_test} -\alias{run_performance_test} -\title{Run the performance test based on a single commit} -\usage{ -run_performance_test(commit, project_path, cypress_file, txt_file, debug) -} -\arguments{ -\item{commit}{A commit hash code or a branch's name} - -\item{project_path}{The path to the project with all needed packages installed} - -\item{cypress_file}{The path to the .js file conteining cypress tests to be recorded} - -\item{txt_file}{The path to the file where it is aimed to save the times} - -\item{debug}{Logical. TRUE to display all the system messages on runtime} -} -\description{ -Run the performance test based on a single commit -} diff --git a/man/run_shinytest2_ptest.Rd b/man/run_shinytest2_ptest.Rd new file mode 100644 index 0000000..a1e6662 --- /dev/null +++ b/man/run_shinytest2_ptest.Rd @@ -0,0 +1,43 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/benchmark_shinytest2.R +\name{run_shinytest2_ptest} +\alias{run_shinytest2_ptest} +\title{Run the performance test based on a single commit using shinytest2} +\usage{ +run_shinytest2_ptest( + commit, + project_path, + app_dir, + shinytest2_dir, + tests_pattern, + use_renv, + renv_prompt, + n_rep, + debug +) +} +\arguments{ +\item{commit}{A commit hash code or a branch's name} + +\item{project_path}{The path to the project} + +\item{app_dir}{The path to the application root} + +\item{shinytest2_dir}{The directory with tests recorded by shinytest2} + +\item{tests_pattern}{shinytest2 files pattern. E.g. 'performance'. If it is NULL, +all the content will be used} + +\item{use_renv}{In case it is set as TRUE, package will try to apply +renv::restore() in all branches. Otherwise, the current loaded list of +packages will be used in all branches.} + +\item{renv_prompt}{Prompt the user before taking any action?} + +\item{n_rep}{Number of replications desired} + +\item{debug}{Logical. TRUE to display all the system messages on runtime} +} +\description{ +Run the performance test based on a single commit using shinytest2 +} diff --git a/man/shiny_benchmark-class.Rd b/man/shiny_benchmark-class.Rd new file mode 100644 index 0000000..6028c28 --- /dev/null +++ b/man/shiny_benchmark-class.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/shiny_benchmark-class.R +\docType{class} +\name{shiny_benchmark-class} +\alias{shiny_benchmark-class} +\alias{shiny_benchmark_class} +\title{An object of 'shiny_benchmark' class} +\description{ +An object of 'shiny_benchmark' class +} +\section{Slots}{ + +\describe{ +\item{\code{call}}{Function call} + +\item{\code{time}}{Time elapsed} + +\item{\code{performance}}{List of measurements (one entry for each commit)} +}} + diff --git a/man/summarise_commit.Rd b/man/summarise_commit.Rd new file mode 100644 index 0000000..623d10a --- /dev/null +++ b/man/summarise_commit.Rd @@ -0,0 +1,15 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{summarise_commit} +\alias{summarise_commit} +\title{Return statistics based on the set of tests replications} +\usage{ +summarise_commit(object) +} +\arguments{ +\item{object}{A shiny_benchmark object} +} +\description{ +Return statistics based on the set of tests replications +} +\keyword{internal} diff --git a/man/summary.shiny_benchmark.Rd b/man/summary.shiny_benchmark.Rd new file mode 100644 index 0000000..6b3c298 --- /dev/null +++ b/man/summary.shiny_benchmark.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/summary.R +\name{summary.shiny_benchmark} +\alias{summary.shiny_benchmark} +\title{Summary for shiny_benchmark class} +\usage{ +\method{summary}{shiny_benchmark}(object, ...) +} +\arguments{ +\item{object}{shiny_benchmark object} + +\item{...}{Other parameters} +} +\description{ +Summary for shiny_benchmark class +} diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml new file mode 100644 index 0000000..a7b789c --- /dev/null +++ b/pkgdown/_pkgdown.yml @@ -0,0 +1,74 @@ +title: shiny.benchmark +template: + bootstrap: 5 + bootswatch: pulse + bslib: + pkgdown-nav-height: 100px + includes: + in_header: | + + + + + before_navbar: | + + +url: https://github.com/Appsilon/shiny.benchmark/ + +navbar: + bg: primary + left: + - icon: fa-home + href: index.html + text: "Start" + - icon: fa-university + href: articles/tutorial/how-to-measure-apps-performance.html + text: "Tutorial" + - icon: fa-file-code-o + text: "Reference" + href: reference/index.html + right: + - icon: fa-github fa-lg + href: https://github.com/Appsilon/shiny.benchmark + - icon: fa-twitter fa-lg + href: https://twitter.com/Appsilon + - icon: fab fa-mastodon fa-lg + href: https://fosstodon.org/@appsilon + +home: + sidebar: + structure: [star, links, license, community, citation, authors, dev] + components: + star: + title: GitHub + text: | + Star + +reference: +- title: Performance tests + contents: + - '`benchmark`' + - '`benchmark_cypress`' + - '`benchmark_shinytest2`' + - '`run_cypress_ptest`' + - '`run_shinytest2_ptest`' +- title: Shiny Benchmark Class + contents: + - '`shiny_benchmark-class`' + - '`summary.shiny_benchmark`' + - '`plot.shiny_benchmark`' + - '`print.shiny_benchmark`' +- title: Other + contents: + - '`load_example`' + +footer: + structure: + left: developed + components: + developed: "Developed with :heart: by [Appsilon](https://appsilon.com)." diff --git a/pkgdown/extra.css b/pkgdown/extra.css new file mode 100644 index 0000000..482904a --- /dev/null +++ b/pkgdown/extra.css @@ -0,0 +1,55 @@ +.navbar { + background-color: rgb(178, 9, 41) !important; +} + +#navbar > ul.navbar-nav > li.nav-item a:hover { + background-color: rgb(178, 9, 41) !important; +} + +.navbar-dark .navbar-nav .active>.nav-link { + background-color: rgb(178, 9, 41) !important; + color: #fff; +} + +.navbar-dark input[type="search"] { + background-color: #fff !important; + color: #444 !important; +} + +nav .text-muted { + color: #d8d8d8 !important; +} + +a { + color: rgb(156, 19, 44); +} + +a:hover { + color: rgb(178, 9, 41); +} + +button.btn.btn-primary.btn-copy-ex { + background-color: rgb(178, 9, 41); + border-color: rgb(178, 9, 41); +} + +.home { + left: 0px; + position: absolute; + padding: 8px 30px; + color: rgba(255,255,255,0.55); +} + +.home:hover { + color: rgba(255,255,255,0.9); +} + +.app-preview { + margin: 1.5em 0.75em; + padding: 0.25em; + box-shadow: + 0 3.9px 4.6px rgba(0, 0, 0, 0.08), + 0 12.3px 8.4px rgba(0, 0, 0, 0.056), + 0 18.8px 19.2px rgba(0, 0, 0, 0.037), + 0 22px 40px rgba(0, 0, 0, 0.019); +} diff --git a/shiny.performance.Rproj b/shiny.benchmark.Rproj similarity index 88% rename from shiny.performance.Rproj rename to shiny.benchmark.Rproj index 69fafd4..6ff5a50 100644 --- a/shiny.performance.Rproj +++ b/shiny.benchmark.Rproj @@ -19,4 +19,4 @@ LineEndingConversion: Posix BuildType: Package PackageUseDevtools: Yes PackageInstallArgs: --no-multiarch --with-keep.source -PackageRoxygenize: rd,collate,namespace +PackageRoxygenize: rd,collate,namespace,vignette diff --git a/tests/end2end/app/.gitignore b/tests/end2end/app/.gitignore new file mode 100644 index 0000000..c131308 --- /dev/null +++ b/tests/end2end/app/.gitignore @@ -0,0 +1,47 @@ +# History files +.Rhistory +.Rapp.history + +# Session Data files +.RData + +# User-specific files +.Ruserdata + +# Example code in package build process +*-Ex.R + +# Output files from R CMD build +/*.tar.gz + +# Output files from R CMD check +/*.Rcheck/ + + # RStudio files + .Rproj.user/ + + # produced vignettes + vignettes/*.html +vignettes/*.pdf + +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth + +# knitr and R markdown default cache directories +*_cache/ + /cache/ + + # Temporary files created by R markdown + *.utf8.md +*.knit.md + +# R Environment Variables +.Renviron +__pycache__ +.Rproj.user +TODO.txt + +# csv opened files +.~lock* + +renv/ diff --git a/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_1.js b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_1.js new file mode 100644 index 0000000..3943a97 --- /dev/null +++ b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_1.js @@ -0,0 +1,30 @@ +describe('Cypress test', () => { + // Test that the app starts at all + // Also it is needed to start other tests + it('The app starts', () => { + cy.visit('/'); + }); + + // Test how long it takes to wait for out1 + it('Out1 time elapsed', () => { + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); + + // Test if we have a title + it('App has a title', () => { + cy.contains('Measuring time in different commits').should('be.visible'); + }); +}); diff --git a/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_2.js b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_2.js new file mode 100644 index 0000000..3943a97 --- /dev/null +++ b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_2.js @@ -0,0 +1,30 @@ +describe('Cypress test', () => { + // Test that the app starts at all + // Also it is needed to start other tests + it('The app starts', () => { + cy.visit('/'); + }); + + // Test how long it takes to wait for out1 + it('Out1 time elapsed', () => { + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); + + // Test if we have a title + it('App has a title', () => { + cy.contains('Measuring time in different commits').should('be.visible'); + }); +}); diff --git a/tests/end2end/app/fake_folder/tests/testthat.R b/tests/end2end/app/fake_folder/tests/testthat.R new file mode 100644 index 0000000..7d25b5b --- /dev/null +++ b/tests/end2end/app/fake_folder/tests/testthat.R @@ -0,0 +1 @@ +shinytest2::test_app() diff --git a/tests/end2end/app/fake_folder/tests/testthat/setup.R b/tests/end2end/app/fake_folder/tests/testthat/setup.R new file mode 100644 index 0000000..be65b4f --- /dev/null +++ b/tests/end2end/app/fake_folder/tests/testthat/setup.R @@ -0,0 +1,2 @@ +# Load application support files into testing environment +shinytest2::load_app_env() diff --git a/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_1.R b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_1.R new file mode 100644 index 0000000..7c504e7 --- /dev/null +++ b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_1.R @@ -0,0 +1,21 @@ +library(shinytest2) + +test_that("{shinytest2} recording: test1", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + + +test_that("{shinytest2} recording: test2", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + + +test_that("{shinytest2} recording: test3", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) diff --git a/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_2.R b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_2.R new file mode 100644 index 0000000..7c504e7 --- /dev/null +++ b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_2.R @@ -0,0 +1,21 @@ +library(shinytest2) + +test_that("{shinytest2} recording: test1", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + + +test_that("{shinytest2} recording: test2", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + + +test_that("{shinytest2} recording: test3", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) diff --git a/tests/end2end/app/global.R b/tests/end2end/app/global.R new file mode 100644 index 0000000..5a0cab8 --- /dev/null +++ b/tests/end2end/app/global.R @@ -0,0 +1 @@ +library(shiny) diff --git a/tests/end2end/app/server.R b/tests/end2end/app/server.R new file mode 100644 index 0000000..84b0505 --- /dev/null +++ b/tests/end2end/app/server.R @@ -0,0 +1,39 @@ +function(input, output, session) { + # Sys.sleep + react1 <- eventReactive(input$run1, { + out <- system.time( + Sys.sleep(0.1) + ) + + return(out[3]) + }) + + react2 <- eventReactive(input$run2, { + out <- system.time( + Sys.sleep(0.1) + ) + + return(out[3]) + }) + + react3 <- eventReactive(input$run3, { + out <- system.time( + Sys.sleep(0.1) + ) + + return(out[1]) + }) + + # outputs + output$out1 <- renderUI({ + tags$span(round(react1()), style = "font-size: 500px;") + }) + + output$out2 <- renderUI({ + tags$span(round(react2()), style = "font-size: 500px;") + }) + + output$out3 <- renderUI({ + tags$span(round(react3()), style = "font-size: 500px;") + }) +} diff --git a/tests/end2end/app/tests/cypress/cypress_use_this_one_1.js b/tests/end2end/app/tests/cypress/cypress_use_this_one_1.js new file mode 100644 index 0000000..3943a97 --- /dev/null +++ b/tests/end2end/app/tests/cypress/cypress_use_this_one_1.js @@ -0,0 +1,30 @@ +describe('Cypress test', () => { + // Test that the app starts at all + // Also it is needed to start other tests + it('The app starts', () => { + cy.visit('/'); + }); + + // Test how long it takes to wait for out1 + it('Out1 time elapsed', () => { + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); + + // Test if we have a title + it('App has a title', () => { + cy.contains('Measuring time in different commits').should('be.visible'); + }); +}); diff --git a/tests/end2end/app/tests/cypress/cypress_use_this_one_2.js b/tests/end2end/app/tests/cypress/cypress_use_this_one_2.js new file mode 100644 index 0000000..3943a97 --- /dev/null +++ b/tests/end2end/app/tests/cypress/cypress_use_this_one_2.js @@ -0,0 +1,30 @@ +describe('Cypress test', () => { + // Test that the app starts at all + // Also it is needed to start other tests + it('The app starts', () => { + cy.visit('/'); + }); + + // Test how long it takes to wait for out1 + it('Out1 time elapsed', () => { + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); + + // Test if we have a title + it('App has a title', () => { + cy.contains('Measuring time in different commits').should('be.visible'); + }); +}); diff --git a/tests/end2end/app/tests/testthat.R b/tests/end2end/app/tests/testthat.R new file mode 100644 index 0000000..7d25b5b --- /dev/null +++ b/tests/end2end/app/tests/testthat.R @@ -0,0 +1 @@ +shinytest2::test_app() diff --git a/tests/end2end/app/tests/testthat/setup.R b/tests/end2end/app/tests/testthat/setup.R new file mode 100644 index 0000000..be65b4f --- /dev/null +++ b/tests/end2end/app/tests/testthat/setup.R @@ -0,0 +1,2 @@ +# Load application support files into testing environment +shinytest2::load_app_env() diff --git a/tests/end2end/app/tests/testthat/test-use_this_one_1.R b/tests/end2end/app/tests/testthat/test-use_this_one_1.R new file mode 100644 index 0000000..7c504e7 --- /dev/null +++ b/tests/end2end/app/tests/testthat/test-use_this_one_1.R @@ -0,0 +1,21 @@ +library(shinytest2) + +test_that("{shinytest2} recording: test1", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + + +test_that("{shinytest2} recording: test2", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + + +test_that("{shinytest2} recording: test3", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) diff --git a/tests/end2end/app/tests/testthat/test-use_this_one_2.R b/tests/end2end/app/tests/testthat/test-use_this_one_2.R new file mode 100644 index 0000000..7c504e7 --- /dev/null +++ b/tests/end2end/app/tests/testthat/test-use_this_one_2.R @@ -0,0 +1,21 @@ +library(shinytest2) + +test_that("{shinytest2} recording: test1", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + + +test_that("{shinytest2} recording: test2", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + + +test_that("{shinytest2} recording: test3", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) diff --git a/tests/end2end/app/ui.R b/tests/end2end/app/ui.R new file mode 100644 index 0000000..8d3615a --- /dev/null +++ b/tests/end2end/app/ui.R @@ -0,0 +1,20 @@ +function() { + bootstrapPage( + tags$h1("Measuring time in different commits"), + column( + width = 4, + actionButton(inputId = "run1", label = "Run 1"), + uiOutput(outputId = "out1") + ), + column( + width = 4, + actionButton(inputId = "run2", label = "Run 2"), + uiOutput(outputId = "out2") + ), + column( + width = 4, + actionButton(inputId = "run3", label = "Run 3"), + uiOutput(outputId = "out3") + ) + ) +} diff --git a/tests/end2end/run_tests.R b/tests/end2end/run_tests.R new file mode 100644 index 0000000..bac5593 --- /dev/null +++ b/tests/end2end/run_tests.R @@ -0,0 +1,51 @@ +#!/usr/bin/env Rscript +args <- commandArgs(trailingOnly = TRUE) +args <- strsplit(args, ",") + +# packages +library(shiny) +library(testthat) +library(shiny.benchmark) + +# commits to compare +type <- args[[1]] +commit_list <- args[[2]] +dir <- args[[3]] +pattern <- args[[4]] +use_renv <- as.logical(args[[5]]) +n_rep <- as.integer(args[[6]]) + +if (type == "cypress") { + # run performance check using Cypress + out <- benchmark( + commit_list = commit_list, + cypress_dir = dir, + tests_pattern = pattern, + app_dir = getwd(), + use_renv = use_renv, + renv_prompt = FALSE, + port = 3333, + n_rep = n_rep, + debug = FALSE + ) +} else { + # run performance check using shinytest2 + out <- benchmark( + commit_list = commit_list, + shinytest2_dir = dir, + tests_pattern = pattern, + app_dir = getwd(), + use_renv = use_renv, + renv_prompt = FALSE, + port = 3333, + n_rep = n_rep, + debug = FALSE + ) +} + +# checks +stopifnot(length(out$performance) == length(commit_list)) +stopifnot(length(out$performance[[1]]) >= n_rep) + +# deactivate renv +renv::deactivate() diff --git a/tests/end2end/setting_branches.sh b/tests/end2end/setting_branches.sh new file mode 100644 index 0000000..e15d555 --- /dev/null +++ b/tests/end2end/setting_branches.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +# starting +git init +git config --global advice.detachedHead false + +# credentials +git config --local user.name "$GITHUB_ACTOR" +git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" + +# STANDARD FUNCTIONALITIES +## master +git add . +git commit -m "first commit" + +## develop +git checkout -b develop +git commit --allow-empty -m "dummy commit to change hash" + +# RENV FUNCTIONALITIES +## No renv at all +git branch renv_missing master +git checkout renv_missing +git commit --allow-empty -m "dummy commit to change hash" + +## Creating renv +git branch renv_shiny1 master +git checkout renv_shiny1 +R -e 'renv::init()' +git add . +git commit -m "renv active" + +## Downgrading shiny +git checkout -b renv_shiny2 +R -e 'renv::install("shiny@1.7.0")' +R -e 'renv::snapshot()' +git add . +git commit -m "downgrading shiny" + +## Switching back to master +git checkout master diff --git a/tests/testthat.R b/tests/testthat.R new file mode 100644 index 0000000..85df7b5 --- /dev/null +++ b/tests/testthat.R @@ -0,0 +1,4 @@ +library(testthat) +library(shiny.benchmark) + +test_check("shiny.benchmark") diff --git a/tests/testthat/test-load_example.R b/tests/testthat/test-load_example.R new file mode 100644 index 0000000..20688e8 --- /dev/null +++ b/tests/testthat/test-load_example.R @@ -0,0 +1,84 @@ +# Necessary test as fs::dir_copy with overwrite has a different behavior +# than file.copy. +# It copies the content of the directory of the "from" path into the +# destination, instead of the directory itself +test_that("Load example creates correct structure", { + example_path <- fs::path(tempdir(), "load_example") + fs::dir_create(example_path) + local({ + local_mock(menu = function(...) stop("Opps, shouldn't reach this")) + load_example(example_path, force = TRUE) + }) + + files <- example_path |> + fs::path( + c( + "run_tests.R", + fs::path("app", "ui.R"), + fs::path("app", "server.R"), + fs::path("app", "global.R"), + fs::path("app", "tests", "testthat.R"), + fs::path("app", "tests", "testthat", "setup.R"), + fs::path("app", "tests", "testthat", "test-use_this_one_1.R"), + fs::path("app", "tests", "testthat", "test-use_this_one_2.R") + ) + ) + + dirs <- example_path |> + fs::path( + c( + fs::path("app", "tests"), + fs::path("app", "tests", "cypress"), + fs::path("app", "tests", "testthat") + ) + ) + + expect_true(all(fs::file_exists(files))) + expect_true(all(fs::is_file(files))) + expect_false(all(fs::is_dir(files))) + + expect_true(all(fs::dir_exists(dirs))) + expect_true(all(fs::is_dir(dirs))) + expect_false(all(fs::is_file(dirs))) +}) + +test_that("Does not create load_examples on non-existing directory", { + example_path <- fs::path( + tempdir(), + glue::glue("load_example_not_existing{unclass(Sys.time())}") + ) + + local({ + local_mock(menu = function(...) stop("Opps, shouldn't reach this")) + load_example(example_path) |> + expect_error("You must provide a valid path") + }) + + fs::dir_create(example_path) + local({ + local_mock(menu = function(...) stop("Opps, shouldn't reach this")) + load_example(example_path) |> + expect_output("app created at") + }) +}) + +test_that("Does not create load_examples if there is a file in directory", { + example_path <- fs::path( + tempdir(), + glue::glue("load_example_not_empty{unclass(Sys.time())}") + ) + fs::dir_create(example_path) + fs::file_create(fs::path(example_path, "touch.txt")) + + local({ + local_mock(menu = function(...) 2) + load_example(example_path) |> + expect_error("Consider creating a new empty path.") + }) + + local({ + local_mock(menu = function(...) 1) + load_example(example_path) |> + expect_output("app created at") + }) +}) diff --git a/tests/testthat/test-performance_tests.R b/tests/testthat/test-performance_tests.R new file mode 100644 index 0000000..07ce8f0 --- /dev/null +++ b/tests/testthat/test-performance_tests.R @@ -0,0 +1,42 @@ +test_that("Function fails in case of missing cypress_file or shinytest2dir", { + expect_error( + performance_tests( + commit_list = list("commit_1", "commit_2"), + cypress_file = NULL, + shinytest2_dir = NULL, + app_dir = getwd(), + port = 3333, + use_renv = TRUE, + renv_prompt = TRUE, + debug = FALSE + ) + ) +}) + +test_that("Function fails in case of divergences between commit_list and files length", { + expect_error( + performance_tests( + commit_list = list("commit_1", "commit_2"), + cypress_file = c("file_1", "file_2", "file_3"), + shinytest2_dir = NULL, + app_dir = getwd(), + port = 3333, + use_renv = TRUE, + renv_prompt = TRUE, + debug = FALSE + ) + ) + + expect_error( + performance_tests( + commit_list = list("commit_1", "commit_2"), + cypress_file = NULL, + shinytest2_dir = c("file_1", "file_2", "file_3"), + app_dir = getwd(), + port = 3333, + use_renv = TRUE, + renv_prompt = TRUE, + debug = FALSE + ) + ) +}) diff --git a/tests/testthat/test-utils_cypress.R b/tests/testthat/test-utils_cypress.R new file mode 100644 index 0000000..b24d12d --- /dev/null +++ b/tests/testthat/test-utils_cypress.R @@ -0,0 +1,27 @@ +test_that("Check if we are able to add Cypress code to a txt file", { + tmp_dir <- tempdir() + add_sendtime2js( + js_file = fs::path(tmp_dir, "test.js"), + txt_file = "test.txt" + ) + + expect_true(fs::file_exists(fs::path(tmp_dir, "test.js"))) +}) + +test_that("Check if we are able to copy file content from a file to another", { + tmp_dir <- tempdir() + tmp_file <- tempfile(tmpdir = tmp_dir, fileext = ".js") + content_before <- "TEST" + writeLines(text = content_before, con = tmp_file) + + integration_dir <- fs::path(tmp_dir, "tests", "cypress", "integration") + fs::dir_create(path = integration_dir) + files <- create_cypress_tests( + project_path = tmp_dir, + cypress_dir = tmp_dir, + tests_pattern = ".js" + ) + content_after <- readLines(con = files$js_file, n = 1) + + expect_true(content_after == content_before) +}) diff --git a/tests/testthat/test-utils_shinytest2.R b/tests/testthat/test-utils_shinytest2.R new file mode 100644 index 0000000..410bd3e --- /dev/null +++ b/tests/testthat/test-utils_shinytest2.R @@ -0,0 +1,21 @@ +test_that("Check if we are able to move files properly", { + tmp_dir <- tempdir() + + tmp_dir1 <- file.path(tmp_dir, "folder1") + dir.create(tmp_dir1, showWarnings = FALSE) + tmp_dir2 <- file.path(tmp_dir, "folder2") + dir.create(tmp_dir2, showWarnings = FALSE) + + shinytest2_dir <- file.path(tmp_dir1, "tst") + shinytest2_dir_copy <- file.path(tmp_dir2, "tst") + dir.create(path = shinytest2_dir, showWarnings = FALSE) + + move_shinytest2_tests(project_path = tmp_dir2, shinytest2_dir = shinytest2_dir) + + expect_true(file.exists(shinytest2_dir_copy)) +}) + +test_that("Check if we are able to create shinytest2 structure", { + tmp_dir <- create_shinytest2_structure(app_dir = ".") + expect_true(file.exists(file.path(tmp_dir, "app.R"))) +}) diff --git a/vignettes/.gitignore b/vignettes/.gitignore new file mode 100644 index 0000000..097b241 --- /dev/null +++ b/vignettes/.gitignore @@ -0,0 +1,2 @@ +*.html +*.R diff --git a/vignettes/tutorial/how-to-measure-apps-performance.Rmd b/vignettes/tutorial/how-to-measure-apps-performance.Rmd new file mode 100644 index 0000000..10b1a95 --- /dev/null +++ b/vignettes/tutorial/how-to-measure-apps-performance.Rmd @@ -0,0 +1,472 @@ +--- +title: "Tutorial: Compare performance of different versions of a shiny application" +output: + rmarkdown::html_vignette: + self_contained: true +vignette: > + %\VignetteIndexEntry{Tutorial: Compare performance of different versions of a shiny application} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +# Setup + +## How to install shiny.benchmark? + +`shiny.benchmark` can use two different engines to test the changes in the performance of your application: [shinytest2](https://rstudio.github.io/shinytest2/) and [Cypress](https://www.cypress.io/). The latter requires `Node` (version 12 or higher) and `yarn` (version 1.22.17 or higher) to be available. To install them on your computer, follow the guidelines on the documentation pages: + +- [Node](https://nodejs.org/en/download/) +- [yarn](https://yarnpkg.com/getting-started/install) + +Besides that, on Linux, it might be required to install other `Cypress` dependencies. Check the [documentation](https://docs.cypress.io/guides/getting-started/installing-cypress#Linux-Prerequisites) to find out more. + +To install `shiny.benchmark` use the following command: + +```r +remotes::install_github("Appsilon/shiny.benchmark") +``` + +`shiny.benchmark` will handle `Cypress` installation. If you face any inconvenience using `Cypress`, please try to use `shinytest2` in the rest of this tutorial. + +---- + +# Create an initial application + +Let's start by creating an application that will serve us as a guide through the `shiny.benchmark` functionalities. + +Save the following code as `ui.R`. It is a simple user interface containing three columns with one action button in each. Also each column has an output which will be created in the server file later. + +```r +function() { + bootstrapPage( + tags$h1("Measuring time in different commits"), + column( + width = 4, + actionButton(inputId = "run1", label = "Run 1"), + uiOutput(outputId = "out1") + ), + column( + width = 4, + actionButton(inputId = "run2", label = "Run 2"), + uiOutput(outputId = "out2") + ), + column( + width = 4, + actionButton(inputId = "run3", label = "Run 3"), + uiOutput(outputId = "out3") + ) + ) +} +``` + +In the server side, the application will use the `Sys.sleep` function to simulate a task every time the user press a button. This will be helpful for us since we can easily increase/decrease the sleep time to simulate improvements/deterioration of the application. Save the following code as `server.R`: + +```r +times <- c(10, 5, 2) + +function(input, output, session) { + # Sys.sleep + react1 <- eventReactive(input$run1, { + out <- system.time( + Sys.sleep(times[1] + rexp(n = 1, rate = 1)) # we will play with the time here + ) + + return(out[3]) + }) + + react2 <- eventReactive(input$run2, { + out <- system.time( + Sys.sleep(times[2] + rexp(n = 1, rate = 1)) # we will play with the time here + ) + + return(out[3]) + }) + + react3 <- eventReactive(input$run3, { + out <- system.time( + Sys.sleep(times[3] + rexp(n = 1, rate = 1)) # we will play with the time here + ) + + return(out[1]) + }) + + # outputs + output$out1 <- renderUI({ + tags$span(round(react1()), style = "font-size: 5vw;") + }) + + output$out2 <- renderUI({ + tags$span(round(react2()), style = "font-size: 5vw;") + }) + + output$out3 <- renderUI({ + tags$span(round(react3()), style = "font-size: 5vw;") + }) +} +``` + +The application should look like this: + +```r +shiny::runApp() +``` + + + +---- + +# Tests engines + +`shiny.benchmark` works under two different engines: `Cypress` and `shinytest2`. + +## shinytest2 + +`shinytest2` is an R package maintained by [Posit](https://posit.co/) (formerly RStudio). It is handy for R users since all tests can be done using R only (differently than Cypress). To set up it easily, run `shinytest2::use_shinytest2()`. It will create configuration files which you do not need to change for this tutorial. + +Save the following code as `tests/testthat/test-set1.R`: + +```r +test_that("Out1 time elapsed - set1", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + +test_that("Out2 time elapsed - set1", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + +test_that("Out3 time elapsed - set1", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) +``` + +This code is simulating clicks in the three buttons we have in our application. Also it waits for the outputs to appear. In a new file, replace `set1` by `set2` in the code and save it as `tests/testthat/test-set2.R` as well. It will be useful to present some functionalities later. + +```r +test_that("Out1 time elapsed - set2", { + app <- AppDriver$new(name = "test1", height = 975, width = 1619) + app$click("run1") + app$expect_values(output = "out1") +}) + +test_that("Out2 time elapsed - set2", { + app <- AppDriver$new(name = "test2", height = 975, width = 1619) + app$click("run2") + app$expect_values(output = "out2") +}) + +test_that("Out3 time elapsed - set2", { + app <- AppDriver$new(name = "test3", height = 975, width = 1619) + app$click("run3") + app$expect_values(output = "out3") +}) +``` + +## Cypress + +Cypress is a widely used end to end testing JavaScript library. Because its broader usage, this engine allows the user to take advantage of a huge number of functionalities in order to test its applications. Also, the community is active and therefore it is easier to find solution for bugs you may encounter while coding. + +Save the following code as `tests/cypress/test-set1.js`: + +```r +describe('Cypress test', () => { + it('Out1 time elapsed - set1', () => { + cy.visit('/'); + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed - set1', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed - set1', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); +}); +``` + +Again, replace `set1` by `set2` in the code and save it as `tests/cypress/test-set2.R` as well. + +```r +describe('Cypress test', () => { + it('Out1 time elapsed - set2', () => { + cy.visit('/'); + cy.get('#run1').click(); + cy.get('#out1', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out2 + it('Out2 time elapsed - set2', () => { + cy.get('#run2').click(); + cy.get('#out2', {timeout: 10000}).should('be.visible'); + }); + + // Test how long it takes to wait for out3 + it('Out3 time elapsed - set2', () => { + cy.get('#run3').click(); + cy.get('#out3', {timeout: 10000}).should('be.visible'); + }); +}); +``` + + +---- + +# Package management + +During the development process, it is normal to use different packages/package versions. `renv` allow us to manage package versions and is used by `shiny.benchmark` by default. Run the following code to setup `renv` in our test application. + +```r +renv::init() +renv::install("remotes") +remotes::install_github("Appsilon/shiny.benchmark") +renv::snapshot(prompt = FALSE) +``` + + +---- + +# Simulating app versions + +In a regular project, you use `git` to maintain the code versioning. In this case, it is natural to have different app's versions in different branches/commits/releases. `shiny.benchmark` take advantage of these different `git` refs to run tests under different code versions. Add the following code to `.gitignore` to avoid problems with uncommitted files later: + +```git +.Rhistory +.Rproj.user/ +.Rproj.user +renv/ +``` + +Now, lets create a `git` repo and commit the current application into the `develop` branch: + +```git +git init +git checkout -b develop +git add . +git commit -m "first commit" +``` + +Also, let's create a new branch called `feature1`: + +```git +git checkout -b feature1 +``` + +At this point, we can simulate improvement in our application. To do so, let's change `Sys.sleep` time in the server function. Replace `times <- c(10, 5, 2)` by `times <- c(5, 2.5, 1)` in first row of `server.R` and then commit the changes. + +```git +git add server.R +git commit -m "improving performance" +``` + +To play with `renv` let's downgrade `shiny` version and snapshot it: + +```git +git checkout -b feature2 +``` + +Replace `times <- c(5, 2.5, 1)` by `times <- c(2.5, 1.25, 0.5)` in first row of `server.R`. Also, run the following code to downgrade `shiny`: + +```r +renv::install("shiny@1.0.0") +renv::snapshot(prompt = FALSE) +``` + +Commit the changes: + +```git +git add . +git commit -m "downgrading shiny" +git checkout develop +``` + +Great! We are all set! + +---- + +# shiny.benchmark + +Now we have all ingredients needed: An application, a set of tests and different versions in a `git` repo. `shiny.benchmark::benchmark` function has only two mandatory arguments: + +- `commit_list`: a named list of `git` refs (commit hashes, branch names, tags, ...) +- `cypress_dir` or `shinytest2_dir`: path to `Cypress` or `shinytest2` tests + +By default, `shiny.benchmark` uses `renv`. To turn `renv` off just set `use_renv = FALSE` in the `benchmark` call. Be aware that this function will take a while to run since the application will be started and tested 3 times (`develop`, `feature1` and `using_renv` branches). + +```r +library(shiny.benchmark) + +commits <- list( + "develop" = "develop", + "feature1" = "feature1", + "using_renv" = "feature2" +) + +cypress_dir <- "tests/cypress/" +testthat_dir <- "tests/" + +cypress_out <- benchmark( + commit_list = commits, + cypress_dir = cypress_dir, + use_renv = FALSE +) + +shinytest2_out <- benchmark( + commit_list = commits, + shinytest2_dir = testthat_dir, + use_renv = FALSE +) +``` + +Instead of a branch name, you can also use the hash code of a desired commit. The console should display something similar to: + + + +You can access the results using `cypress_out$performance` or `shinytest2_out$performance`: + +```r +cypress_out$performance +``` + +```{r echo = FALSE, eval = TRUE} +list(develop = list(structure(list(date = structure(c(1673034247, +1673034247, 1673034247, 1673034247, 1673034247, 1673034247), class = c("POSIXct", +"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1", +"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2", +"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(10782L, +6091L, 2804L, 10591L, 6768L, 3944L)), class = "data.frame", row.names = c(NA, +-6L))), feature1 = list(structure(list(date = structure(c(1673034279, +1673034279, 1673034279, 1673034279, 1673034279, 1673034279), class = c("POSIXct", +"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1", +"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2", +"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(6471L, +6442L, 1422L, 5613L, 3593L, 1272L)), class = "data.frame", row.names = c(NA, +-6L))), using_renv = list(structure(list(date = structure(c(1673034314, +1673034314, 1673034314, 1673034314, 1673034314, 1673034314), class = c("POSIXct", +"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1", +"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2", +"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(3941L, +3010L, 995L, 3082L, 2130L, 2458L)), class = "data.frame", row.names = c(NA, +-6L)))) +``` + +You can notice that both tests files are reported (`test-set1` and `test-set2`). Also, the result is a list of `data.frames` in which each entry correspond to a specific commit. + +For now on we will use only `shinytest2`. However, everything is also applied for `Cypress`. + +## Package management + +In order to use `renv`, simply assign `use_renv = TRUE`. You can also use `renv_prompt = TRUE` if you want to see what renv is applying in the background. + +```r +shinytest2_out <- benchmark( + commit_list = commits, + shinytest2_dir = testthat_dir, + use_renv = TRUE, + renv_prompt = TRUE +) +``` + +## Handling multiple files + +Sometimes it is not our interest to measure performance of all the tests we have. In order to select specific files you can use the argument `tests_pattern`. This argument accept either a vector of files (one for each item in commit list). Also, it is possible to search for a pattern in `tests` files. + +```r +shinytest2_out <- benchmark( + commit_list = commits, + shinytest2_dir = testthat_dir, + use_renv = FALSE, + tests_pattern = c("set[0-9]", "set1", "set2") +) +shinytest2_out$performance +``` + +```{r echo=FALSE} +list(develop = list(structure(list(date = structure(c(1673034247, +1673034247, 1673034247, 1673034247, 1673034247, 1673034247), class = c("POSIXct", +"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1", +"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2", +"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(12.0559999999996, +8.16600000000017, 6.94699999999921, 13.3299999999999, 7.09899999999925, +6.84699999999975)), class = "data.frame", row.names = c(NA, -6L +))), feature1 = list(structure(list(date = structure(c(1673034279, +1673034279, 1673034279), class = c("POSIXct", "POSIXt"), tzone = ""), + rep_id = c(1L, 1L, 1L), test_name = c("Out1 time elapsed - set1", + "Out2 time elapsed - set1", "Out3 time elapsed - set1"), + duration_ms = c(6.96900000000005, 5.58899999999994, 3.23300000000017 + )), class = "data.frame", row.names = c(NA, -3L))), using_renv = list( + structure(list(date = structure(c(1673034314, 1673034314, + 1673034314), class = c("POSIXct", "POSIXt"), tzone = ""), + rep_id = c(1L, 1L, 1L), test_name = c("Out1 time elapsed - set2", + "Out2 time elapsed - set2", "Out3 time elapsed - set2" + ), duration_ms = c(4.59799999999996, 4.01999999999953, + 3.41100000000006)), class = "data.frame", row.names = c(NA, + -3L)))) +``` + +Now the output is sightly different. For `develop` branch both files (`test-set1` and `test-set2`) are in use since they match the `test-set[0-9]` pattern. For `feature1` and `feature2` only one file is in use since we directly requested `test-set1` and `test-set2` files respectively. It can be useful when new tests are added during the development process and you need to run different tests for different versions. + +## Repetitions + +Sometimes it is important to repeat the measurement several times to have a distribution of the performance times instead of an unique measurement. To do so, it is possible to use the `n_rep argument` as follows: + +```r +shinytest2_out <- benchmark( + commit_list = commits, + shinytest2_dir = testthat_dir, + use_renv = FALSE, + tests_pattern = "set1", + n_rep = 5 +) +``` + +It is faster than running the benchmark several times since the test structure is created only once internally saving some execution time. + +Some methods are implemented to make it easy to explore the results. `summary` brings summarized statistics as mean, median, minimum and maximum while `plot` shows a plot with the average times for each `git` ref and test. Also it presents maximum and minimum range. + +```r +summary(shinytest2_out) +``` + +```{r echo = FALSE} +structure(list(commit = c("develop", "develop", "develop", "feature1", +"feature1", "feature1", "using_renv", "using_renv", "using_renv" +), test_name = c("Out1 time elapsed - set1", "Out2 time elapsed - set1", +"Out3 time elapsed - set1", "Out1 time elapsed - set1", "Out2 time elapsed - set1", +"Out3 time elapsed - set1", "Out1 time elapsed - set1", "Out2 time elapsed - set1", +"Out3 time elapsed - set1"), n = c(5L, 5L, 5L, 5L, 5L, 5L, 5L, +5L, 5L), mean = c(12.3748, 7.37940000000017, 5.71140000000014, +7.72180000000008, 5.35640000000003, 4.28839999999982, 5.36419999999998, +4.74899999999998, 4.52899999999991), median = c(12.2960000000003, +7.1279999999997, 6.1220000000003, 7.32099999999991, 5.4320000000007, +4.39900000000034, 5.34699999999975, 4.79299999999967, 4.5019999999995 +), sd = c(0.473558549706366, 0.628408147624124, 1.21808653223053, +1.07820856052986, 0.603523653223451, 0.775558379491765, 0.455512019599966, +0.748540246078935, 0.982901826227147), min = c(11.9110000000001, +6.91499999999996, 4.26799999999912, 7.01000000000022, 4.76000000000022, +3.1279999999997, 4.9340000000002, 3.70100000000002, 3.65899999999965 +), max = c(13.1099999999997, 8.45000000000073, 7.15100000000075, +9.61200000000008, 6.27800000000025, 5.26899999999932, 6.10800000000017, +5.50500000000011, 6.11400000000049)), class = c("tbl_df", "tbl", +"data.frame"), row.names = c(NA, -9L)) +``` + +```r +plot(shinytest2_out) +``` + + + +---- + +**Congratulations! You are now able to apply your knowledge to check the performance improvements in your own projects!** diff --git a/vignettes/tutorial/images/app.png b/vignettes/tutorial/images/app.png new file mode 100644 index 0000000..d862262 Binary files /dev/null and b/vignettes/tutorial/images/app.png differ diff --git a/vignettes/tutorial/images/console_basic.png b/vignettes/tutorial/images/console_basic.png new file mode 100644 index 0000000..17c7152 Binary files /dev/null and b/vignettes/tutorial/images/console_basic.png differ diff --git a/vignettes/tutorial/images/plot.png b/vignettes/tutorial/images/plot.png new file mode 100644 index 0000000..2d4d415 Binary files /dev/null and b/vignettes/tutorial/images/plot.png differ