diff --git a/.Rbuildignore b/.Rbuildignore
index 91114bf..03026e5 100644
--- a/.Rbuildignore
+++ b/.Rbuildignore
@@ -1,2 +1,8 @@
^.*\.Rproj$
^\.Rproj\.user$
+.github
+.lintr
+tests/end2end
+pkgdown
+docs
+vignettes
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000..16a9edf
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,31 @@
+### Link to the Issue
+
+...
+
+### Definition of Done
+
+...
+
+### How to test changes
+
+...
+
+
+### Tasks for PR author
+
+- [ ] Test your change and ensure there is no regression
+- [ ] Change has a corresponding issue. ***Ensure it is linked in GitHub***
+- [ ] Author of the change opened a pull request and assigned a reviewer
+
+### General policy:
+
+- If applicable - add instructions for testing
+- If there’s no issue, create it. Each issue needs to be well defined and described.
+- All interaction with a user, user-facing messages, plots, reports etc. are written from the perspective of the person using or receiving it. They are understandable and helpful to this person. If a user sees an error message, there is a call to action, i.e. the user knows what to do to fix it.
+- README, other documentation and code comments that we have is updated with all information related to the change.
+- All code has been peer-reviewed before merging into any main branch
+- All changes have been merged into the main branch we use for development.
+- Continuous integration checks (linter, unit tests, integration tests) are configured and pass.
+- Optional: unit tests added for all new or changed logic.
+- All task requirements satisfied. If not describe it here. The reviewer is responsible to verify each aspect of the task.
+- Change covers only things in task. Please create new PR if you want to fix something else.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 0000000..ff8cef2
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,64 @@
+on: push
+
+name: R-CMD-check
+
+jobs:
+ main:
+ name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+ runs-on: ${{ matrix.config.os }}
+
+ timeout-minutes: 30
+
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - {os: macOS-latest, r: 'release'}
+ - {os: windows-latest, r: 'release'}
+ - {os: ubuntu-22.04, r: 'devel'}
+ - {os: ubuntu-22.04, r: 'release'}
+ - {os: ubuntu-22.04, r: 'oldrel'}
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Install R
+ uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: ${{ matrix.config.r }}
+
+ - name: Install R package dependencies
+ uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ extra-packages: local::. # Necessary to avoid object usage linter errors.
+
+ - name: R CMD check
+ if: always()
+ uses: r-lib/actions/check-r-package@v2
+ with:
+ error-on: '"note"'
+
+ - name: Lint
+ if: always()
+ shell: Rscript {0}
+ run: |
+ lints <- lintr::lint_package()
+ for (lint in lints) print(lint)
+ quit(status = length(lints) > 0)
+
+ - name: Spell Check
+ if : always()
+ shell: Rscript {0}
+ run: |
+ spell_check <- spelling::spell_check_package(use_wordlist = TRUE)
+ if (nrow(spell_check) > 0) {
+ print(spell_check)
+ }
+ quit(status = nrow(spell_check) > 0)
+
+ - name: Test coverage
+ if: matrix.config.os == 'ubuntu-22.04' && matrix.config.r == 'release'
+ run: |
+ Rscript -e 'covr::codecov(token = "${{secrets.CODECOV_TOKEN}}")'
diff --git a/.github/workflows/pkgdown.yml b/.github/workflows/pkgdown.yml
new file mode 100644
index 0000000..a8edc0a
--- /dev/null
+++ b/.github/workflows/pkgdown.yml
@@ -0,0 +1,34 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+ push:
+ branches:
+ - main
+ workflow_dispatch:
+
+name: pkgdown
+
+jobs:
+ pkgdown:
+ runs-on: ubuntu-latest
+ env:
+ GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - uses: actions/checkout@v3
+
+ - uses: r-lib/actions/setup-pandoc@v2
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ use-public-rspm: true
+
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ extra-packages: any::pkgdown, local::.
+ needs: website
+
+ - name: Deploy to gh-pages branch
+ run: |
+ git config --local user.name "$GITHUB_ACTOR"
+ git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com"
+ Rscript -e 'pkgdown::deploy_to_branch(new_process = FALSE)'
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..c34d443
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,100 @@
+on: push
+
+name: package-usage-tests
+
+defaults:
+ run:
+ working-directory: ./tests/end2end/app/
+
+jobs:
+ main:
+ name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+ runs-on: ${{ matrix.config.os }}
+
+ timeout-minutes: 30
+
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - {os: macOS-latest, r: 'release'}
+ - {os: windows-latest, r: 'release'}
+ - {os: ubuntu-22.04, r: 'devel'}
+ - {os: ubuntu-22.04, r: 'release'}
+ - {os: ubuntu-22.04, r: 'oldrel'}
+
+ env:
+ GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+ BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Install R
+ uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: ${{ matrix.config.r }}
+
+ - name: Install R package dependencies
+ uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ extra-packages: local::. # Necessary to avoid object usage linter errors.
+
+ - name: Install shiny.benchmark
+ run: |
+ Rscript -e "install.packages('remotes')"
+ Rscript -e "remotes::install_local('../../../', quiet = TRUE)"
+
+ - name: Create app structure
+ run: |
+ bash ./../setting_branches.sh
+
+ - name: Check basic functionality - Cypress
+ run: |
+ Rscript ../run_tests.R cypress master,develop tests/cypress/ use_this_one_1 FALSE 1
+
+ - name: Check basic functionality - shinytest2
+ run: |
+ Rscript ../run_tests.R shinytest2 master,develop tests/ use_this_one_1 FALSE 1
+
+ - name: Check if it fails when renv not present - Cypress
+ run: |
+ Rscript ./../run_tests.R cypress renv_missing,renv_shiny1,renv_shiny2 tests/cypress/ use_this_one_1 FALSE 1
+
+ - name: Check if it fails when renv not present - shinytest2
+ run: |
+ Rscript ./../run_tests.R shinytest2 renv_missing,renv_shiny1,renv_shiny2 tests/ use_this_one_1 FALSE 1
+
+ - name: Check if it can handle renv - Cypress
+ run: |
+ Rscript ./../run_tests.R cypress renv_missing,renv_shiny1,renv_shiny2 tests/cypress/ use_this_one_1 TRUE 1
+
+ - name: Check if it can handle renv - shinytest2
+ run: |
+ Rscript ./../run_tests.R shinytest2 renv_missing,renv_shiny1,renv_shiny2 tests/ use_this_one_1 TRUE 1
+
+ - name: Check if it can handle multiple files - Cypress
+ run: |
+ Rscript ./../run_tests.R cypress renv_shiny1,renv_shiny2 tests/cypress/,fake_folder/tests/cypress/ use_this_one_1 TRUE 1
+
+ - name: Check if it can handle multiple files - shinytest2
+ run: |
+ Rscript ./../run_tests.R shinytest2 renv_shiny1,renv_shiny2 tests/,fake_folder/tests/ use_this_one_1 TRUE 1
+
+ - name: Check if we can replicate tests - Cypress
+ run: |
+ Rscript ./../run_tests.R cypress master,develop tests/cypress/ use_this_one_1 FALSE 2
+
+ - name: Check if we can replicate tests - shinytest2
+ run: |
+ Rscript ./../run_tests.R shinytest2 master,develop tests/ use_this_one_1 FALSE 2
+
+ - name: Check if we can run tests based on file patterns - Cypress
+ run: |
+ Rscript ./../run_tests.R cypress master,develop tests/cypress/ use_this_one_[0-9] FALSE 1
+
+ - name: Check if we can run tests based on file patterns - shinytest2
+ run: |
+ Rscript ./../run_tests.R shinytest2 master,develop tests/ use_this_one_[0-9] FALSE 1
diff --git a/.gitignore b/.gitignore
index fae8299..7b46709 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,7 @@ vignettes/*.pdf
# R Environment Variables
.Renviron
+
+# documentation page
+docs
+inst/doc
diff --git a/.lintr b/.lintr
new file mode 100644
index 0000000..51a8fad
--- /dev/null
+++ b/.lintr
@@ -0,0 +1,8 @@
+linters:
+ linters_with_defaults(
+ line_length_linter = line_length_linter(100)
+ )
+exclusions:
+ c(
+ "vignettes"
+ )
diff --git a/DESCRIPTION b/DESCRIPTION
index f067e11..4e73149 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,21 +1,41 @@
-Package: shiny.performance
-Title: Compare performance of several versions of a shiny app
-Version: 0.1.0
+Package: shiny.benchmark
+Title: Benchmark the Performance of Shiny Applications
+Version: 0.1.1
Authors@R:
c(
- person(given = "Douglas", family = "Azevedo", email = "douglas@appsilon.com", role = "aut"),
- person(given = "Pedro", family = "Silva", email = "pedro@appsilon.com", role = "aut"),
- person("Developers", "Appsilon", email = "support+opensource@appsilon.com", role = "cre"),
- person(family = "Appsilon Sp. z o.o.", role = "cph")
+ person(given = "Douglas", family = "Azevedo", email = "opensource+douglas@appsilon.com", role = c("aut", "cre")),
+ person(family = "Appsilon Sp. z o.o.", role = "cph", email = "opensource@appsilon.com")
)
-Description: Compare performance of several versions of a shiny app based on commit hashs
-License: LGPL-3 + file LICENSE
-URL: https://github.com/Appsilon/shiny.performance
-SystemRequirements: yarn 1.22.17 or higher, cypress 9.4.1 or higher, xvfb
+Description: Compare performance between different versions of a Shiny application based on Git references.
+License: LGPL-3
+URL: https://github.com/Appsilon/shiny.benchmark, https://github.com/Appsilon/shiny.benchmark
+BugReports: https://github.com/Appsilon/shiny.benchmark/issues
+SystemRequirements: yarn 1.22.17 or higher, Node 12 or higher
Encoding: UTF-8
LazyData: true
Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.1.2
+RoxygenNote: 7.2.3
VignetteBuilder: knitr
Depends:
R (>= 3.1.0)
+Suggests:
+ covr,
+ knitr,
+ lintr,
+ rcmdcheck,
+ rmarkdown,
+ mockr,
+ spelling
+Imports:
+ dplyr,
+ ggplot2,
+ glue,
+ jsonlite,
+ methods,
+ progress,
+ renv,
+ shinytest2,
+ stringr,
+ testthat,
+ fs
+Language: en-US
diff --git a/NAMESPACE b/NAMESPACE
index aa9a7a4..a51ca51 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -1,8 +1,28 @@
# Generated by roxygen2: do not edit by hand
-export(performance_tests)
-export(run_performance_test)
-importFrom(git2r,checkout)
+S3method(plot,shiny_benchmark)
+S3method(print,shiny_benchmark)
+S3method(summary,shiny_benchmark)
+export(benchmark)
+export(benchmark_cypress)
+export(benchmark_shinytest2)
+export(load_example)
+export(run_cypress_ptest)
+export(run_shinytest2_ptest)
+export(shiny_benchmark_class)
+exportClasses(shiny_benchmark)
+import(dplyr)
+import(ggplot2)
importFrom(glue,glue)
importFrom(jsonlite,write_json)
+importFrom(methods,new)
+importFrom(progress,progress_bar)
+importFrom(renv,activate)
+importFrom(renv,restore)
+importFrom(shinytest2,test_app)
+importFrom(stats,median)
importFrom(stringr,str_trim)
+importFrom(testthat,ListReporter)
+importFrom(utils,globalVariables)
+importFrom(utils,menu)
+importFrom(utils,read.table)
diff --git a/NEWS.md b/NEWS.md
new file mode 100644
index 0000000..d253fef
--- /dev/null
+++ b/NEWS.md
@@ -0,0 +1,3 @@
+# 0.1.1
+
+Adding a minimal example using `shinytest2`
diff --git a/R/benchmark.R b/R/benchmark.R
new file mode 100644
index 0000000..b86fc3c
--- /dev/null
+++ b/R/benchmark.R
@@ -0,0 +1,106 @@
+#' @title Execute performance tests for a list of commits
+#'
+#' @param commit_list A list of commit hash codes, branches' names or anything
+#' else you can use with git checkout [...]
+#' @param cypress_dir The directory with tests recorded by Cypress.
+#' It can also be a vector of the same size of commit_list
+#' @param shinytest2_dir The directory with tests recorded by shinytest2
+#' It can also be a vector of the same size of commit_list
+#' @param tests_pattern Cypress/shinytest2 files pattern. E.g. 'performance'
+#' It can also be a vector of the same size of commit_list. If it is NULL,
+#' all the content in cypress_dir/shinytest2_dir will be used
+#' @param app_dir The path to the application root
+#' @param port Port to run the app
+#' @param use_renv In case it is set as TRUE, package will try to apply
+#' renv::restore() in all branches. Otherwise, the current loaded list of
+#' packages will be used in all branches.
+#' @param renv_prompt Prompt the user before taking any action?
+#' @param n_rep Number of replications desired
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @importFrom glue glue
+#' @export
+benchmark <- function(
+ commit_list,
+ cypress_dir = NULL,
+ shinytest2_dir = NULL,
+ tests_pattern = NULL,
+ app_dir = getwd(),
+ port = 3333,
+ use_renv = TRUE,
+ renv_prompt = TRUE,
+ n_rep = 1,
+ debug = FALSE
+) {
+ # Get the call parameters
+ call_benchmark <- match.call()
+
+ # Number of commits to test
+ n_commits <- length(commit_list)
+
+ # Test whether we have everything we need
+ if (is.null(cypress_dir) && is.null(shinytest2_dir))
+ stop("You must provide a cypress_dir or the shinytest2_dir")
+
+ if (!is.null(cypress_dir) && !is.null(shinytest2_dir)) {
+ message("Using the cypress file only")
+ shinytest2_dir <- NULL
+ }
+
+ type <- ifelse(!is.null(cypress_dir), "cypress", "shinytest2")
+ obj_name <- ifelse(type == "cypress", "cypress_dir", "shinytest2_dir")
+
+ if (length(get(obj_name)) == 1)
+ assign(obj_name, rep(get(obj_name), n_commits))
+ if (length(get(obj_name)) != n_commits)
+ stop(glue("You must provide 1 or {n_commits} paths for {obj_name}"))
+
+ if (is.null(tests_pattern))
+ tests_pattern <- vector(mode = "list", length = n_commits)
+ if (length(tests_pattern) == 1)
+ tests_pattern <- as.list(rep(tests_pattern, n_commits))
+
+ n_rep <- as.integer(n_rep)
+ if (n_rep < 1)
+ stop("You must provide an integer greater than 1 for n_rep")
+
+ # check if the repo is ready for running the checks
+ check_uncommitted_files()
+
+ # run tests
+ total_time <- system.time(
+ if (type == "cypress") {
+ perf_list <- benchmark_cypress(
+ commit_list = commit_list,
+ cypress_dir = cypress_dir,
+ tests_pattern = tests_pattern,
+ app_dir = app_dir,
+ port = port,
+ use_renv = use_renv,
+ renv_prompt = renv_prompt,
+ n_rep = n_rep,
+ debug = debug
+ )
+ } else {
+ perf_list <- benchmark_shinytest2(
+ commit_list,
+ shinytest2_dir,
+ tests_pattern = tests_pattern,
+ app_dir,
+ use_renv = use_renv,
+ renv_prompt = renv_prompt,
+ n_rep = n_rep,
+ debug = debug
+ )
+ }
+ )
+
+ out <- list(
+ call = call_benchmark,
+ time = total_time,
+ performance = perf_list
+ )
+ class(out) <- "shiny_benchmark"
+
+ return(out)
+}
diff --git a/R/benchmark_cypress.R b/R/benchmark_cypress.R
new file mode 100644
index 0000000..cee35c5
--- /dev/null
+++ b/R/benchmark_cypress.R
@@ -0,0 +1,148 @@
+#' @title Run the performance test based on multiple commits using Cypress
+#'
+#' @param commit_list A list of commit hash codes, branches' names or anything
+#' else you can use with git checkout [...]
+#' @param cypress_dir The directory with tests recorded by Cypress.
+#' It can also be a vector of the same size of commit_list
+#' @param tests_pattern Cypress/shinytest2 files pattern. E.g. 'shinytest2'
+#' It can also be a vector of the same size of commit_list. If it is NULL,
+#' all the content in cypress_dir/shinytest2_dir will be used
+#' @param app_dir The path to the application root
+#' @param port Port to run the app
+#' @param use_renv In case it is set as TRUE, package will try to apply
+#' renv::restore() in all branches. Otherwise, the current loaded list of
+#' packages will be used in all branches.
+#' @param renv_prompt Prompt the user before taking any action?
+#' @param n_rep Number of replications desired
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @export
+benchmark_cypress <- function(
+ commit_list,
+ cypress_dir,
+ tests_pattern,
+ app_dir,
+ port,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+) {
+ # creating the structure
+ project_path <- create_cypress_structure(
+ app_dir = app_dir,
+ port = port,
+ debug = debug
+ )
+
+ # getting the current branch
+ current_branch <- get_commit_hash()
+
+ # apply the tests for each branch/commit
+ perf_list <- tryCatch(
+ expr = {
+ mapply(
+ commit_list,
+ cypress_dir,
+ tests_pattern,
+ FUN = run_cypress_ptest,
+ project_path = project_path,
+ use_renv = use_renv,
+ renv_prompt = renv_prompt,
+ n_rep = n_rep,
+ debug = debug,
+ SIMPLIFY = FALSE
+ )
+ },
+ error = function(e) {
+ message(e)
+ },
+ finally = {
+ # Checkout to the main branch
+ checkout(branch = current_branch, debug = debug)
+ message(glue("Switched back to {current_branch}"))
+
+ # Restore renv
+ if (use_renv)
+ restore_env(branch = current_branch, renv_prompt = renv_prompt)
+
+ # Cleaning the temporary directory
+ fs::file_delete(fs::path(project_path, "node"))
+ fs::file_delete(fs::path(project_path, "tests"))
+ }
+ )
+
+ return(perf_list)
+}
+
+#' @title Run the performance test based on a single commit using Cypress
+#'
+#' @param commit A commit hash code or a branch's name
+#' @param project_path The path to the project with all needed packages
+#' installed
+#' @param cypress_dir The directory with tests recorded by Cypress
+#' @param tests_pattern Cypress files pattern. E.g. 'performance'. If it is NULL,
+#' all the content will be used
+#' @param use_renv In case it is set as TRUE, package will try to apply
+#' renv::restore() in all branches. Otherwise, the current loaded list of
+#' packages will be used in all branches.
+#' @param renv_prompt Prompt the user before taking any action?
+#' @param n_rep Number of replications desired
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @importFrom utils read.table
+#' @export
+run_cypress_ptest <- function(
+ commit,
+ project_path,
+ cypress_dir,
+ tests_pattern,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+) {
+ # checkout to the desired commit
+ checkout(branch = commit, debug = debug)
+ date <- get_commit_date(branch = commit)
+ message(glue("Switched to {commit}"))
+ if (use_renv) restore_env(branch = commit, renv_prompt = renv_prompt)
+
+ # get Cypress files
+ files <- create_cypress_tests(
+ project_path = project_path,
+ cypress_dir = cypress_dir,
+ tests_pattern = tests_pattern
+ )
+ js_file <- files$js_file
+ txt_file <- files$txt_file
+
+ # replicate tests
+ perf_file <- list()
+ pb <- create_progress_bar(total = n_rep)
+ for (i in 1:n_rep) {
+ # increment progress bar
+ pb$tick()
+
+ # run tests there
+ command <- performance_test_cmd(project_path)
+ system(command, ignore.stdout = !debug, ignore.stderr = !debug)
+
+ # read the file saved by cypress
+ perf_file[[i]] <- read.table(file = txt_file, header = FALSE, sep = ";")
+ perf_file[[i]] <- cbind.data.frame(date = date, rep_id = i, perf_file[[i]])
+ colnames(perf_file[[i]]) <- c("date", "rep_id", "test_name", "duration_ms")
+
+ # removing txt measures
+ fs::file_delete(txt_file)
+ }
+
+ # removing js tests
+ fs::file_delete(js_file)
+
+ # removing anything new in the github repo
+ checkout_files(debug = debug)
+
+ # return times
+ return(perf_file)
+}
diff --git a/R/benchmark_shinytest2.R b/R/benchmark_shinytest2.R
new file mode 100644
index 0000000..590d673
--- /dev/null
+++ b/R/benchmark_shinytest2.R
@@ -0,0 +1,149 @@
+#' @title Run the performance test based on a multiple commits using shinytest2
+#'
+#' @param commit_list A list of commit hash codes, branches' names or anything
+#' else you can use with git checkout [...]
+#' @param shinytest2_dir The directory with tests recorded by shinytest2
+#' It can also be a vector of the same size of commit_list
+#' @param tests_pattern shinytest2 files pattern. E.g. 'performance'
+#' It can also be a vector of the same size of commit_list. If it is NULL,
+#' all the content in cypress_dir/shinytest2_dir will be used
+#' @param app_dir The path to the application root
+#' @param use_renv In case it is set as TRUE, package will try to apply
+#' renv::restore() in all branches. Otherwise, the current loaded list of
+#' packages will be used in all branches.
+#' @param renv_prompt Prompt the user before taking any action?
+#' @param n_rep Number of replications desired
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @export
+benchmark_shinytest2 <- function(
+ commit_list,
+ shinytest2_dir,
+ tests_pattern,
+ app_dir,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+) {
+
+ # creating the structure
+ project_path <- create_shinytest2_structure(app_dir = app_dir)
+
+ # getting the current branch
+ current_branch <- get_commit_hash()
+
+ # apply the tests for each branch/commit
+ perf_list <- tryCatch(
+ expr = {
+ mapply(
+ commit_list,
+ shinytest2_dir,
+ tests_pattern,
+ FUN = run_shinytest2_ptest,
+ app_dir = app_dir,
+ project_path = project_path,
+ use_renv = use_renv,
+ renv_prompt = renv_prompt,
+ n_rep = n_rep,
+ debug = debug,
+ SIMPLIFY = FALSE
+ )
+ },
+ error = function(e) {
+ message(e)
+ },
+ finally = {
+ # Checkout to the main branch
+ checkout(branch = current_branch, debug = debug)
+ message(glue("Switched back to {current_branch}"))
+
+ # Restore renv
+ if (use_renv)
+ restore_env(branch = current_branch, renv_prompt = renv_prompt)
+
+ # Cleaning the temporary directory
+ # couldn't use fs::file_delete / fs::directory_delete as a process
+ # is accessing one of the files and it fails. unlink does not
+ unlink(fs::path(project_path, "tests"))
+ }
+ )
+
+ return(perf_list)
+}
+
+#' @title Run the performance test based on a single commit using shinytest2
+#'
+#' @param commit A commit hash code or a branch's name
+#' @param app_dir The path to the application root
+#' @param project_path The path to the project
+#' @param shinytest2_dir The directory with tests recorded by shinytest2
+#' @param tests_pattern shinytest2 files pattern. E.g. 'performance'. If it is NULL,
+#' all the content will be used
+#' @param use_renv In case it is set as TRUE, package will try to apply
+#' renv::restore() in all branches. Otherwise, the current loaded list of
+#' packages will be used in all branches.
+#' @param renv_prompt Prompt the user before taking any action?
+#' @param n_rep Number of replications desired
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @importFrom testthat ListReporter
+#' @importFrom shinytest2 test_app
+#' @export
+run_shinytest2_ptest <- function(
+ commit,
+ project_path,
+ app_dir,
+ shinytest2_dir,
+ tests_pattern,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+) {
+ # checkout to the desired commit
+ checkout(branch = commit, debug = debug)
+ date <- get_commit_date(branch = commit)
+ message(glue("Switched to {commit}"))
+ if (use_renv) restore_env(branch = commit, renv_prompt = renv_prompt)
+
+ # move test files to the project folder
+ tests_dir <- move_shinytest2_tests(
+ project_path = project_path,
+ shinytest2_dir = shinytest2_dir
+ )
+
+ perf_file <- list()
+ pb <- create_progress_bar(total = n_rep)
+ for (i in 1:n_rep) {
+ # increment progress bar
+ pb$tick()
+
+ # run tests there
+ my_reporter <- ListReporter$new()
+ test_app(
+ app_dir = dirname(tests_dir),
+ reporter = my_reporter,
+ stop_on_failure = FALSE,
+ stop_on_warning = FALSE,
+ filter = tests_pattern
+ )
+
+ perf_file[[i]] <- as.data.frame(my_reporter$get_results())
+ perf_file[[i]] <- perf_file[[i]][, c("test", "real")]
+ perf_file[[i]]$test <- gsub(
+ x = perf_file[[i]]$test,
+ pattern = "\\{shinytest2\\} recording: ",
+ replacement = ""
+ )
+
+ perf_file[[i]] <- cbind.data.frame(date = date, rep_id = i, perf_file[[i]])
+ colnames(perf_file[[i]]) <- c("date", "rep_id", "test_name", "duration_ms")
+ }
+
+ # removing anything new in the github repo
+ checkout_files(debug = debug)
+
+ # return times
+ return(perf_file)
+}
diff --git a/R/globals.R b/R/globals.R
new file mode 100644
index 0000000..06155fd
--- /dev/null
+++ b/R/globals.R
@@ -0,0 +1,14 @@
+utils::globalVariables(
+ c(
+ "commit",
+ "date",
+ "duration_ms",
+ "max",
+ "mean",
+ "min",
+ "n",
+ "sd",
+ "test_name",
+ "total_time"
+ )
+)
diff --git a/R/performance_tests.R b/R/performance_tests.R
deleted file mode 100644
index a4506b4..0000000
--- a/R/performance_tests.R
+++ /dev/null
@@ -1,91 +0,0 @@
-#' @title Execute performance tests for a list of commits
-#'
-#' @param commit_list A list of commit hash codes, branches' names or anything else you can use with git checkout [...]
-#' @param cypress_file The path to the .js file containing cypress tests to be recorded
-#' @param app_dir The path to the application root
-#' @param port Port to run the app
-#' @param debug Logical. TRUE to display all the system messages on runtime
-#'
-#' @importFrom git2r checkout
-#'
-#' @export
-performance_tests <- function(commit_list, cypress_file, app_dir = getwd(), port = 3333, debug = FALSE) {
- # getting the current branch
- current_branch <- get_commit_hash()
-
- # creating the structure
- project_path <- create_tests_structure(app_dir = app_dir, port = port, debug = debug)
-
- # copy the cypress test file from the current location and store it
- cypress_file_cp <- file.path(project_path, "cypress_tests.js")
- file.copy(from = cypress_file, to = cypress_file_cp)
-
- # apply the tests for each branch/commit
- perf_list <- tryCatch(
- expr = {
- lapply(
- X = commit_list,
- FUN = run_performance_test,
- project_path = project_path,
- cypress_file = cypress_file_cp,
- debug = debug
- )
- },
- error = function(e) {
- message(e)
- },
- finally = {
- checkout(branch = current_branch)
- message(glue("Switched back to {current_branch}"))
-
- # Cleaning the temporary directory
- unlink(
- x = c(
- file.path(project_path, "node"),
- file.path(project_path, "tests")
- ),
- recursive = TRUE
- )
- }
- )
-
- return(perf_list)
-}
-
-#' @title Run the performance test based on a single commit
-#'
-#' @param commit A commit hash code or a branch's name
-#' @param project_path The path to the project with all needed packages installed
-#' @param cypress_file The path to the .js file conteining cypress tests to be recorded
-#' @param txt_file The path to the file where it is aimed to save the times
-#' @param debug Logical. TRUE to display all the system messages on runtime
-#'
-#' @export
-run_performance_test <- function(commit, project_path, cypress_file, txt_file, debug) {
- files <- create_cypress_tests(project_path = project_path, cypress_file = cypress_file)
- js_file <- files$js_file
- txt_file <- files$txt_file
-
- # checkout to the desired commit
- checkout(branch = commit)
- date <- get_commit_date(branch = commit)
- message(glue("Switched to {commit}"))
-
- # run tests there
- command <- glue("cd {project_path}; set -eu; exec yarn --cwd node performance-test")
- system(command, ignore.stdout = !debug, ignore.stderr = !debug)
-
- # read the file saved by cypress
- perf_file <- read.table(file = txt_file, header = FALSE, sep = ";")
- perf_file <- cbind.data.frame(date = date, perf_file)
- colnames(perf_file) <- c("date", "test_name", "duration_ms")
-
- # removing temp files
- unlink(x = c(js_file, txt_file))
-
- # removing anything new in the github repo
- checkout_files()
-
- # return times
- return(perf_file)
-}
diff --git a/R/plot.R b/R/plot.R
new file mode 100644
index 0000000..53a990d
--- /dev/null
+++ b/R/plot.R
@@ -0,0 +1,36 @@
+#' Plot for shiny_benchmark class
+#'
+#' @param x shiny_benchmark object
+#' @param ... Other parameters
+#'
+#' @method plot shiny_benchmark
+#' @import dplyr
+#' @import ggplot2
+#' @importFrom utils globalVariables
+#' @export
+plot.shiny_benchmark <- function(x, ...) {
+ if (!requireNamespace(package = "ggplot2", quietly = TRUE))
+ stop("ggplot2 is missing. Please, consider intalling ggplot2.")
+
+ plot_df <- lapply(X = x$performance, FUN = bind_rows) %>%
+ bind_rows(.id = "commit") %>%
+ arrange(date) %>%
+ mutate(commit = factor(x = commit, levels = unique(commit))) %>%
+ group_by(commit, test_name) %>%
+ summarise(
+ min = min(duration_ms),
+ mean = mean(duration_ms),
+ max = max(duration_ms),
+ .groups = "keep"
+ ) %>%
+ ungroup()
+
+ g <- ggplot(data = plot_df, mapping = aes(x = commit, y = mean)) +
+ geom_pointrange(mapping = aes(ymin = min, ymax = max)) +
+ facet_wrap(~test_name) +
+ ylab("Duration (ms)") +
+ xlab("Commit") +
+ theme_bw()
+
+ return(g)
+}
diff --git a/R/print.R b/R/print.R
new file mode 100644
index 0000000..f42d9e8
--- /dev/null
+++ b/R/print.R
@@ -0,0 +1,21 @@
+#' Print for shiny_benchmark class
+#'
+#' @param x shiny_benchmark object
+#' @param ... Other parameters
+#'
+#' @method print shiny_benchmark
+#' @export
+print.shiny_benchmark <- function(x, ...) {
+ cat("Shiny benchmark: \n")
+ cat("\n")
+ cat("Call:")
+ cat("\n")
+ print(x$call)
+ cat("\n")
+ cat("Total time ellapsed:")
+ cat("\n")
+ print(x$time[["elapsed"]])
+ cat("\n")
+ cat("Fit measures: \n")
+ print(x$performance)
+}
diff --git a/R/shiny_benchmark-class.R b/R/shiny_benchmark-class.R
new file mode 100644
index 0000000..34bf8ee
--- /dev/null
+++ b/R/shiny_benchmark-class.R
@@ -0,0 +1,18 @@
+#' @title An object of 'shiny_benchmark' class
+#'
+#' @slot call Function call
+#' @slot time Time elapsed
+#' @slot performance List of measurements (one entry for each commit)
+#'
+#' @importFrom methods new
+#'
+#' @export
+
+shiny_benchmark_class <- setClass(
+ Class = "shiny_benchmark",
+ representation(
+ call = "call",
+ time = "proc_time",
+ performance = "list"
+ )
+)
diff --git a/R/summary.R b/R/summary.R
new file mode 100644
index 0000000..56b1acb
--- /dev/null
+++ b/R/summary.R
@@ -0,0 +1,16 @@
+#' Summary for shiny_benchmark class
+#'
+#' @param object shiny_benchmark object
+#' @param ... Other parameters
+#'
+#' @method summary shiny_benchmark
+#' @export
+summary.shiny_benchmark <- function(object, ...) {
+ if (!requireNamespace(package = "dplyr", quietly = TRUE))
+ stop("dplyr is missing. Please, consider intalling dplyr.")
+
+ summary_results <- lapply(X = object$performance, FUN = summarise_commit)
+ summary_results <- bind_rows(summary_results, .id = "commit")
+
+ return(summary_results)
+}
diff --git a/R/utils.R b/R/utils.R
index 72d5896..a15774c 100644
--- a/R/utils.R
+++ b/R/utils.R
@@ -1,173 +1,12 @@
-#' @title Create a temporary directory to store everything needed by Cypress
-#'
-#' @param app_dir The path to the application root
-#' @param port Port to run the app
-#' @param debug Logical. TRUE to display all the system messages on runtime
-#'
-#' @importFrom jsonlite write_json
-create_tests_structure <- function(app_dir, port, debug) {
- # temp dir to run the tests
- dir_cypress <- tempdir()
-
- # node path
- node_path <- file.path(dir_cypress, "node")
- root_path <- file.path(node_path, "root")
-
- # test path
- tests_path <- file.path(dir_cypress, "tests")
- cypress_path <- file.path(tests_path, "cypress")
- integration_path <- file.path(cypress_path, "integration")
- plugins_path <- file.path(cypress_path, "plugins")
-
- # creating paths
- dir.create(path = node_path, showWarnings = FALSE)
- dir.create(path = tests_path, showWarnings = FALSE)
- dir.create(path = cypress_path, showWarnings = FALSE)
- dir.create(path = integration_path, showWarnings = FALSE)
- dir.create(path = plugins_path, showWarnings = FALSE)
-
- # create a path root linked to the main directory app
- symlink_cmd <- glue("cd {dir_cypress}; ln -s {app_dir} {root_path}")
- system(symlink_cmd)
-
- # create the packages.json file
- json_txt <- create_node_list(tests_path = tests_path, port = port)
- json_file <- file.path(node_path, "package.json")
- write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE)
-
- # install everything that is needed
- install_deps <- glue("yarn --cwd {node_path}")
- system(install_deps, ignore.stdout = !debug, ignore.stderr = !debug)
-
- # creating cypress plugin file
- js_txt <- create_cypress_plugins()
- js_file <- file.path(plugins_path, "index.js")
- writeLines(text = js_txt, con = js_file)
-
- # creating cypress.json
- json_txt <- create_cypress_list(plugins_file = js_file, port = port)
- json_file <- file.path(tests_path, "cypress.json")
- write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE)
-
- # returning the project folder
- message(glue("Structure created at {dir_cypress}"))
-
- return(dir_cypress)
-}
-
-#' @title Create the list of needed libraries
-#'
-#' @param tests_path The path to project
-create_node_list <- function(tests_path, port) {
- json_list <- list(
- private = TRUE,
- scripts = list(
- "performance-test" = glue("start-server-and-test run-app http://localhost:{port} run-cypress"),
- "run-app" = glue("cd root && Rscript -e 'shiny::runApp(port = {port})'"),
- "run-cypress" = glue("cypress run --project {tests_path}")
- ),
- "devDependencies" = list(
- "cypress" = "^7.6.0",
- "start-server-and-test" = "^1.12.6"
- )
- )
-
- return(json_list)
-}
-
-#' @title Create the cypress configuration list
-#'
-#' @param plugins_file The path to the Cypress plugins
-create_cypress_list <- function(plugins_file, port) {
- json_list <- list(
- baseUrl = glue("http://localhost:{port}"),
- pluginsFile = plugins_file,
- supportFile = FALSE
- )
-
- return(json_list)
-}
-
-#' @title Create the JS code to track execution time
-create_cypress_plugins <- function() {
- js_txt <- "
- const fs = require('fs')
- module.exports = (on, config) => {
- on('task', {
- performanceTimes (attributes) {
- fs.writeFile(attributes.fileOut, `${ attributes.title }; ${ attributes.duration }\n`, { flag: 'a' })
- return null
- }
- })
- }"
-
- return(js_txt)
-}
-
-#' @title Create the cypress files under project directory
-#'
-#' @param project_path The path to the project with all needed packages installed
-#' @param cypress_file The path to the .js file conteining cypress tests to be recorded
-create_cypress_tests <- function(project_path, cypress_file) {
- # creating a copy to be able to edit the js file
- js_file <- file.path(project_path, "tests", "cypress", "integration", "app.spec.js")
- file.copy(from = cypress_file, to = js_file, overwrite = TRUE)
-
- # file to store the times
- txt_file <- file.path(project_path, "tests", "cypress", "performance.txt")
- add_sendtime2js(js_file = js_file, txt_file = txt_file)
-
- # returning the file location
- return(list(js_file = js_file, txt_file = txt_file))
-}
-
-#' @title Add the sendTime function to the .js file
-#'
-#' @param js_file Path to the .js file to add code
-#' @param txt_file Path to the file to record the execution times
-add_sendtime2js <- function(js_file, txt_file) {
- lines_to_add <- glue(
- "
- // Returning the time for each test
- // https://www.cypress.io/blog/2020/05/22/where-does-the-test-spend-its-time/
- let commands = []
- let performanceAttrs
- Cypress.on('test:before:run', () => {
- commands.length = 0
- })
- Cypress.on('test:after:run', (attributes) => {
- performanceAttrs = {
- title: attributes.title,
- duration: attributes.duration,
- commands: Cypress._.cloneDeep(commands),
- }
- })
- const sendTestTimings = () => {
- if (!performanceAttrs) {
- return
- }
- const attr = performanceAttrs
- attr.fileOut = '{{txt_file}}'
- performanceAttrs = null
- cy.task('performanceTimes', attr)
- }
- // Calling the sendTestTimings function
- beforeEach(sendTestTimings)
- after(sendTestTimings)
- ",
- .open = "{{", .close = "}}"
- )
-
- write(x = lines_to_add, file = js_file, append = TRUE)
-}
-
#' @title Get the commit date in POSIXct format
#'
#' @param branch Commit hash code or branch name
#' @importFrom glue glue
+#'
+#' @keywords internal
get_commit_date <- function(branch) {
date <- system(
- glue("git show -s --format=%ci {branch}"),
+ command = glue("git show -s --format=%ci {branch}"),
intern = TRUE
)
date <- as.POSIXct(date[1])
@@ -176,22 +15,30 @@ get_commit_date <- function(branch) {
}
#' @title Find the hash code of the current commit
+#'
#' @importFrom glue glue
#' @importFrom stringr str_trim
+#'
+#' @keywords internal
get_commit_hash <- function() {
- hash <- system("git show -s --format=%H", intern = TRUE)[1]
+ hash <- system(command = "git show -s --format=%H", intern = TRUE)[1]
+
branch <- system(
- glue("git branch --contains {hash}"),
+ command = glue("git branch --contains {hash}"),
intern = TRUE
)
branch <- str_trim(
- string = gsub(x = branch[length(branch)], pattern = "\\*\\s", replacement = ""),
+ string = gsub(
+ x = branch[length(branch)],
+ pattern = "\\*\\s",
+ replacement = ""
+ ),
side = "both"
)
hash_head <- system(
- glue("git rev-parse {branch}"),
+ command = glue("git rev-parse {branch}"),
intern = TRUE
)
@@ -204,8 +51,181 @@ get_commit_hash <- function() {
#' @title Checkout GitHub files
#'
-#' @description checkout anything created by the app. It prevents errors when
+#' @description Checkout anything created by the app. It prevents errors when
#' changing branches
-checkout_files <- function() {
- system("git checkout .")
+#'
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @keywords internal
+checkout_files <- function(debug) {
+ system(
+ command = "git checkout .",
+ ignore.stdout = !debug,
+ ignore.stderr = !debug
+ )
+}
+
+#' @title Checkout GitHub branch
+#'
+#' @description checkout and go to a different branch
+#'
+#' @param branch Commit hash code or branch name
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @keywords internal
+checkout <- function(branch, debug) {
+ system(
+ command = glue("git checkout {branch}"),
+ ignore.stdout = !debug,
+ ignore.stderr = !debug
+ )
+}
+
+#' @title Running the node script "performance_test" is system-dependent
+#'
+#' @param project_path path to project directory (one level above node)
+#'
+#' @keywords internal
+performance_test_cmd <- function(project_path) {
+ glue("yarn --cwd \"{fs::path(project_path, 'node')}\" performance-test")
+}
+
+#' @title Check for uncommitted files
+#'
+#' @keywords internal
+check_uncommitted_files <- function() {
+ changes <- system("git status --porcelain", intern = TRUE)
+
+ if (length(changes) != 0) {
+ system("git status -u")
+ stop("You have uncommitted files. Please resolve it before running the performance checks.")
+ } else {
+ return(invisible(TRUE))
+ }
+}
+
+#' @title Check and restore renv
+#'
+#' @description Check whether renv is in use in the current branch. Raise error
+#' if renv is not in use or apply renv:restore() in the case the package is
+#' present
+#'
+#' @param branch Commit hash code or branch name. Useful to create an
+#' informative error message
+#' @param renv_prompt Prompt the user before taking any action?
+#' @importFrom glue glue
+#' @importFrom renv activate restore
+#'
+#' @keywords internal
+restore_env <- function(branch, renv_prompt) {
+ # handling renv
+ tryCatch(
+ expr = {
+ activate()
+ restore(prompt = renv_prompt)
+ },
+ error = function(e) {
+ stop(glue("Unexpected error activating renv in branch {branch}: {e}\n"))
+ }
+ )
+}
+
+#' @title Create a progress bar to follow the execution
+#'
+#' @param total Total number of replications
+#' @importFrom progress progress_bar
+#'
+#' @keywords internal
+create_progress_bar <- function(total = 100) {
+ pb <- progress_bar$new(
+ format = "Iteration :current/:total",
+ total = total,
+ clear = FALSE
+ )
+
+ return(pb)
+}
+
+#' @title Return statistics based on the set of tests replications
+#'
+#' @param object A shiny_benchmark object
+#'
+#' @import dplyr
+#' @importFrom stats median
+#'
+#' @keywords internal
+summarise_commit <- function(object) {
+ out <- bind_rows(object) %>%
+ group_by(test_name) %>%
+ summarise(
+ n = n(),
+ mean = mean(duration_ms),
+ median = median(duration_ms),
+ sd = sd(duration_ms),
+ min = min(duration_ms),
+ max = max(duration_ms)
+ )
+
+ return(out)
+}
+
+#' @title Load an application and instructions to run shiny.benchmark
+#' @description This function aims to generate a template to be used
+#' by shiny.benchmark. It will create the necessary structure on `path` with
+#' some examples of tests using Cypress and shinytest2. Also, a simple
+#' application will be added to the folder as well as instructions on how
+#' to perform the performance checks. Be aware that a new git repo is need in
+#' the selected `path`.
+#'
+#' @param path A character vector of full path name
+#' @param force Create example even if directory does not exist or is not empty
+#'
+#' @importFrom glue glue
+#' @importFrom utils menu
+#' @export
+#' @examples
+#' load_example(file.path(tempdir(), "example_destination"), force = TRUE)
+load_example <- function(path, force = FALSE) {
+ # see if path exists
+ if (!force && !fs::file_exists(path))
+ stop("You must provide a valid path")
+ else if (!fs::file_exists(path)) {
+ fs::dir_create(path, recurse = TRUE)
+ }
+
+ if (!force && length(fs::dir_ls(path))) {
+ choice <- menu(
+ choices = c("Yes", "No"),
+ title = glue("{path} seems to not be empty. Would you like to proceed?")
+ )
+
+ if (choice == 2)
+ stop("Process aborted by user. Consider creating a new empty path.")
+ } else if (length(fs::dir_ls(path))) {
+ message(glue(
+ "{path} seems to not be empty. ",
+ "Continuing as parameter `force = TRUE`"
+ ))
+ }
+
+ ex_path <- system.file(
+ "examples",
+ package = "shiny.benchmark",
+ mustWork = TRUE
+ )
+ files <- fs::dir_ls(path = ex_path, fun = fs::path_real)
+
+ for (file in files) {
+ if (fs::is_dir(file)) {
+ # Due to overwrite = TRUE the destination must include the name of the
+ # directory to be created
+ fs::dir_copy(file, fs::path(path, fs::path_file(file)), overwrite = TRUE)
+ } else {
+ fs::file_copy(file, path, overwrite = TRUE)
+ }
+ print(glue("{basename(file)} created at {path}"))
+ }
+
+ fpath <- fs::path(path, "run_tests.R") # nolint
+ message(glue("Follow instructions in {fpath}"))
}
diff --git a/R/utils_cypress.R b/R/utils_cypress.R
new file mode 100644
index 0000000..7533d05
--- /dev/null
+++ b/R/utils_cypress.R
@@ -0,0 +1,235 @@
+#' @title Create a temporary directory to store everything needed by Cypress
+#'
+#' @param app_dir The path to the application root
+#' @param port Port to run the app
+#' @param debug Logical. TRUE to display all the system messages on runtime
+#'
+#' @importFrom jsonlite write_json
+#'
+#' @keywords internal
+create_cypress_structure <- function(app_dir, port, debug) {
+ # temp dir to run the tests
+ dir_tests <- tempdir()
+
+ # node path
+ node_path <- fs::path(dir_tests, "node")
+ root_path <- fs::path(node_path, "root") # nolint
+
+ # test path
+ tests_path <- fs::path(dir_tests, "tests")
+ cypress_path <- fs::path(tests_path, "cypress")
+ integration_path <- fs::path(cypress_path, "integration")
+ plugins_path <- fs::path(cypress_path, "plugins")
+
+ # creating paths
+ fs::dir_create(path = node_path)
+ fs::dir_create(path = tests_path)
+ fs::dir_create(path = cypress_path)
+ fs::dir_create(path = integration_path)
+ fs::dir_create(path = plugins_path)
+
+ # create a path root linked to the main directory app
+ tryCatch(
+ expr = {
+ fs::link_create(app_dir, root_path, symbolic = TRUE)
+ },
+ error = function(e) {
+
+ choice <- menu(
+ choices = c("Yes", "No"),
+ title = glue(
+ "A symbolic link cannot be created, it is possible to clone ",
+ "the repository, but it can take some time and space on disk. ",
+ "Would you like to proceed with this operations?")
+ )
+
+ if (choice == 2)
+ stop("Process aborted by user.")
+
+ # If system cannot symlink then try to clone the repository
+ # This may happen on some windows versions
+ # This can be an expensive operation on big repositories
+ message(
+ "Could not create symbolic link with fs package, ",
+ "trying with git clone..."
+ )
+ system(glue::glue("git clone \"{app_dir}\" \"{root_path}\""))
+ system("git submodule init")
+ system("git submodule update ")
+ })
+
+ # create the packages.json file
+ json_txt <- create_node_list(tests_path = tests_path, port = port)
+ json_file <- fs::path(node_path, "package.json")
+ write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE)
+
+ # install everything that is needed
+ install_deps <- glue("yarn --cwd {node_path}")
+ system(install_deps, ignore.stdout = !debug, ignore.stderr = !debug)
+
+ # creating cypress plugin file
+ js_txt <- create_cypress_plugins()
+ js_file <- fs::path(plugins_path, "index.js")
+ writeLines(text = js_txt, con = js_file)
+
+ # creating cypress.json
+ json_txt <- create_cypress_list(plugins_file = js_file, port = port)
+ json_file <- fs::path(tests_path, "cypress.json")
+ write_json(x = json_txt, path = json_file, pretty = TRUE, auto_unbox = TRUE)
+
+ # returning the project folder
+ message(glue("Structure created at {dir_tests}"))
+ return(dir_tests)
+}
+
+#' @title Create the list of needed libraries
+#'
+#' @param tests_path The path to project
+#' @param port Port to run the app
+#'
+#' @keywords internal
+create_node_list <- function(tests_path, port) {
+ json_list <- list(
+ private = TRUE,
+ scripts = list(
+ "performance-test" = glue(
+ "start-server-and-test run-app http://localhost:{port} run-cypress"
+ ),
+ "run-app" = glue(
+ "cd root && ",
+ "Rscript -e \"shiny::runApp(port = {port})\""
+ ),
+ "run-cypress" = glue("cypress run --project {tests_path}")
+ ),
+ "devDependencies" = list(
+ "cypress" = "^7.6.0",
+ "start-server-and-test" = "^1.12.6"
+ )
+ )
+
+ return(json_list)
+}
+
+#' @title Create the cypress configuration list
+#'
+#' @param plugins_file The path to the Cypress plugins
+#' @param port Port to run the app
+#'
+#' @keywords internal
+create_cypress_list <- function(plugins_file, port) {
+ json_list <- list(
+ baseUrl = glue("http://localhost:{port}"),
+ pluginsFile = plugins_file,
+ supportFile = FALSE
+ )
+
+ return(json_list)
+}
+
+#' @title Create the JS code to track execution time
+#'
+#' @keywords internal
+create_cypress_plugins <- function() {
+ js_txt <- "
+ const fs = require('fs')
+ module.exports = (on, config) => {
+ on('task', {
+ performanceTimes (attributes) {
+ fs.writeFile(
+ attributes.fileOut,
+ `${ attributes.title }; ${ attributes.duration }\n`,
+ { flag: 'a' }
+ )
+ return null
+ }
+ })
+ }"
+
+ return(js_txt)
+}
+
+#' @title Create the cypress files under project directory
+#'
+#' @param project_path The path to the project with all needed packages
+#' installed
+#' @param cypress_dir The directory with tests recorded by Cypress
+#' @param tests_pattern Cypress files pattern. E.g. 'performance'. If it is NULL,
+#' all the content will be used
+#'
+#' @keywords internal
+create_cypress_tests <- function(project_path, cypress_dir, tests_pattern) {
+ # locate files
+ cypress_files <- list.files(
+ path = cypress_dir,
+ pattern = tests_pattern,
+ full.names = TRUE,
+ recursive = TRUE
+ )
+ cypress_files <- grep(x = cypress_files, pattern = "\\.js$", value = TRUE)
+
+ # creating a copy to be able to edit the js file
+ js_file <- fs::path(
+ project_path,
+ "tests",
+ "cypress",
+ "integration",
+ "app.spec.js"
+ )
+
+ # combine all files into one
+ for (i in seq_along(cypress_files)) {
+ text <- readLines(con = cypress_files[i])
+ write(x = text, file = js_file, append = TRUE)
+ }
+
+ # file to store the times
+ txt_file <- fs::path(project_path, "tests", "cypress", "performance.txt")
+
+ add_sendtime2js(js_file = js_file, txt_file = txt_file)
+
+ # returning the file location
+ return(list(js_file = js_file, txt_file = txt_file))
+}
+
+#' @title Add the sendTime function to the .js file
+#'
+#' @param js_file Path to the .js file to add code
+#' @param txt_file Path to the file to record the execution times
+#'
+#' @keywords internal
+add_sendtime2js <- function(js_file, txt_file) {
+ lines_to_add <- glue(
+ "
+ describe('Finalizing tests', () => {it('Ending tests', () => {})})
+
+ // Returning the time for each test
+ // https://www.cypress.io/blog/2020/05/22/where-does-the-test-spend-its-time/
+ let commands = []
+ let performanceAttrs
+ Cypress.on('test:before:run', () => {
+ commands.length = 0
+ })
+ Cypress.on('test:after:run', (attributes) => {
+ performanceAttrs = {
+ title: attributes.title,
+ duration: attributes.duration,
+ commands: Cypress._.cloneDeep(commands),
+ }
+ })
+ const sendTestTimings = () => {
+ if (!performanceAttrs) {
+ return
+ }
+ const attr = performanceAttrs
+ attr.fileOut = '{{txt_file}}'
+ performanceAttrs = null
+ cy.task('performanceTimes', attr)
+ }
+ // Calling the sendTestTimings function
+ beforeEach(sendTestTimings)
+ ",
+ .open = "{{", .close = "}}"
+ )
+
+ write(x = lines_to_add, file = js_file, append = TRUE)
+}
diff --git a/R/utils_shinytest2.R b/R/utils_shinytest2.R
new file mode 100644
index 0000000..eaeadd8
--- /dev/null
+++ b/R/utils_shinytest2.R
@@ -0,0 +1,36 @@
+#' @title Create a temporary directory to store everything needed by shinytest2
+#'
+#' @param app_dir The path to the application root
+#'
+#' @importFrom glue glue
+#'
+#' @keywords internal
+create_shinytest2_structure <- function(app_dir) {
+ # temp dir to run the tests
+ dir_tests <- tempdir()
+
+ # shiny call
+ writeLines(
+ text = glue('shiny::runApp(appDir = "{app_dir}")'),
+ con = fs::path(dir_tests, "app.R")
+ )
+
+ # returning the project folder
+ message(glue("Structure created at {dir_tests}"))
+
+ return(dir_tests)
+}
+
+#' @title Move tests to a temporary folder
+#'
+#' @param project_path The path to the project
+#' @param shinytest2_dir The directory with tests recorded by shinytest2
+#'
+#' @keywords internal
+move_shinytest2_tests <- function(project_path, shinytest2_dir) {
+ # copy everything to the temporary directory
+ file.copy(from = shinytest2_dir, to = project_path, recursive = TRUE)
+ tests_dir <- file.path(project_path, "tests")
+
+ return(tests_dir)
+}
diff --git a/README.md b/README.md
index 4009b6c..c976450 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,175 @@
-# shiny.performance
-Tools to measure performance improvements in shiny apps
+# shiny.benchmark
+
+> _Tools to measure performance improvements in Shiny apps._
+
+
+[![CRAN status](https://www.r-pkg.org/badges/version/shiny.benchmark)](https://cran.r-project.org/package=shiny.benchmark)
+[![R-CMD-check](https://github.com/Appsilon/shiny.benchmark/workflows/R-CMD-check/badge.svg)](https://github.com/Appsilon/shiny.benchmark/actions?workflow=R-CMD-check)
+
+
+`shiny.benchmark` is a tool aimed to measure and compare the performance of different versions of a `shiny` application. Based on a list of different application versions, accessible by a git repo by its refs (commit hash or branch name), the user can write instructions to be executed using Cypress or `shinytest2`. These instructions are then evaluated by the different versions of your `shiny` application and therefore the performance's improvement/deterioration (time elapsed) are be recorded.
+
+The package is flexible enough to allow different sets of tests for the different refs as well as different package versions (via `renv`). Also, the user can replicate the tests to have more accurate measures of performance.
+
+## How to install?
+
+```r
+remotes::install_github("Appsilon/shiny.benchmark")
+```
+
+## Dependencies
+
+`shiny.benchmark` can use two different engines to test the change in the performance of your application: [shinytest2](https://rstudio.github.io/shinytest2/) and [Cypress](https://www.cypress.io/).
+The latter requires `Node` (version 12 or higher) and `yarn` (version 1.22.17 or higher) to be available.
+To install them on your computer, follow the guidelines on the documentation pages:
+
+- [Node](https://nodejs.org/en/download/)
+- [yarn](https://yarnpkg.com/getting-started/install)
+
+Besides that, on Linux, it might be required to install other `Cypress` dependencies.
+Check the [documentation](https://docs.cypress.io/guides/getting-started/installing-cypress#Linux-Prerequisites) to find out more.
+
+## How to use it?
+
+The best way to start using `shiny.benchmark` is through an example. If you want a start point, you can use the `load_example` function. In order to use this, create a new folder in your computer and use the following code to generate an application to serve us as example for our performance checks:
+
+```r
+library(shiny.benchmark)
+
+load_example(path = "path/to/new/project")
+```
+
+It will create some useful files under `path/to/new/project`. The most important one is the `run_tests.R` which provides several instructions at the very top.
+
+As we are comparing versions of the same application, we need different app versions in different branches/commits in `git`. Start using `cd app; git init` to initiate git inside `app/` folder.
+
+Get familiar with `app/server.R` file in order to generate more interesting scenarios. The basic idea is to use the `Sys.sleep` function to simulate some app's functionalities. Remember that, when running the benchmark, that is the amount of time it will take to measure the performance.
+
+When you are ready, commit your changes in master/main using `git add .; git commit -m "your commit message"`. Make some editions and commit these new changes into a new branch or in the same branch your are testing (it will have a different commit hash). Repeat the process adding as many new modifications as you want. E.g. add renv, add more tests, change the names of the tests/test files and so on.
+
+Here is a complete example on how to setup your `git`:
+
+```git
+# starting
+git init
+echo .Rproj.user >> .gitignore
+echo *.Rproj >> .gitignore
+echo .Rprofile >> .gitignore
+echo renv >> .gitignore
+echo .Rprofile >> .gitignore
+
+# master
+git add .
+git commit -m "first commit"
+
+# develop (decrease Sys.sleep times in server.R)
+git checkout -b develop
+git add .
+git commit -m "improving performance"
+
+## Using renv
+git branch renv_shiny1 develop
+git checkout renv_shiny1
+R -e 'renv::init()'
+git add .
+git commit -m "renv active"
+
+## Downgrading shiny
+git checkout -b renv_shiny2
+R -e 'renv::install("shiny@1.7.0")'
+R -e 'renv::snapshot()'
+git add .
+git commit -m "downgrading shiny"
+
+## Switching back to develop
+git checkout develop
+```
+
+Now you are ready to go. The `benchmark` function provides several arguments to make your life easier when running your performance checks. The mandatory arguments are:
+
+- `commit_list`: A vector with commits, branches or anything else you can use in `git checkout`
+- `cypress_dir` or `shinytest2_dir`: Folder containing the tests we want to check the performance. In our case it is `tests/cypress` and `tests` respectively.
+
+The default behavior is to try to use `renv` in your project. If you do not have the renv structure, you can turn `renv` off using `use_renv = FALSE`
+
+```r
+library(shiny.benchmark)
+
+# commits to compare
+commit_list <- c("develop", "renv_shiny1", "renv_shiny2")
+
+# run performance check using Cypress
+benchmark(
+ commit_list = commit_list,
+ cypress_dir = "tests/cypress"
+)
+```
+
+That is all you need to run your `Cypress` tests. If you don't use `Cypress`, you may want to use `shinytest2` instead:
+
+```r
+benchmark(
+ commit_list = commit_list,
+ shinytest2_dir = "tests"
+)
+```
+
+To run just specific tests, you can take advantage of the `tests_pattern` argument. It will filter the test file's names based on regular expression:
+
+```r
+benchmark(
+ commit_list = commit_list,
+ shinytest2_dir = "tests",
+ tests_pattern = "use_this_one_[0-9]"
+)
+```
+
+If your project has `renv` structure, you can set `use_renv` to `TRUE` to guarantee that, for each application version your are using the correct packages. If you want to approve/reprove `renv::restore()`, you can set `renv_prompt = TRUE`.
+
+```r
+benchmark(
+ commit_list = commit_list,
+ shinytest2_dir = "tests",
+ tests_pattern = "use_this_one_[0-9]",
+ use_renv = TRUE, # default
+ renv_prompt = TRUE
+)
+```
+
+To have more accurate information about the time your application takes to perform some actions, you may need to replicate the tests. In this case, you can use the `n_rep` argument:
+
+```r
+out <- benchmark(
+ commit_list = commit_list,
+ cypress_dir = "tests/cypress",
+ tests_pattern = "use_this_one_[0-9]",
+ use_renv = FALSE,
+ n_rep = 15
+)
+
+out
+```
+
+For fast information about the tests results, you can use the `summary` and also the `plot` methods:
+
+```r
+summary(out)
+plot(out)
+```
+
+## How to contribute?
+
+If you want to contribute to this project please submit a regular PR, once you're done with new feature or bug fix.
+
+Reporting a bug is also helpful - please use [GitHub issues](https://github.com/Appsilon/shiny.benchmark/issues) and describe your problem as detailed as possible.
+
+## Appsilon
+
+
+
+Appsilon is a **Posit (formerly RStudio) Full Service Certified Partner**. Learn more
+at [appsilon.com](https://appsilon.com).
+
+Get in touch [opensource@appsilon.com](mailto:opensource@appsilon.com)
+
+
diff --git a/inst/WORDLIST b/inst/WORDLIST
new file mode 100644
index 0000000..6cab662
--- /dev/null
+++ b/inst/WORDLIST
@@ -0,0 +1,13 @@
+codecov
+CMD
+JS
+POSIXct
+Posit
+RStudio
+appsilon
+dir
+js
+renv
+repo
+sendTime
+shinytest
diff --git a/inst/examples/app/global.R b/inst/examples/app/global.R
new file mode 100644
index 0000000..5a0cab8
--- /dev/null
+++ b/inst/examples/app/global.R
@@ -0,0 +1 @@
+library(shiny)
diff --git a/inst/examples/app/server.R b/inst/examples/app/server.R
new file mode 100644
index 0000000..b843f12
--- /dev/null
+++ b/inst/examples/app/server.R
@@ -0,0 +1,39 @@
+function(input, output, session) {
+ # Sys.sleep
+ react1 <- eventReactive(input$run1, {
+ out <- system.time(
+ Sys.sleep(1 + rexp(n = 1, rate = 10))
+ )
+
+ return(out[3])
+ })
+
+ react2 <- eventReactive(input$run2, {
+ out <- system.time(
+ Sys.sleep(0.5 + rexp(n = 1, rate = 10))
+ )
+
+ return(out[3])
+ })
+
+ react3 <- eventReactive(input$run3, {
+ out <- system.time(
+ Sys.sleep(0.1 + rexp(n = 1, rate = 10))
+ )
+
+ return(out[1])
+ })
+
+ # outputs
+ output$out1 <- renderUI({
+ tags$span(round(react1()), style = "font-size: 500px;")
+ })
+
+ output$out2 <- renderUI({
+ tags$span(round(react2()), style = "font-size: 500px;")
+ })
+
+ output$out3 <- renderUI({
+ tags$span(round(react3()), style = "font-size: 500px;")
+ })
+}
diff --git a/inst/examples/app/tests/cypress/cypress_use_this_one_1.js b/inst/examples/app/tests/cypress/cypress_use_this_one_1.js
new file mode 100644
index 0000000..015b865
--- /dev/null
+++ b/inst/examples/app/tests/cypress/cypress_use_this_one_1.js
@@ -0,0 +1,19 @@
+describe('Cypress test', () => {
+ it('Out1 time elapsed - 1', () => {
+ cy.visit('/');
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed - 1', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed - 1', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+});
diff --git a/inst/examples/app/tests/cypress/cypress_use_this_one_2.js b/inst/examples/app/tests/cypress/cypress_use_this_one_2.js
new file mode 100644
index 0000000..c01f42e
--- /dev/null
+++ b/inst/examples/app/tests/cypress/cypress_use_this_one_2.js
@@ -0,0 +1,19 @@
+describe('Cypress test', () => {
+ it('Out1 time elapsed - 2', () => {
+ cy.visit('/');
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed - 2', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed - 2', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+});
diff --git a/inst/examples/app/tests/testthat.R b/inst/examples/app/tests/testthat.R
new file mode 100644
index 0000000..7d25b5b
--- /dev/null
+++ b/inst/examples/app/tests/testthat.R
@@ -0,0 +1 @@
+shinytest2::test_app()
diff --git a/inst/examples/app/tests/testthat/setup.R b/inst/examples/app/tests/testthat/setup.R
new file mode 100644
index 0000000..be65b4f
--- /dev/null
+++ b/inst/examples/app/tests/testthat/setup.R
@@ -0,0 +1,2 @@
+# Load application support files into testing environment
+shinytest2::load_app_env()
diff --git a/inst/examples/app/tests/testthat/test-use_this_one_1.R b/inst/examples/app/tests/testthat/test-use_this_one_1.R
new file mode 100644
index 0000000..a530c0a
--- /dev/null
+++ b/inst/examples/app/tests/testthat/test-use_this_one_1.R
@@ -0,0 +1,19 @@
+library(shinytest2)
+
+test_that("{shinytest2} recording: test1", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+test_that("{shinytest2} recording: test2", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+test_that("{shinytest2} recording: test3", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
diff --git a/inst/examples/app/tests/testthat/test-use_this_one_2.R b/inst/examples/app/tests/testthat/test-use_this_one_2.R
new file mode 100644
index 0000000..f2fc5c6
--- /dev/null
+++ b/inst/examples/app/tests/testthat/test-use_this_one_2.R
@@ -0,0 +1,19 @@
+library(shinytest2)
+
+test_that("{shinytest2} recording: test4", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+test_that("{shinytest2} recording: test5", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+test_that("{shinytest2} recording: test6", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
diff --git a/inst/examples/app/ui.R b/inst/examples/app/ui.R
new file mode 100644
index 0000000..8d3615a
--- /dev/null
+++ b/inst/examples/app/ui.R
@@ -0,0 +1,20 @@
+function() {
+ bootstrapPage(
+ tags$h1("Measuring time in different commits"),
+ column(
+ width = 4,
+ actionButton(inputId = "run1", label = "Run 1"),
+ uiOutput(outputId = "out1")
+ ),
+ column(
+ width = 4,
+ actionButton(inputId = "run2", label = "Run 2"),
+ uiOutput(outputId = "out2")
+ ),
+ column(
+ width = 4,
+ actionButton(inputId = "run3", label = "Run 3"),
+ uiOutput(outputId = "out3")
+ )
+ )
+}
diff --git a/inst/examples/run_tests.R b/inst/examples/run_tests.R
new file mode 100644
index 0000000..568da2e
--- /dev/null
+++ b/inst/examples/run_tests.R
@@ -0,0 +1,65 @@
+###############################################################################
+# Start a git repo under app/ folder and create some branches. It can be more #
+# fun if you change the Sys.sleep time in app/server.R #
+# #
+# suggestion: #
+# git init #
+# #
+# main #
+# git add . #
+# git commit -m "first commit" #
+# #
+# # develop #
+# git checkout -b develop #
+# git commit --allow-empty -m "dummy commit to change hash" #
+# #
+# # feature #
+# git checkout -b feature #
+# git commit --allow-empty -m "dummy commit to change hash" #
+# #
+# For a more complete example see: #
+# https://github.com/Appsilon/shiny.benchmark #
+###############################################################################
+
+# packages
+library(shiny.benchmark)
+
+# commits to compare
+type <- "cypress"
+commit_list <- c("develop", "feature")
+dir <- "tests/cypress"
+pattern <- "use_this_one_[0-9]"
+use_renv <- FALSE
+n_rep <- 5
+
+if (type == "cypress") {
+ # run performance check using Cypress
+ out <- benchmark(
+ commit_list = commit_list,
+ cypress_dir = dir,
+ tests_pattern = pattern,
+ app_dir = getwd(),
+ use_renv = use_renv,
+ renv_prompt = TRUE,
+ port = 3333,
+ n_rep = n_rep,
+ debug = FALSE
+ )
+} else {
+ # run performance check using shinytest2
+ out <- benchmark(
+ commit_list = commit_list,
+ shinytest2_dir = dir,
+ tests_pattern = pattern,
+ app_dir = getwd(),
+ use_renv = use_renv,
+ renv_prompt = TRUE,
+ port = 3333,
+ n_rep = n_rep,
+ debug = FALSE
+ )
+}
+
+out
+summary(out)
+plot(out)
diff --git a/man/add_sendtime2js.Rd b/man/add_sendtime2js.Rd
index c86b595..f5ce1f8 100644
--- a/man/add_sendtime2js.Rd
+++ b/man/add_sendtime2js.Rd
@@ -1,5 +1,5 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/utils.R
+% Please edit documentation in R/utils_cypress.R
\name{add_sendtime2js}
\alias{add_sendtime2js}
\title{Add the sendTime function to the .js file}
@@ -14,3 +14,4 @@ add_sendtime2js(js_file, txt_file)
\description{
Add the sendTime function to the .js file
}
+\keyword{internal}
diff --git a/man/benchmark.Rd b/man/benchmark.Rd
new file mode 100644
index 0000000..b21b421
--- /dev/null
+++ b/man/benchmark.Rd
@@ -0,0 +1,50 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/benchmark.R
+\name{benchmark}
+\alias{benchmark}
+\title{Execute performance tests for a list of commits}
+\usage{
+benchmark(
+ commit_list,
+ cypress_dir = NULL,
+ shinytest2_dir = NULL,
+ tests_pattern = NULL,
+ app_dir = getwd(),
+ port = 3333,
+ use_renv = TRUE,
+ renv_prompt = TRUE,
+ n_rep = 1,
+ debug = FALSE
+)
+}
+\arguments{
+\item{commit_list}{A list of commit hash codes, branches' names or anything
+else you can use with git checkout \link{...}}
+
+\item{cypress_dir}{The directory with tests recorded by Cypress.
+It can also be a vector of the same size of commit_list}
+
+\item{shinytest2_dir}{The directory with tests recorded by shinytest2
+It can also be a vector of the same size of commit_list}
+
+\item{tests_pattern}{Cypress/shinytest2 files pattern. E.g. 'performance'
+It can also be a vector of the same size of commit_list. If it is NULL,
+all the content in cypress_dir/shinytest2_dir will be used}
+
+\item{app_dir}{The path to the application root}
+
+\item{port}{Port to run the app}
+
+\item{use_renv}{In case it is set as TRUE, package will try to apply
+renv::restore() in all branches. Otherwise, the current loaded list of
+packages will be used in all branches.}
+
+\item{renv_prompt}{Prompt the user before taking any action?}
+
+\item{n_rep}{Number of replications desired}
+
+\item{debug}{Logical. TRUE to display all the system messages on runtime}
+}
+\description{
+Execute performance tests for a list of commits
+}
diff --git a/man/benchmark_cypress.Rd b/man/benchmark_cypress.Rd
new file mode 100644
index 0000000..123e346
--- /dev/null
+++ b/man/benchmark_cypress.Rd
@@ -0,0 +1,46 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/benchmark_cypress.R
+\name{benchmark_cypress}
+\alias{benchmark_cypress}
+\title{Run the performance test based on multiple commits using Cypress}
+\usage{
+benchmark_cypress(
+ commit_list,
+ cypress_dir,
+ tests_pattern,
+ app_dir,
+ port,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+)
+}
+\arguments{
+\item{commit_list}{A list of commit hash codes, branches' names or anything
+else you can use with git checkout \link{...}}
+
+\item{cypress_dir}{The directory with tests recorded by Cypress.
+It can also be a vector of the same size of commit_list}
+
+\item{tests_pattern}{Cypress/shinytest2 files pattern. E.g. 'shinytest2'
+It can also be a vector of the same size of commit_list. If it is NULL,
+all the content in cypress_dir/shinytest2_dir will be used}
+
+\item{app_dir}{The path to the application root}
+
+\item{port}{Port to run the app}
+
+\item{use_renv}{In case it is set as TRUE, package will try to apply
+renv::restore() in all branches. Otherwise, the current loaded list of
+packages will be used in all branches.}
+
+\item{renv_prompt}{Prompt the user before taking any action?}
+
+\item{n_rep}{Number of replications desired}
+
+\item{debug}{Logical. TRUE to display all the system messages on runtime}
+}
+\description{
+Run the performance test based on multiple commits using Cypress
+}
diff --git a/man/benchmark_shinytest2.Rd b/man/benchmark_shinytest2.Rd
new file mode 100644
index 0000000..952e632
--- /dev/null
+++ b/man/benchmark_shinytest2.Rd
@@ -0,0 +1,43 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/benchmark_shinytest2.R
+\name{benchmark_shinytest2}
+\alias{benchmark_shinytest2}
+\title{Run the performance test based on a multiple commits using shinytest2}
+\usage{
+benchmark_shinytest2(
+ commit_list,
+ shinytest2_dir,
+ tests_pattern,
+ app_dir,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+)
+}
+\arguments{
+\item{commit_list}{A list of commit hash codes, branches' names or anything
+else you can use with git checkout \link{...}}
+
+\item{shinytest2_dir}{The directory with tests recorded by shinytest2
+It can also be a vector of the same size of commit_list}
+
+\item{tests_pattern}{shinytest2 files pattern. E.g. 'performance'
+It can also be a vector of the same size of commit_list. If it is NULL,
+all the content in cypress_dir/shinytest2_dir will be used}
+
+\item{app_dir}{The path to the application root}
+
+\item{use_renv}{In case it is set as TRUE, package will try to apply
+renv::restore() in all branches. Otherwise, the current loaded list of
+packages will be used in all branches.}
+
+\item{renv_prompt}{Prompt the user before taking any action?}
+
+\item{n_rep}{Number of replications desired}
+
+\item{debug}{Logical. TRUE to display all the system messages on runtime}
+}
+\description{
+Run the performance test based on a multiple commits using shinytest2
+}
diff --git a/man/check_uncommitted_files.Rd b/man/check_uncommitted_files.Rd
new file mode 100644
index 0000000..c26be49
--- /dev/null
+++ b/man/check_uncommitted_files.Rd
@@ -0,0 +1,12 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{check_uncommitted_files}
+\alias{check_uncommitted_files}
+\title{Check for uncommitted files}
+\usage{
+check_uncommitted_files()
+}
+\description{
+Check for uncommitted files
+}
+\keyword{internal}
diff --git a/man/checkout.Rd b/man/checkout.Rd
new file mode 100644
index 0000000..090a811
--- /dev/null
+++ b/man/checkout.Rd
@@ -0,0 +1,17 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{checkout}
+\alias{checkout}
+\title{Checkout GitHub branch}
+\usage{
+checkout(branch, debug)
+}
+\arguments{
+\item{branch}{Commit hash code or branch name}
+
+\item{debug}{Logical. TRUE to display all the system messages on runtime}
+}
+\description{
+checkout and go to a different branch
+}
+\keyword{internal}
diff --git a/man/checkout_files.Rd b/man/checkout_files.Rd
index 4827d50..b27e7f7 100644
--- a/man/checkout_files.Rd
+++ b/man/checkout_files.Rd
@@ -4,9 +4,13 @@
\alias{checkout_files}
\title{Checkout GitHub files}
\usage{
-checkout_files()
+checkout_files(debug)
+}
+\arguments{
+\item{debug}{Logical. TRUE to display all the system messages on runtime}
}
\description{
-checkout anything created by the app. It prevents errors when
+Checkout anything created by the app. It prevents errors when
changing branches
}
+\keyword{internal}
diff --git a/man/create_cypress_list.Rd b/man/create_cypress_list.Rd
index 104eb3a..d970359 100644
--- a/man/create_cypress_list.Rd
+++ b/man/create_cypress_list.Rd
@@ -1,5 +1,5 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/utils.R
+% Please edit documentation in R/utils_cypress.R
\name{create_cypress_list}
\alias{create_cypress_list}
\title{Create the cypress configuration list}
@@ -8,7 +8,10 @@ create_cypress_list(plugins_file, port)
}
\arguments{
\item{plugins_file}{The path to the Cypress plugins}
+
+\item{port}{Port to run the app}
}
\description{
Create the cypress configuration list
}
+\keyword{internal}
diff --git a/man/create_cypress_plugins.Rd b/man/create_cypress_plugins.Rd
index 208e23c..6e27410 100644
--- a/man/create_cypress_plugins.Rd
+++ b/man/create_cypress_plugins.Rd
@@ -1,5 +1,5 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/utils.R
+% Please edit documentation in R/utils_cypress.R
\name{create_cypress_plugins}
\alias{create_cypress_plugins}
\title{Create the JS code to track execution time}
@@ -9,3 +9,4 @@ create_cypress_plugins()
\description{
Create the JS code to track execution time
}
+\keyword{internal}
diff --git a/man/create_tests_structure.Rd b/man/create_cypress_structure.Rd
similarity index 68%
rename from man/create_tests_structure.Rd
rename to man/create_cypress_structure.Rd
index 8d22fdc..292c26a 100644
--- a/man/create_tests_structure.Rd
+++ b/man/create_cypress_structure.Rd
@@ -1,10 +1,10 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/utils.R
-\name{create_tests_structure}
-\alias{create_tests_structure}
+% Please edit documentation in R/utils_cypress.R
+\name{create_cypress_structure}
+\alias{create_cypress_structure}
\title{Create a temporary directory to store everything needed by Cypress}
\usage{
-create_tests_structure(app_dir, port, debug)
+create_cypress_structure(app_dir, port, debug)
}
\arguments{
\item{app_dir}{The path to the application root}
@@ -16,3 +16,4 @@ create_tests_structure(app_dir, port, debug)
\description{
Create a temporary directory to store everything needed by Cypress
}
+\keyword{internal}
diff --git a/man/create_cypress_tests.Rd b/man/create_cypress_tests.Rd
index 789df5e..6baad5d 100644
--- a/man/create_cypress_tests.Rd
+++ b/man/create_cypress_tests.Rd
@@ -1,16 +1,21 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/utils.R
+% Please edit documentation in R/utils_cypress.R
\name{create_cypress_tests}
\alias{create_cypress_tests}
\title{Create the cypress files under project directory}
\usage{
-create_cypress_tests(project_path, cypress_file)
+create_cypress_tests(project_path, cypress_dir, tests_pattern)
}
\arguments{
-\item{project_path}{The path to the project with all needed packages installed}
+\item{project_path}{The path to the project with all needed packages
+installed}
-\item{cypress_file}{The path to the .js file conteining cypress tests to be recorded}
+\item{cypress_dir}{The directory with tests recorded by Cypress}
+
+\item{tests_pattern}{Cypress files pattern. E.g. 'performance'. If it is NULL,
+all the content will be used}
}
\description{
Create the cypress files under project directory
}
+\keyword{internal}
diff --git a/man/create_node_list.Rd b/man/create_node_list.Rd
index 249eaaf..4c6dd4f 100644
--- a/man/create_node_list.Rd
+++ b/man/create_node_list.Rd
@@ -1,5 +1,5 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/utils.R
+% Please edit documentation in R/utils_cypress.R
\name{create_node_list}
\alias{create_node_list}
\title{Create the list of needed libraries}
@@ -8,7 +8,10 @@ create_node_list(tests_path, port)
}
\arguments{
\item{tests_path}{The path to project}
+
+\item{port}{Port to run the app}
}
\description{
Create the list of needed libraries
}
+\keyword{internal}
diff --git a/man/create_progress_bar.Rd b/man/create_progress_bar.Rd
new file mode 100644
index 0000000..f1bcf92
--- /dev/null
+++ b/man/create_progress_bar.Rd
@@ -0,0 +1,15 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{create_progress_bar}
+\alias{create_progress_bar}
+\title{Create a progress bar to follow the execution}
+\usage{
+create_progress_bar(total = 100)
+}
+\arguments{
+\item{total}{Total number of replications}
+}
+\description{
+Create a progress bar to follow the execution
+}
+\keyword{internal}
diff --git a/man/create_shinytest2_structure.Rd b/man/create_shinytest2_structure.Rd
new file mode 100644
index 0000000..ef03ad8
--- /dev/null
+++ b/man/create_shinytest2_structure.Rd
@@ -0,0 +1,15 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils_shinytest2.R
+\name{create_shinytest2_structure}
+\alias{create_shinytest2_structure}
+\title{Create a temporary directory to store everything needed by shinytest2}
+\usage{
+create_shinytest2_structure(app_dir)
+}
+\arguments{
+\item{app_dir}{The path to the application root}
+}
+\description{
+Create a temporary directory to store everything needed by shinytest2
+}
+\keyword{internal}
diff --git a/man/figures/shiny_benchmark.png b/man/figures/shiny_benchmark.png
new file mode 100644
index 0000000..76f943b
Binary files /dev/null and b/man/figures/shiny_benchmark.png differ
diff --git a/man/get_commit_date.Rd b/man/get_commit_date.Rd
index f218074..fd23414 100644
--- a/man/get_commit_date.Rd
+++ b/man/get_commit_date.Rd
@@ -12,3 +12,4 @@ get_commit_date(branch)
\description{
Get the commit date in POSIXct format
}
+\keyword{internal}
diff --git a/man/get_commit_hash.Rd b/man/get_commit_hash.Rd
index 90f2e44..514dd25 100644
--- a/man/get_commit_hash.Rd
+++ b/man/get_commit_hash.Rd
@@ -9,3 +9,4 @@ get_commit_hash()
\description{
Find the hash code of the current commit
}
+\keyword{internal}
diff --git a/man/load_example.Rd b/man/load_example.Rd
new file mode 100644
index 0000000..df8ada0
--- /dev/null
+++ b/man/load_example.Rd
@@ -0,0 +1,24 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{load_example}
+\alias{load_example}
+\title{Load an application and instructions to run shiny.benchmark}
+\usage{
+load_example(path, force = FALSE)
+}
+\arguments{
+\item{path}{A character vector of full path name}
+
+\item{force}{Create example even if directory does not exist or is not empty}
+}
+\description{
+This function aims to generate a template to be used
+by shiny.benchmark. It will create the necessary structure on \code{path} with
+some examples of tests using Cypress and shinytest2. Also, a simple
+application will be added to the folder as well as instructions on how
+to perform the performance checks. Be aware that a new git repo is need in
+the selected \code{path}.
+}
+\examples{
+load_example(file.path(tempdir(), "example_destination"), force = TRUE)
+}
diff --git a/man/move_shinytest2_tests.Rd b/man/move_shinytest2_tests.Rd
new file mode 100644
index 0000000..a47c0f7
--- /dev/null
+++ b/man/move_shinytest2_tests.Rd
@@ -0,0 +1,17 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils_shinytest2.R
+\name{move_shinytest2_tests}
+\alias{move_shinytest2_tests}
+\title{Move tests to a temporary folder}
+\usage{
+move_shinytest2_tests(project_path, shinytest2_dir)
+}
+\arguments{
+\item{project_path}{The path to the project}
+
+\item{shinytest2_dir}{The directory with tests recorded by shinytest2}
+}
+\description{
+Move tests to a temporary folder
+}
+\keyword{internal}
diff --git a/man/performance_test_cmd.Rd b/man/performance_test_cmd.Rd
new file mode 100644
index 0000000..82a390a
--- /dev/null
+++ b/man/performance_test_cmd.Rd
@@ -0,0 +1,15 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{performance_test_cmd}
+\alias{performance_test_cmd}
+\title{Running the node script "performance_test" is system-dependent}
+\usage{
+performance_test_cmd(project_path)
+}
+\arguments{
+\item{project_path}{path to project directory (one level above node)}
+}
+\description{
+Running the node script "performance_test" is system-dependent
+}
+\keyword{internal}
diff --git a/man/performance_tests.Rd b/man/performance_tests.Rd
deleted file mode 100644
index 586d634..0000000
--- a/man/performance_tests.Rd
+++ /dev/null
@@ -1,28 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/performance_tests.R
-\name{performance_tests}
-\alias{performance_tests}
-\title{Execute performance tests for a list of commits}
-\usage{
-performance_tests(
- commit_list,
- cypress_file,
- app_dir = getwd(),
- port = 3333,
- debug = FALSE
-)
-}
-\arguments{
-\item{commit_list}{A list of commit hash codes, branches' names or anything else you can use with git checkout \link{...}}
-
-\item{cypress_file}{The path to the .js file containing cypress tests to be recorded}
-
-\item{app_dir}{The path to the application root}
-
-\item{port}{Port to run the app}
-
-\item{debug}{Logical. TRUE to display all the system messages on runtime}
-}
-\description{
-Execute performance tests for a list of commits
-}
diff --git a/man/plot.shiny_benchmark.Rd b/man/plot.shiny_benchmark.Rd
new file mode 100644
index 0000000..f3d2036
--- /dev/null
+++ b/man/plot.shiny_benchmark.Rd
@@ -0,0 +1,16 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/plot.R
+\name{plot.shiny_benchmark}
+\alias{plot.shiny_benchmark}
+\title{Plot for shiny_benchmark class}
+\usage{
+\method{plot}{shiny_benchmark}(x, ...)
+}
+\arguments{
+\item{x}{shiny_benchmark object}
+
+\item{...}{Other parameters}
+}
+\description{
+Plot for shiny_benchmark class
+}
diff --git a/man/print.shiny_benchmark.Rd b/man/print.shiny_benchmark.Rd
new file mode 100644
index 0000000..dcdea35
--- /dev/null
+++ b/man/print.shiny_benchmark.Rd
@@ -0,0 +1,16 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/print.R
+\name{print.shiny_benchmark}
+\alias{print.shiny_benchmark}
+\title{Print for shiny_benchmark class}
+\usage{
+\method{print}{shiny_benchmark}(x, ...)
+}
+\arguments{
+\item{x}{shiny_benchmark object}
+
+\item{...}{Other parameters}
+}
+\description{
+Print for shiny_benchmark class
+}
diff --git a/man/restore_env.Rd b/man/restore_env.Rd
new file mode 100644
index 0000000..ec881e0
--- /dev/null
+++ b/man/restore_env.Rd
@@ -0,0 +1,20 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{restore_env}
+\alias{restore_env}
+\title{Check and restore renv}
+\usage{
+restore_env(branch, renv_prompt)
+}
+\arguments{
+\item{branch}{Commit hash code or branch name. Useful to create an
+informative error message}
+
+\item{renv_prompt}{Prompt the user before taking any action?}
+}
+\description{
+Check whether renv is in use in the current branch. Raise error
+if renv is not in use or apply renv:restore() in the case the package is
+present
+}
+\keyword{internal}
diff --git a/man/run_cypress_ptest.Rd b/man/run_cypress_ptest.Rd
new file mode 100644
index 0000000..fc978db
--- /dev/null
+++ b/man/run_cypress_ptest.Rd
@@ -0,0 +1,41 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/benchmark_cypress.R
+\name{run_cypress_ptest}
+\alias{run_cypress_ptest}
+\title{Run the performance test based on a single commit using Cypress}
+\usage{
+run_cypress_ptest(
+ commit,
+ project_path,
+ cypress_dir,
+ tests_pattern,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+)
+}
+\arguments{
+\item{commit}{A commit hash code or a branch's name}
+
+\item{project_path}{The path to the project with all needed packages
+installed}
+
+\item{cypress_dir}{The directory with tests recorded by Cypress}
+
+\item{tests_pattern}{Cypress files pattern. E.g. 'performance'. If it is NULL,
+all the content will be used}
+
+\item{use_renv}{In case it is set as TRUE, package will try to apply
+renv::restore() in all branches. Otherwise, the current loaded list of
+packages will be used in all branches.}
+
+\item{renv_prompt}{Prompt the user before taking any action?}
+
+\item{n_rep}{Number of replications desired}
+
+\item{debug}{Logical. TRUE to display all the system messages on runtime}
+}
+\description{
+Run the performance test based on a single commit using Cypress
+}
diff --git a/man/run_performance_test.Rd b/man/run_performance_test.Rd
deleted file mode 100644
index 2271ac9..0000000
--- a/man/run_performance_test.Rd
+++ /dev/null
@@ -1,22 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/performance_tests.R
-\name{run_performance_test}
-\alias{run_performance_test}
-\title{Run the performance test based on a single commit}
-\usage{
-run_performance_test(commit, project_path, cypress_file, txt_file, debug)
-}
-\arguments{
-\item{commit}{A commit hash code or a branch's name}
-
-\item{project_path}{The path to the project with all needed packages installed}
-
-\item{cypress_file}{The path to the .js file conteining cypress tests to be recorded}
-
-\item{txt_file}{The path to the file where it is aimed to save the times}
-
-\item{debug}{Logical. TRUE to display all the system messages on runtime}
-}
-\description{
-Run the performance test based on a single commit
-}
diff --git a/man/run_shinytest2_ptest.Rd b/man/run_shinytest2_ptest.Rd
new file mode 100644
index 0000000..a1e6662
--- /dev/null
+++ b/man/run_shinytest2_ptest.Rd
@@ -0,0 +1,43 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/benchmark_shinytest2.R
+\name{run_shinytest2_ptest}
+\alias{run_shinytest2_ptest}
+\title{Run the performance test based on a single commit using shinytest2}
+\usage{
+run_shinytest2_ptest(
+ commit,
+ project_path,
+ app_dir,
+ shinytest2_dir,
+ tests_pattern,
+ use_renv,
+ renv_prompt,
+ n_rep,
+ debug
+)
+}
+\arguments{
+\item{commit}{A commit hash code or a branch's name}
+
+\item{project_path}{The path to the project}
+
+\item{app_dir}{The path to the application root}
+
+\item{shinytest2_dir}{The directory with tests recorded by shinytest2}
+
+\item{tests_pattern}{shinytest2 files pattern. E.g. 'performance'. If it is NULL,
+all the content will be used}
+
+\item{use_renv}{In case it is set as TRUE, package will try to apply
+renv::restore() in all branches. Otherwise, the current loaded list of
+packages will be used in all branches.}
+
+\item{renv_prompt}{Prompt the user before taking any action?}
+
+\item{n_rep}{Number of replications desired}
+
+\item{debug}{Logical. TRUE to display all the system messages on runtime}
+}
+\description{
+Run the performance test based on a single commit using shinytest2
+}
diff --git a/man/shiny_benchmark-class.Rd b/man/shiny_benchmark-class.Rd
new file mode 100644
index 0000000..6028c28
--- /dev/null
+++ b/man/shiny_benchmark-class.Rd
@@ -0,0 +1,20 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/shiny_benchmark-class.R
+\docType{class}
+\name{shiny_benchmark-class}
+\alias{shiny_benchmark-class}
+\alias{shiny_benchmark_class}
+\title{An object of 'shiny_benchmark' class}
+\description{
+An object of 'shiny_benchmark' class
+}
+\section{Slots}{
+
+\describe{
+\item{\code{call}}{Function call}
+
+\item{\code{time}}{Time elapsed}
+
+\item{\code{performance}}{List of measurements (one entry for each commit)}
+}}
+
diff --git a/man/summarise_commit.Rd b/man/summarise_commit.Rd
new file mode 100644
index 0000000..623d10a
--- /dev/null
+++ b/man/summarise_commit.Rd
@@ -0,0 +1,15 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{summarise_commit}
+\alias{summarise_commit}
+\title{Return statistics based on the set of tests replications}
+\usage{
+summarise_commit(object)
+}
+\arguments{
+\item{object}{A shiny_benchmark object}
+}
+\description{
+Return statistics based on the set of tests replications
+}
+\keyword{internal}
diff --git a/man/summary.shiny_benchmark.Rd b/man/summary.shiny_benchmark.Rd
new file mode 100644
index 0000000..6b3c298
--- /dev/null
+++ b/man/summary.shiny_benchmark.Rd
@@ -0,0 +1,16 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/summary.R
+\name{summary.shiny_benchmark}
+\alias{summary.shiny_benchmark}
+\title{Summary for shiny_benchmark class}
+\usage{
+\method{summary}{shiny_benchmark}(object, ...)
+}
+\arguments{
+\item{object}{shiny_benchmark object}
+
+\item{...}{Other parameters}
+}
+\description{
+Summary for shiny_benchmark class
+}
diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml
new file mode 100644
index 0000000..a7b789c
--- /dev/null
+++ b/pkgdown/_pkgdown.yml
@@ -0,0 +1,74 @@
+title: shiny.benchmark
+template:
+ bootstrap: 5
+ bootswatch: pulse
+ bslib:
+ pkgdown-nav-height: 100px
+ includes:
+ in_header: |
+
+
+
+
+ before_navbar: |
+
+
+url: https://github.com/Appsilon/shiny.benchmark/
+
+navbar:
+ bg: primary
+ left:
+ - icon: fa-home
+ href: index.html
+ text: "Start"
+ - icon: fa-university
+ href: articles/tutorial/how-to-measure-apps-performance.html
+ text: "Tutorial"
+ - icon: fa-file-code-o
+ text: "Reference"
+ href: reference/index.html
+ right:
+ - icon: fa-github fa-lg
+ href: https://github.com/Appsilon/shiny.benchmark
+ - icon: fa-twitter fa-lg
+ href: https://twitter.com/Appsilon
+ - icon: fab fa-mastodon fa-lg
+ href: https://fosstodon.org/@appsilon
+
+home:
+ sidebar:
+ structure: [star, links, license, community, citation, authors, dev]
+ components:
+ star:
+ title: GitHub
+ text: |
+ Star
+
+reference:
+- title: Performance tests
+ contents:
+ - '`benchmark`'
+ - '`benchmark_cypress`'
+ - '`benchmark_shinytest2`'
+ - '`run_cypress_ptest`'
+ - '`run_shinytest2_ptest`'
+- title: Shiny Benchmark Class
+ contents:
+ - '`shiny_benchmark-class`'
+ - '`summary.shiny_benchmark`'
+ - '`plot.shiny_benchmark`'
+ - '`print.shiny_benchmark`'
+- title: Other
+ contents:
+ - '`load_example`'
+
+footer:
+ structure:
+ left: developed
+ components:
+ developed: "Developed with :heart: by [Appsilon](https://appsilon.com)."
diff --git a/pkgdown/extra.css b/pkgdown/extra.css
new file mode 100644
index 0000000..482904a
--- /dev/null
+++ b/pkgdown/extra.css
@@ -0,0 +1,55 @@
+.navbar {
+ background-color: rgb(178, 9, 41) !important;
+}
+
+#navbar > ul.navbar-nav > li.nav-item a:hover {
+ background-color: rgb(178, 9, 41) !important;
+}
+
+.navbar-dark .navbar-nav .active>.nav-link {
+ background-color: rgb(178, 9, 41) !important;
+ color: #fff;
+}
+
+.navbar-dark input[type="search"] {
+ background-color: #fff !important;
+ color: #444 !important;
+}
+
+nav .text-muted {
+ color: #d8d8d8 !important;
+}
+
+a {
+ color: rgb(156, 19, 44);
+}
+
+a:hover {
+ color: rgb(178, 9, 41);
+}
+
+button.btn.btn-primary.btn-copy-ex {
+ background-color: rgb(178, 9, 41);
+ border-color: rgb(178, 9, 41);
+}
+
+.home {
+ left: 0px;
+ position: absolute;
+ padding: 8px 30px;
+ color: rgba(255,255,255,0.55);
+}
+
+.home:hover {
+ color: rgba(255,255,255,0.9);
+}
+
+.app-preview {
+ margin: 1.5em 0.75em;
+ padding: 0.25em;
+ box-shadow:
+ 0 3.9px 4.6px rgba(0, 0, 0, 0.08),
+ 0 12.3px 8.4px rgba(0, 0, 0, 0.056),
+ 0 18.8px 19.2px rgba(0, 0, 0, 0.037),
+ 0 22px 40px rgba(0, 0, 0, 0.019);
+}
diff --git a/shiny.performance.Rproj b/shiny.benchmark.Rproj
similarity index 88%
rename from shiny.performance.Rproj
rename to shiny.benchmark.Rproj
index 69fafd4..6ff5a50 100644
--- a/shiny.performance.Rproj
+++ b/shiny.benchmark.Rproj
@@ -19,4 +19,4 @@ LineEndingConversion: Posix
BuildType: Package
PackageUseDevtools: Yes
PackageInstallArgs: --no-multiarch --with-keep.source
-PackageRoxygenize: rd,collate,namespace
+PackageRoxygenize: rd,collate,namespace,vignette
diff --git a/tests/end2end/app/.gitignore b/tests/end2end/app/.gitignore
new file mode 100644
index 0000000..c131308
--- /dev/null
+++ b/tests/end2end/app/.gitignore
@@ -0,0 +1,47 @@
+# History files
+.Rhistory
+.Rapp.history
+
+# Session Data files
+.RData
+
+# User-specific files
+.Ruserdata
+
+# Example code in package build process
+*-Ex.R
+
+# Output files from R CMD build
+/*.tar.gz
+
+# Output files from R CMD check
+/*.Rcheck/
+
+ # RStudio files
+ .Rproj.user/
+
+ # produced vignettes
+ vignettes/*.html
+vignettes/*.pdf
+
+# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3
+.httr-oauth
+
+# knitr and R markdown default cache directories
+*_cache/
+ /cache/
+
+ # Temporary files created by R markdown
+ *.utf8.md
+*.knit.md
+
+# R Environment Variables
+.Renviron
+__pycache__
+.Rproj.user
+TODO.txt
+
+# csv opened files
+.~lock*
+
+renv/
diff --git a/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_1.js b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_1.js
new file mode 100644
index 0000000..3943a97
--- /dev/null
+++ b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_1.js
@@ -0,0 +1,30 @@
+describe('Cypress test', () => {
+ // Test that the app starts at all
+ // Also it is needed to start other tests
+ it('The app starts', () => {
+ cy.visit('/');
+ });
+
+ // Test how long it takes to wait for out1
+ it('Out1 time elapsed', () => {
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test if we have a title
+ it('App has a title', () => {
+ cy.contains('Measuring time in different commits').should('be.visible');
+ });
+});
diff --git a/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_2.js b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_2.js
new file mode 100644
index 0000000..3943a97
--- /dev/null
+++ b/tests/end2end/app/fake_folder/tests/cypress/cypress_use_this_one_2.js
@@ -0,0 +1,30 @@
+describe('Cypress test', () => {
+ // Test that the app starts at all
+ // Also it is needed to start other tests
+ it('The app starts', () => {
+ cy.visit('/');
+ });
+
+ // Test how long it takes to wait for out1
+ it('Out1 time elapsed', () => {
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test if we have a title
+ it('App has a title', () => {
+ cy.contains('Measuring time in different commits').should('be.visible');
+ });
+});
diff --git a/tests/end2end/app/fake_folder/tests/testthat.R b/tests/end2end/app/fake_folder/tests/testthat.R
new file mode 100644
index 0000000..7d25b5b
--- /dev/null
+++ b/tests/end2end/app/fake_folder/tests/testthat.R
@@ -0,0 +1 @@
+shinytest2::test_app()
diff --git a/tests/end2end/app/fake_folder/tests/testthat/setup.R b/tests/end2end/app/fake_folder/tests/testthat/setup.R
new file mode 100644
index 0000000..be65b4f
--- /dev/null
+++ b/tests/end2end/app/fake_folder/tests/testthat/setup.R
@@ -0,0 +1,2 @@
+# Load application support files into testing environment
+shinytest2::load_app_env()
diff --git a/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_1.R b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_1.R
new file mode 100644
index 0000000..7c504e7
--- /dev/null
+++ b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_1.R
@@ -0,0 +1,21 @@
+library(shinytest2)
+
+test_that("{shinytest2} recording: test1", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+
+test_that("{shinytest2} recording: test2", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+
+test_that("{shinytest2} recording: test3", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
diff --git a/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_2.R b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_2.R
new file mode 100644
index 0000000..7c504e7
--- /dev/null
+++ b/tests/end2end/app/fake_folder/tests/testthat/test-use_this_one_2.R
@@ -0,0 +1,21 @@
+library(shinytest2)
+
+test_that("{shinytest2} recording: test1", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+
+test_that("{shinytest2} recording: test2", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+
+test_that("{shinytest2} recording: test3", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
diff --git a/tests/end2end/app/global.R b/tests/end2end/app/global.R
new file mode 100644
index 0000000..5a0cab8
--- /dev/null
+++ b/tests/end2end/app/global.R
@@ -0,0 +1 @@
+library(shiny)
diff --git a/tests/end2end/app/server.R b/tests/end2end/app/server.R
new file mode 100644
index 0000000..84b0505
--- /dev/null
+++ b/tests/end2end/app/server.R
@@ -0,0 +1,39 @@
+function(input, output, session) {
+ # Sys.sleep
+ react1 <- eventReactive(input$run1, {
+ out <- system.time(
+ Sys.sleep(0.1)
+ )
+
+ return(out[3])
+ })
+
+ react2 <- eventReactive(input$run2, {
+ out <- system.time(
+ Sys.sleep(0.1)
+ )
+
+ return(out[3])
+ })
+
+ react3 <- eventReactive(input$run3, {
+ out <- system.time(
+ Sys.sleep(0.1)
+ )
+
+ return(out[1])
+ })
+
+ # outputs
+ output$out1 <- renderUI({
+ tags$span(round(react1()), style = "font-size: 500px;")
+ })
+
+ output$out2 <- renderUI({
+ tags$span(round(react2()), style = "font-size: 500px;")
+ })
+
+ output$out3 <- renderUI({
+ tags$span(round(react3()), style = "font-size: 500px;")
+ })
+}
diff --git a/tests/end2end/app/tests/cypress/cypress_use_this_one_1.js b/tests/end2end/app/tests/cypress/cypress_use_this_one_1.js
new file mode 100644
index 0000000..3943a97
--- /dev/null
+++ b/tests/end2end/app/tests/cypress/cypress_use_this_one_1.js
@@ -0,0 +1,30 @@
+describe('Cypress test', () => {
+ // Test that the app starts at all
+ // Also it is needed to start other tests
+ it('The app starts', () => {
+ cy.visit('/');
+ });
+
+ // Test how long it takes to wait for out1
+ it('Out1 time elapsed', () => {
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test if we have a title
+ it('App has a title', () => {
+ cy.contains('Measuring time in different commits').should('be.visible');
+ });
+});
diff --git a/tests/end2end/app/tests/cypress/cypress_use_this_one_2.js b/tests/end2end/app/tests/cypress/cypress_use_this_one_2.js
new file mode 100644
index 0000000..3943a97
--- /dev/null
+++ b/tests/end2end/app/tests/cypress/cypress_use_this_one_2.js
@@ -0,0 +1,30 @@
+describe('Cypress test', () => {
+ // Test that the app starts at all
+ // Also it is needed to start other tests
+ it('The app starts', () => {
+ cy.visit('/');
+ });
+
+ // Test how long it takes to wait for out1
+ it('Out1 time elapsed', () => {
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test if we have a title
+ it('App has a title', () => {
+ cy.contains('Measuring time in different commits').should('be.visible');
+ });
+});
diff --git a/tests/end2end/app/tests/testthat.R b/tests/end2end/app/tests/testthat.R
new file mode 100644
index 0000000..7d25b5b
--- /dev/null
+++ b/tests/end2end/app/tests/testthat.R
@@ -0,0 +1 @@
+shinytest2::test_app()
diff --git a/tests/end2end/app/tests/testthat/setup.R b/tests/end2end/app/tests/testthat/setup.R
new file mode 100644
index 0000000..be65b4f
--- /dev/null
+++ b/tests/end2end/app/tests/testthat/setup.R
@@ -0,0 +1,2 @@
+# Load application support files into testing environment
+shinytest2::load_app_env()
diff --git a/tests/end2end/app/tests/testthat/test-use_this_one_1.R b/tests/end2end/app/tests/testthat/test-use_this_one_1.R
new file mode 100644
index 0000000..7c504e7
--- /dev/null
+++ b/tests/end2end/app/tests/testthat/test-use_this_one_1.R
@@ -0,0 +1,21 @@
+library(shinytest2)
+
+test_that("{shinytest2} recording: test1", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+
+test_that("{shinytest2} recording: test2", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+
+test_that("{shinytest2} recording: test3", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
diff --git a/tests/end2end/app/tests/testthat/test-use_this_one_2.R b/tests/end2end/app/tests/testthat/test-use_this_one_2.R
new file mode 100644
index 0000000..7c504e7
--- /dev/null
+++ b/tests/end2end/app/tests/testthat/test-use_this_one_2.R
@@ -0,0 +1,21 @@
+library(shinytest2)
+
+test_that("{shinytest2} recording: test1", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+
+test_that("{shinytest2} recording: test2", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+
+test_that("{shinytest2} recording: test3", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
diff --git a/tests/end2end/app/ui.R b/tests/end2end/app/ui.R
new file mode 100644
index 0000000..8d3615a
--- /dev/null
+++ b/tests/end2end/app/ui.R
@@ -0,0 +1,20 @@
+function() {
+ bootstrapPage(
+ tags$h1("Measuring time in different commits"),
+ column(
+ width = 4,
+ actionButton(inputId = "run1", label = "Run 1"),
+ uiOutput(outputId = "out1")
+ ),
+ column(
+ width = 4,
+ actionButton(inputId = "run2", label = "Run 2"),
+ uiOutput(outputId = "out2")
+ ),
+ column(
+ width = 4,
+ actionButton(inputId = "run3", label = "Run 3"),
+ uiOutput(outputId = "out3")
+ )
+ )
+}
diff --git a/tests/end2end/run_tests.R b/tests/end2end/run_tests.R
new file mode 100644
index 0000000..bac5593
--- /dev/null
+++ b/tests/end2end/run_tests.R
@@ -0,0 +1,51 @@
+#!/usr/bin/env Rscript
+args <- commandArgs(trailingOnly = TRUE)
+args <- strsplit(args, ",")
+
+# packages
+library(shiny)
+library(testthat)
+library(shiny.benchmark)
+
+# commits to compare
+type <- args[[1]]
+commit_list <- args[[2]]
+dir <- args[[3]]
+pattern <- args[[4]]
+use_renv <- as.logical(args[[5]])
+n_rep <- as.integer(args[[6]])
+
+if (type == "cypress") {
+ # run performance check using Cypress
+ out <- benchmark(
+ commit_list = commit_list,
+ cypress_dir = dir,
+ tests_pattern = pattern,
+ app_dir = getwd(),
+ use_renv = use_renv,
+ renv_prompt = FALSE,
+ port = 3333,
+ n_rep = n_rep,
+ debug = FALSE
+ )
+} else {
+ # run performance check using shinytest2
+ out <- benchmark(
+ commit_list = commit_list,
+ shinytest2_dir = dir,
+ tests_pattern = pattern,
+ app_dir = getwd(),
+ use_renv = use_renv,
+ renv_prompt = FALSE,
+ port = 3333,
+ n_rep = n_rep,
+ debug = FALSE
+ )
+}
+
+# checks
+stopifnot(length(out$performance) == length(commit_list))
+stopifnot(length(out$performance[[1]]) >= n_rep)
+
+# deactivate renv
+renv::deactivate()
diff --git a/tests/end2end/setting_branches.sh b/tests/end2end/setting_branches.sh
new file mode 100644
index 0000000..e15d555
--- /dev/null
+++ b/tests/end2end/setting_branches.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+# starting
+git init
+git config --global advice.detachedHead false
+
+# credentials
+git config --local user.name "$GITHUB_ACTOR"
+git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+# STANDARD FUNCTIONALITIES
+## master
+git add .
+git commit -m "first commit"
+
+## develop
+git checkout -b develop
+git commit --allow-empty -m "dummy commit to change hash"
+
+# RENV FUNCTIONALITIES
+## No renv at all
+git branch renv_missing master
+git checkout renv_missing
+git commit --allow-empty -m "dummy commit to change hash"
+
+## Creating renv
+git branch renv_shiny1 master
+git checkout renv_shiny1
+R -e 'renv::init()'
+git add .
+git commit -m "renv active"
+
+## Downgrading shiny
+git checkout -b renv_shiny2
+R -e 'renv::install("shiny@1.7.0")'
+R -e 'renv::snapshot()'
+git add .
+git commit -m "downgrading shiny"
+
+## Switching back to master
+git checkout master
diff --git a/tests/testthat.R b/tests/testthat.R
new file mode 100644
index 0000000..85df7b5
--- /dev/null
+++ b/tests/testthat.R
@@ -0,0 +1,4 @@
+library(testthat)
+library(shiny.benchmark)
+
+test_check("shiny.benchmark")
diff --git a/tests/testthat/test-load_example.R b/tests/testthat/test-load_example.R
new file mode 100644
index 0000000..20688e8
--- /dev/null
+++ b/tests/testthat/test-load_example.R
@@ -0,0 +1,84 @@
+# Necessary test as fs::dir_copy with overwrite has a different behavior
+# than file.copy.
+# It copies the content of the directory of the "from" path into the
+# destination, instead of the directory itself
+test_that("Load example creates correct structure", {
+ example_path <- fs::path(tempdir(), "load_example")
+ fs::dir_create(example_path)
+ local({
+ local_mock(menu = function(...) stop("Opps, shouldn't reach this"))
+ load_example(example_path, force = TRUE)
+ })
+
+ files <- example_path |>
+ fs::path(
+ c(
+ "run_tests.R",
+ fs::path("app", "ui.R"),
+ fs::path("app", "server.R"),
+ fs::path("app", "global.R"),
+ fs::path("app", "tests", "testthat.R"),
+ fs::path("app", "tests", "testthat", "setup.R"),
+ fs::path("app", "tests", "testthat", "test-use_this_one_1.R"),
+ fs::path("app", "tests", "testthat", "test-use_this_one_2.R")
+ )
+ )
+
+ dirs <- example_path |>
+ fs::path(
+ c(
+ fs::path("app", "tests"),
+ fs::path("app", "tests", "cypress"),
+ fs::path("app", "tests", "testthat")
+ )
+ )
+
+ expect_true(all(fs::file_exists(files)))
+ expect_true(all(fs::is_file(files)))
+ expect_false(all(fs::is_dir(files)))
+
+ expect_true(all(fs::dir_exists(dirs)))
+ expect_true(all(fs::is_dir(dirs)))
+ expect_false(all(fs::is_file(dirs)))
+})
+
+test_that("Does not create load_examples on non-existing directory", {
+ example_path <- fs::path(
+ tempdir(),
+ glue::glue("load_example_not_existing{unclass(Sys.time())}")
+ )
+
+ local({
+ local_mock(menu = function(...) stop("Opps, shouldn't reach this"))
+ load_example(example_path) |>
+ expect_error("You must provide a valid path")
+ })
+
+ fs::dir_create(example_path)
+ local({
+ local_mock(menu = function(...) stop("Opps, shouldn't reach this"))
+ load_example(example_path) |>
+ expect_output("app created at")
+ })
+})
+
+test_that("Does not create load_examples if there is a file in directory", {
+ example_path <- fs::path(
+ tempdir(),
+ glue::glue("load_example_not_empty{unclass(Sys.time())}")
+ )
+ fs::dir_create(example_path)
+ fs::file_create(fs::path(example_path, "touch.txt"))
+
+ local({
+ local_mock(menu = function(...) 2)
+ load_example(example_path) |>
+ expect_error("Consider creating a new empty path.")
+ })
+
+ local({
+ local_mock(menu = function(...) 1)
+ load_example(example_path) |>
+ expect_output("app created at")
+ })
+})
diff --git a/tests/testthat/test-performance_tests.R b/tests/testthat/test-performance_tests.R
new file mode 100644
index 0000000..07ce8f0
--- /dev/null
+++ b/tests/testthat/test-performance_tests.R
@@ -0,0 +1,42 @@
+test_that("Function fails in case of missing cypress_file or shinytest2dir", {
+ expect_error(
+ performance_tests(
+ commit_list = list("commit_1", "commit_2"),
+ cypress_file = NULL,
+ shinytest2_dir = NULL,
+ app_dir = getwd(),
+ port = 3333,
+ use_renv = TRUE,
+ renv_prompt = TRUE,
+ debug = FALSE
+ )
+ )
+})
+
+test_that("Function fails in case of divergences between commit_list and files length", {
+ expect_error(
+ performance_tests(
+ commit_list = list("commit_1", "commit_2"),
+ cypress_file = c("file_1", "file_2", "file_3"),
+ shinytest2_dir = NULL,
+ app_dir = getwd(),
+ port = 3333,
+ use_renv = TRUE,
+ renv_prompt = TRUE,
+ debug = FALSE
+ )
+ )
+
+ expect_error(
+ performance_tests(
+ commit_list = list("commit_1", "commit_2"),
+ cypress_file = NULL,
+ shinytest2_dir = c("file_1", "file_2", "file_3"),
+ app_dir = getwd(),
+ port = 3333,
+ use_renv = TRUE,
+ renv_prompt = TRUE,
+ debug = FALSE
+ )
+ )
+})
diff --git a/tests/testthat/test-utils_cypress.R b/tests/testthat/test-utils_cypress.R
new file mode 100644
index 0000000..b24d12d
--- /dev/null
+++ b/tests/testthat/test-utils_cypress.R
@@ -0,0 +1,27 @@
+test_that("Check if we are able to add Cypress code to a txt file", {
+ tmp_dir <- tempdir()
+ add_sendtime2js(
+ js_file = fs::path(tmp_dir, "test.js"),
+ txt_file = "test.txt"
+ )
+
+ expect_true(fs::file_exists(fs::path(tmp_dir, "test.js")))
+})
+
+test_that("Check if we are able to copy file content from a file to another", {
+ tmp_dir <- tempdir()
+ tmp_file <- tempfile(tmpdir = tmp_dir, fileext = ".js")
+ content_before <- "TEST"
+ writeLines(text = content_before, con = tmp_file)
+
+ integration_dir <- fs::path(tmp_dir, "tests", "cypress", "integration")
+ fs::dir_create(path = integration_dir)
+ files <- create_cypress_tests(
+ project_path = tmp_dir,
+ cypress_dir = tmp_dir,
+ tests_pattern = ".js"
+ )
+ content_after <- readLines(con = files$js_file, n = 1)
+
+ expect_true(content_after == content_before)
+})
diff --git a/tests/testthat/test-utils_shinytest2.R b/tests/testthat/test-utils_shinytest2.R
new file mode 100644
index 0000000..410bd3e
--- /dev/null
+++ b/tests/testthat/test-utils_shinytest2.R
@@ -0,0 +1,21 @@
+test_that("Check if we are able to move files properly", {
+ tmp_dir <- tempdir()
+
+ tmp_dir1 <- file.path(tmp_dir, "folder1")
+ dir.create(tmp_dir1, showWarnings = FALSE)
+ tmp_dir2 <- file.path(tmp_dir, "folder2")
+ dir.create(tmp_dir2, showWarnings = FALSE)
+
+ shinytest2_dir <- file.path(tmp_dir1, "tst")
+ shinytest2_dir_copy <- file.path(tmp_dir2, "tst")
+ dir.create(path = shinytest2_dir, showWarnings = FALSE)
+
+ move_shinytest2_tests(project_path = tmp_dir2, shinytest2_dir = shinytest2_dir)
+
+ expect_true(file.exists(shinytest2_dir_copy))
+})
+
+test_that("Check if we are able to create shinytest2 structure", {
+ tmp_dir <- create_shinytest2_structure(app_dir = ".")
+ expect_true(file.exists(file.path(tmp_dir, "app.R")))
+})
diff --git a/vignettes/.gitignore b/vignettes/.gitignore
new file mode 100644
index 0000000..097b241
--- /dev/null
+++ b/vignettes/.gitignore
@@ -0,0 +1,2 @@
+*.html
+*.R
diff --git a/vignettes/tutorial/how-to-measure-apps-performance.Rmd b/vignettes/tutorial/how-to-measure-apps-performance.Rmd
new file mode 100644
index 0000000..10b1a95
--- /dev/null
+++ b/vignettes/tutorial/how-to-measure-apps-performance.Rmd
@@ -0,0 +1,472 @@
+---
+title: "Tutorial: Compare performance of different versions of a shiny application"
+output:
+ rmarkdown::html_vignette:
+ self_contained: true
+vignette: >
+ %\VignetteIndexEntry{Tutorial: Compare performance of different versions of a shiny application}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+# Setup
+
+## How to install shiny.benchmark?
+
+`shiny.benchmark` can use two different engines to test the changes in the performance of your application: [shinytest2](https://rstudio.github.io/shinytest2/) and [Cypress](https://www.cypress.io/). The latter requires `Node` (version 12 or higher) and `yarn` (version 1.22.17 or higher) to be available. To install them on your computer, follow the guidelines on the documentation pages:
+
+- [Node](https://nodejs.org/en/download/)
+- [yarn](https://yarnpkg.com/getting-started/install)
+
+Besides that, on Linux, it might be required to install other `Cypress` dependencies. Check the [documentation](https://docs.cypress.io/guides/getting-started/installing-cypress#Linux-Prerequisites) to find out more.
+
+To install `shiny.benchmark` use the following command:
+
+```r
+remotes::install_github("Appsilon/shiny.benchmark")
+```
+
+`shiny.benchmark` will handle `Cypress` installation. If you face any inconvenience using `Cypress`, please try to use `shinytest2` in the rest of this tutorial.
+
+----
+
+# Create an initial application
+
+Let's start by creating an application that will serve us as a guide through the `shiny.benchmark` functionalities.
+
+Save the following code as `ui.R`. It is a simple user interface containing three columns with one action button in each. Also each column has an output which will be created in the server file later.
+
+```r
+function() {
+ bootstrapPage(
+ tags$h1("Measuring time in different commits"),
+ column(
+ width = 4,
+ actionButton(inputId = "run1", label = "Run 1"),
+ uiOutput(outputId = "out1")
+ ),
+ column(
+ width = 4,
+ actionButton(inputId = "run2", label = "Run 2"),
+ uiOutput(outputId = "out2")
+ ),
+ column(
+ width = 4,
+ actionButton(inputId = "run3", label = "Run 3"),
+ uiOutput(outputId = "out3")
+ )
+ )
+}
+```
+
+In the server side, the application will use the `Sys.sleep` function to simulate a task every time the user press a button. This will be helpful for us since we can easily increase/decrease the sleep time to simulate improvements/deterioration of the application. Save the following code as `server.R`:
+
+```r
+times <- c(10, 5, 2)
+
+function(input, output, session) {
+ # Sys.sleep
+ react1 <- eventReactive(input$run1, {
+ out <- system.time(
+ Sys.sleep(times[1] + rexp(n = 1, rate = 1)) # we will play with the time here
+ )
+
+ return(out[3])
+ })
+
+ react2 <- eventReactive(input$run2, {
+ out <- system.time(
+ Sys.sleep(times[2] + rexp(n = 1, rate = 1)) # we will play with the time here
+ )
+
+ return(out[3])
+ })
+
+ react3 <- eventReactive(input$run3, {
+ out <- system.time(
+ Sys.sleep(times[3] + rexp(n = 1, rate = 1)) # we will play with the time here
+ )
+
+ return(out[1])
+ })
+
+ # outputs
+ output$out1 <- renderUI({
+ tags$span(round(react1()), style = "font-size: 5vw;")
+ })
+
+ output$out2 <- renderUI({
+ tags$span(round(react2()), style = "font-size: 5vw;")
+ })
+
+ output$out3 <- renderUI({
+ tags$span(round(react3()), style = "font-size: 5vw;")
+ })
+}
+```
+
+The application should look like this:
+
+```r
+shiny::runApp()
+```
+
+
+
+----
+
+# Tests engines
+
+`shiny.benchmark` works under two different engines: `Cypress` and `shinytest2`.
+
+## shinytest2
+
+`shinytest2` is an R package maintained by [Posit](https://posit.co/) (formerly RStudio). It is handy for R users since all tests can be done using R only (differently than Cypress). To set up it easily, run `shinytest2::use_shinytest2()`. It will create configuration files which you do not need to change for this tutorial.
+
+Save the following code as `tests/testthat/test-set1.R`:
+
+```r
+test_that("Out1 time elapsed - set1", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+test_that("Out2 time elapsed - set1", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+test_that("Out3 time elapsed - set1", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
+```
+
+This code is simulating clicks in the three buttons we have in our application. Also it waits for the outputs to appear. In a new file, replace `set1` by `set2` in the code and save it as `tests/testthat/test-set2.R` as well. It will be useful to present some functionalities later.
+
+```r
+test_that("Out1 time elapsed - set2", {
+ app <- AppDriver$new(name = "test1", height = 975, width = 1619)
+ app$click("run1")
+ app$expect_values(output = "out1")
+})
+
+test_that("Out2 time elapsed - set2", {
+ app <- AppDriver$new(name = "test2", height = 975, width = 1619)
+ app$click("run2")
+ app$expect_values(output = "out2")
+})
+
+test_that("Out3 time elapsed - set2", {
+ app <- AppDriver$new(name = "test3", height = 975, width = 1619)
+ app$click("run3")
+ app$expect_values(output = "out3")
+})
+```
+
+## Cypress
+
+Cypress is a widely used end to end testing JavaScript library. Because its broader usage, this engine allows the user to take advantage of a huge number of functionalities in order to test its applications. Also, the community is active and therefore it is easier to find solution for bugs you may encounter while coding.
+
+Save the following code as `tests/cypress/test-set1.js`:
+
+```r
+describe('Cypress test', () => {
+ it('Out1 time elapsed - set1', () => {
+ cy.visit('/');
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed - set1', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed - set1', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+});
+```
+
+Again, replace `set1` by `set2` in the code and save it as `tests/cypress/test-set2.R` as well.
+
+```r
+describe('Cypress test', () => {
+ it('Out1 time elapsed - set2', () => {
+ cy.visit('/');
+ cy.get('#run1').click();
+ cy.get('#out1', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out2
+ it('Out2 time elapsed - set2', () => {
+ cy.get('#run2').click();
+ cy.get('#out2', {timeout: 10000}).should('be.visible');
+ });
+
+ // Test how long it takes to wait for out3
+ it('Out3 time elapsed - set2', () => {
+ cy.get('#run3').click();
+ cy.get('#out3', {timeout: 10000}).should('be.visible');
+ });
+});
+```
+
+
+----
+
+# Package management
+
+During the development process, it is normal to use different packages/package versions. `renv` allow us to manage package versions and is used by `shiny.benchmark` by default. Run the following code to setup `renv` in our test application.
+
+```r
+renv::init()
+renv::install("remotes")
+remotes::install_github("Appsilon/shiny.benchmark")
+renv::snapshot(prompt = FALSE)
+```
+
+
+----
+
+# Simulating app versions
+
+In a regular project, you use `git` to maintain the code versioning. In this case, it is natural to have different app's versions in different branches/commits/releases. `shiny.benchmark` take advantage of these different `git` refs to run tests under different code versions. Add the following code to `.gitignore` to avoid problems with uncommitted files later:
+
+```git
+.Rhistory
+.Rproj.user/
+.Rproj.user
+renv/
+```
+
+Now, lets create a `git` repo and commit the current application into the `develop` branch:
+
+```git
+git init
+git checkout -b develop
+git add .
+git commit -m "first commit"
+```
+
+Also, let's create a new branch called `feature1`:
+
+```git
+git checkout -b feature1
+```
+
+At this point, we can simulate improvement in our application. To do so, let's change `Sys.sleep` time in the server function. Replace `times <- c(10, 5, 2)` by `times <- c(5, 2.5, 1)` in first row of `server.R` and then commit the changes.
+
+```git
+git add server.R
+git commit -m "improving performance"
+```
+
+To play with `renv` let's downgrade `shiny` version and snapshot it:
+
+```git
+git checkout -b feature2
+```
+
+Replace `times <- c(5, 2.5, 1)` by `times <- c(2.5, 1.25, 0.5)` in first row of `server.R`. Also, run the following code to downgrade `shiny`:
+
+```r
+renv::install("shiny@1.0.0")
+renv::snapshot(prompt = FALSE)
+```
+
+Commit the changes:
+
+```git
+git add .
+git commit -m "downgrading shiny"
+git checkout develop
+```
+
+Great! We are all set!
+
+----
+
+# shiny.benchmark
+
+Now we have all ingredients needed: An application, a set of tests and different versions in a `git` repo. `shiny.benchmark::benchmark` function has only two mandatory arguments:
+
+- `commit_list`: a named list of `git` refs (commit hashes, branch names, tags, ...)
+- `cypress_dir` or `shinytest2_dir`: path to `Cypress` or `shinytest2` tests
+
+By default, `shiny.benchmark` uses `renv`. To turn `renv` off just set `use_renv = FALSE` in the `benchmark` call. Be aware that this function will take a while to run since the application will be started and tested 3 times (`develop`, `feature1` and `using_renv` branches).
+
+```r
+library(shiny.benchmark)
+
+commits <- list(
+ "develop" = "develop",
+ "feature1" = "feature1",
+ "using_renv" = "feature2"
+)
+
+cypress_dir <- "tests/cypress/"
+testthat_dir <- "tests/"
+
+cypress_out <- benchmark(
+ commit_list = commits,
+ cypress_dir = cypress_dir,
+ use_renv = FALSE
+)
+
+shinytest2_out <- benchmark(
+ commit_list = commits,
+ shinytest2_dir = testthat_dir,
+ use_renv = FALSE
+)
+```
+
+Instead of a branch name, you can also use the hash code of a desired commit. The console should display something similar to:
+
+
+
+You can access the results using `cypress_out$performance` or `shinytest2_out$performance`:
+
+```r
+cypress_out$performance
+```
+
+```{r echo = FALSE, eval = TRUE}
+list(develop = list(structure(list(date = structure(c(1673034247,
+1673034247, 1673034247, 1673034247, 1673034247, 1673034247), class = c("POSIXct",
+"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1",
+"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2",
+"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(10782L,
+6091L, 2804L, 10591L, 6768L, 3944L)), class = "data.frame", row.names = c(NA,
+-6L))), feature1 = list(structure(list(date = structure(c(1673034279,
+1673034279, 1673034279, 1673034279, 1673034279, 1673034279), class = c("POSIXct",
+"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1",
+"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2",
+"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(6471L,
+6442L, 1422L, 5613L, 3593L, 1272L)), class = "data.frame", row.names = c(NA,
+-6L))), using_renv = list(structure(list(date = structure(c(1673034314,
+1673034314, 1673034314, 1673034314, 1673034314, 1673034314), class = c("POSIXct",
+"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1",
+"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2",
+"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(3941L,
+3010L, 995L, 3082L, 2130L, 2458L)), class = "data.frame", row.names = c(NA,
+-6L))))
+```
+
+You can notice that both tests files are reported (`test-set1` and `test-set2`). Also, the result is a list of `data.frames` in which each entry correspond to a specific commit.
+
+For now on we will use only `shinytest2`. However, everything is also applied for `Cypress`.
+
+## Package management
+
+In order to use `renv`, simply assign `use_renv = TRUE`. You can also use `renv_prompt = TRUE` if you want to see what renv is applying in the background.
+
+```r
+shinytest2_out <- benchmark(
+ commit_list = commits,
+ shinytest2_dir = testthat_dir,
+ use_renv = TRUE,
+ renv_prompt = TRUE
+)
+```
+
+## Handling multiple files
+
+Sometimes it is not our interest to measure performance of all the tests we have. In order to select specific files you can use the argument `tests_pattern`. This argument accept either a vector of files (one for each item in commit list). Also, it is possible to search for a pattern in `tests` files.
+
+```r
+shinytest2_out <- benchmark(
+ commit_list = commits,
+ shinytest2_dir = testthat_dir,
+ use_renv = FALSE,
+ tests_pattern = c("set[0-9]", "set1", "set2")
+)
+shinytest2_out$performance
+```
+
+```{r echo=FALSE}
+list(develop = list(structure(list(date = structure(c(1673034247,
+1673034247, 1673034247, 1673034247, 1673034247, 1673034247), class = c("POSIXct",
+"POSIXt"), tzone = ""), rep_id = c(1L, 1L, 1L, 1L, 1L, 1L), test_name = c("Out1 time elapsed - set1",
+"Out2 time elapsed - set1", "Out3 time elapsed - set1", "Out1 time elapsed - set2",
+"Out2 time elapsed - set2", "Out3 time elapsed - set2"), duration_ms = c(12.0559999999996,
+8.16600000000017, 6.94699999999921, 13.3299999999999, 7.09899999999925,
+6.84699999999975)), class = "data.frame", row.names = c(NA, -6L
+))), feature1 = list(structure(list(date = structure(c(1673034279,
+1673034279, 1673034279), class = c("POSIXct", "POSIXt"), tzone = ""),
+ rep_id = c(1L, 1L, 1L), test_name = c("Out1 time elapsed - set1",
+ "Out2 time elapsed - set1", "Out3 time elapsed - set1"),
+ duration_ms = c(6.96900000000005, 5.58899999999994, 3.23300000000017
+ )), class = "data.frame", row.names = c(NA, -3L))), using_renv = list(
+ structure(list(date = structure(c(1673034314, 1673034314,
+ 1673034314), class = c("POSIXct", "POSIXt"), tzone = ""),
+ rep_id = c(1L, 1L, 1L), test_name = c("Out1 time elapsed - set2",
+ "Out2 time elapsed - set2", "Out3 time elapsed - set2"
+ ), duration_ms = c(4.59799999999996, 4.01999999999953,
+ 3.41100000000006)), class = "data.frame", row.names = c(NA,
+ -3L))))
+```
+
+Now the output is sightly different. For `develop` branch both files (`test-set1` and `test-set2`) are in use since they match the `test-set[0-9]` pattern. For `feature1` and `feature2` only one file is in use since we directly requested `test-set1` and `test-set2` files respectively. It can be useful when new tests are added during the development process and you need to run different tests for different versions.
+
+## Repetitions
+
+Sometimes it is important to repeat the measurement several times to have a distribution of the performance times instead of an unique measurement. To do so, it is possible to use the `n_rep argument` as follows:
+
+```r
+shinytest2_out <- benchmark(
+ commit_list = commits,
+ shinytest2_dir = testthat_dir,
+ use_renv = FALSE,
+ tests_pattern = "set1",
+ n_rep = 5
+)
+```
+
+It is faster than running the benchmark several times since the test structure is created only once internally saving some execution time.
+
+Some methods are implemented to make it easy to explore the results. `summary` brings summarized statistics as mean, median, minimum and maximum while `plot` shows a plot with the average times for each `git` ref and test. Also it presents maximum and minimum range.
+
+```r
+summary(shinytest2_out)
+```
+
+```{r echo = FALSE}
+structure(list(commit = c("develop", "develop", "develop", "feature1",
+"feature1", "feature1", "using_renv", "using_renv", "using_renv"
+), test_name = c("Out1 time elapsed - set1", "Out2 time elapsed - set1",
+"Out3 time elapsed - set1", "Out1 time elapsed - set1", "Out2 time elapsed - set1",
+"Out3 time elapsed - set1", "Out1 time elapsed - set1", "Out2 time elapsed - set1",
+"Out3 time elapsed - set1"), n = c(5L, 5L, 5L, 5L, 5L, 5L, 5L,
+5L, 5L), mean = c(12.3748, 7.37940000000017, 5.71140000000014,
+7.72180000000008, 5.35640000000003, 4.28839999999982, 5.36419999999998,
+4.74899999999998, 4.52899999999991), median = c(12.2960000000003,
+7.1279999999997, 6.1220000000003, 7.32099999999991, 5.4320000000007,
+4.39900000000034, 5.34699999999975, 4.79299999999967, 4.5019999999995
+), sd = c(0.473558549706366, 0.628408147624124, 1.21808653223053,
+1.07820856052986, 0.603523653223451, 0.775558379491765, 0.455512019599966,
+0.748540246078935, 0.982901826227147), min = c(11.9110000000001,
+6.91499999999996, 4.26799999999912, 7.01000000000022, 4.76000000000022,
+3.1279999999997, 4.9340000000002, 3.70100000000002, 3.65899999999965
+), max = c(13.1099999999997, 8.45000000000073, 7.15100000000075,
+9.61200000000008, 6.27800000000025, 5.26899999999932, 6.10800000000017,
+5.50500000000011, 6.11400000000049)), class = c("tbl_df", "tbl",
+"data.frame"), row.names = c(NA, -9L))
+```
+
+```r
+plot(shinytest2_out)
+```
+
+
+
+----
+
+**Congratulations! You are now able to apply your knowledge to check the performance improvements in your own projects!**
diff --git a/vignettes/tutorial/images/app.png b/vignettes/tutorial/images/app.png
new file mode 100644
index 0000000..d862262
Binary files /dev/null and b/vignettes/tutorial/images/app.png differ
diff --git a/vignettes/tutorial/images/console_basic.png b/vignettes/tutorial/images/console_basic.png
new file mode 100644
index 0000000..17c7152
Binary files /dev/null and b/vignettes/tutorial/images/console_basic.png differ
diff --git a/vignettes/tutorial/images/plot.png b/vignettes/tutorial/images/plot.png
new file mode 100644
index 0000000..2d4d415
Binary files /dev/null and b/vignettes/tutorial/images/plot.png differ