diff --git a/DESCRIPTION b/DESCRIPTION index c4b3394..92e1607 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: octopus Type: Package Title: A Database Management Tool -Version: 0.3.0 +Version: 0.4.0 Authors@R: person("Marcus", "Codrescu", , "m.codrescu@outlook.com", role = c("aut", "cre")) Maintainer: Marcus Codrescu Description: A database management tool built as a 'shiny' application. Connect to various @@ -14,12 +14,12 @@ Encoding: UTF-8 Suggests: duckdb, keyring, knitr, odbc, readr, rmarkdown, RMySQL, RPostgres, RSQLite, testthat (>= 3.0.0) Config/testthat/edition: 3 -Imports: bslib, DBI, dplyr, DT, glue, httr, janitor, rio, shiny, - shinyAce, shinyjs, utils +Imports: bslib, data.table, DBI, dplyr, DT, glue, httr, janitor, rio, + shiny, shinyAce, shinyjs, utils RoxygenNote: 7.2.1 VignetteBuilder: knitr NeedsCompilation: no -Packaged: 2023-06-16 19:37:09 UTC; mc678p +Packaged: 2023-10-28 16:55:57 UTC; mc678p Author: Marcus Codrescu [aut, cre] Repository: CRAN -Date/Publication: 2023-06-16 20:10:02 UTC +Date/Publication: 2023-10-28 21:40:07 UTC diff --git a/MD5 b/MD5 index 00032b8..ece7af6 100644 --- a/MD5 +++ b/MD5 @@ -1,28 +1,28 @@ -098edecdb848d80a53f273eeb3f52848 *DESCRIPTION +9f3cef3b7608ec6c8ddd3693aaea90fe *DESCRIPTION 3e1863b11eeb4064746c5621d02ee3b5 *LICENSE -05ecbebff9cf3b09dfc7371aab85c228 *NAMESPACE -1a60940f3b10b0c2c570ff7cce6e0291 *NEWS.md +f3813ad5dad0d3570792162816ecc19d *NAMESPACE +5f139886bb48b261d37870408806719f *NEWS.md 5c59d3a9a03a568524ee43d6894da94c *R/duckdb_connection.R 815cebfc89e33597fe69a1fc134613d4 *R/get_database_functions.R a9615406ff958edb36a67679776b2283 *R/list_drivers.R b131fa0932e6edc47ee9980af76d0e4f *R/mssql_connection.R bdc8850a82cbb2e519eb89a15079378d *R/mysql_connection.R da906db3adb62f15c7387a18394c2ae2 *R/postgres_connection.R -eaa00eae4aaa84664d00e75cfa07c744 *R/snowflake_connection.R +653283f316b964ddbc0fbf8693745463 *R/snowflake_connection.R 3f599c544cd113f9096a4df584a7c18d *R/sqlite_connection.R -3d294a35010d3b37b6e48bad79e18556 *R/submit_query.R -adc4d6589d650d697f632306a9a0bd62 *R/table_modal_w_download.R +9f3c8bfd98964b1d3f9e0281a3455983 *R/submit_query.R +c3bd0ac13598a5c5e41aa0c49f366a89 *R/table_modal_w_download.R a49d8894e75861df9f6b99a87366910c *R/vertica_connection.R -31444afc8f25788e2fd51b94052e0644 *R/view_database.R +fa3293d56f188fc22c6262c9d095fe56 *R/view_database.R 84dc5d43c41f06d0f38712966578536c *README.md 3925e6b9bc74923bb376be32a04488b5 *build/vignette.rds 7d0b4995b5c4a26de637be14eb823878 *inst/doc/octopus.R 27e054ce1ee2fdc93738665235100040 *inst/doc/octopus.Rmd -e98d5b7fcf5411f4222639446bf2b369 *inst/doc/octopus.html +66f8984d7ae6bb91416e3ac2500ecf07 *inst/doc/octopus.html d1474ddceea47e984eecb41867997494 *man/list_drivers.Rd -8ff237823b0797721426490f227c8f17 *man/view_database.Rd +f156f74d75e86f8b02e8c0a841f223b4 *man/view_database.Rd 2cdc915fcce6cb2afcfe9c42c75118a3 *tests/testthat.R -5d095ab381a757670ff0b5d85842b8fe *tests/testthat/test-duckdb_connection.R +394d2178bf6ae5bbf606f6836ac7e8d6 *tests/testthat/test-duckdb_connection.R feb38bbf81f7a1a7358d524fac1b3335 *tests/testthat/test-list_drivers.R 161277c19abe9c3f49b3c6a9cc8ff33f *tests/testthat/test-mssql_connection.R cad1ad2698ce18f33536de3f7decfa07 *tests/testthat/test-mysql_connection.R @@ -30,6 +30,7 @@ f8674af77b52f7bb9bb30784f7c81a23 *tests/testthat/test-postgres_connection.R 1820b9c07d01f0c17fcddac6daf208ee *tests/testthat/test-snowflake_connection.R 2aa54c6031cde1fdffe07ac7b117e985 *tests/testthat/test-sqlite_connection.R aba724e504ec003d87146be2a73b0119 *tests/testthat/test-submit_query.R +25379c234f00aa2781dd45991b2db091 *tests/testthat/test-table_modal_w_download.R 454811f85411f293708b557a0a56fbb7 *tests/testthat/test-vertica_connection.R -7c2b5bd4fa0b99ab323eda43bd7ac411 *tests/testthat/test-view_database.R +e36c2d2a4353c9aff67440ce0c60f769 *tests/testthat/test-view_database.R 27e054ce1ee2fdc93738665235100040 *vignettes/octopus.Rmd diff --git a/NAMESPACE b/NAMESPACE index fcb56ea..f0b93b3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,12 +4,17 @@ export(list_drivers) export(view_database) importFrom(DBI,Id) importFrom(DBI,dbClearResult) +importFrom(DBI,dbFetch) importFrom(DBI,dbGetQuery) +importFrom(DBI,dbHasCompleted) importFrom(DBI,dbListTables) +importFrom(DBI,dbQuoteIdentifier) importFrom(DBI,dbSendQuery) importFrom(DBI,dbWriteTable) importFrom(DT,renderDataTable) importFrom(bslib,bs_theme) +importFrom(data.table,fwrite) +importFrom(data.table,rbindlist) importFrom(dplyr,pull) importFrom(glue,glue) importFrom(httr,GET) @@ -32,6 +37,7 @@ importFrom(shiny,p) importFrom(shiny,removeModal) importFrom(shiny,req) importFrom(shiny,selectInput) +importFrom(shiny,setProgress) importFrom(shiny,shinyApp) importFrom(shiny,showModal) importFrom(shiny,showNotification) @@ -40,6 +46,7 @@ importFrom(shiny,tagList) importFrom(shiny,tags) importFrom(shiny,updateSelectInput) importFrom(shiny,updateSelectizeInput) +importFrom(shiny,withProgress) importFrom(shinyAce,aceEditor) importFrom(shinyAce,updateAceEditor) importFrom(shinyjs,hideElement) diff --git a/NEWS.md b/NEWS.md index f30706d..167e2aa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,11 @@ +# octopus 0.4.0 +* Increased file upload size allowed limit +* Return number of rows of every query +* Allow for easier column name copying +* Auto suggest column names in query editor with ctrl+space +* Allow downloading of an entire database table +* Change selector to uploaded table name after uploading a table + # octopus 0.3.0 * Added helpful loading notifications * Added faster row counts for Snowflake diff --git a/R/snowflake_connection.R b/R/snowflake_connection.R index 53e9e03..8d48ac7 100644 --- a/R/snowflake_connection.R +++ b/R/snowflake_connection.R @@ -29,7 +29,7 @@ get_schemas_snowflake <- function(con) { #' @noRd #' #' @param con A database connection object. -#' @param schema A +#' @param schema A string containing the schema name. #' #' @importFrom DBI dbGetQuery #' @importFrom glue glue diff --git a/R/submit_query.R b/R/submit_query.R index 0466acd..1dbf89e 100644 --- a/R/submit_query.R +++ b/R/submit_query.R @@ -20,7 +20,11 @@ submit_query <- ) } else if(n_rows > 50000){ stop( - "Your query returned a result too large." + paste( + "Your query returned a result too large.", + format(n_rows, big.mark = ","), + "rows returned." + ) ) } else { diff --git a/R/table_modal_w_download.R b/R/table_modal_w_download.R index a4dc2aa..e280545 100644 --- a/R/table_modal_w_download.R +++ b/R/table_modal_w_download.R @@ -21,6 +21,7 @@ #' @return A shiny tagList table_modal_w_download_UI <- function(id, title, download_title, n_rows, result) { ns <- shiny::NS(id) + options(scipen = 99999) shiny::tagList( shiny::showModal( shiny::modalDialog( @@ -28,13 +29,13 @@ table_modal_w_download_UI <- function(id, title, download_title, n_rows, result) size = "xl", shiny::h3(glue("{title}")), shiny::p( - glue::glue("{n_rows} rows") + glue::glue("{format(n_rows, big.mark = \",\")} rows") ), shiny::div( class = "table-responsive", style = "max-height: 70vh;", DT::renderDataTable( - options = list(dom = "t", paging = FALSE), + options = list(dom = "t", paging = FALSE, ordering = FALSE), server = TRUE, rownames = FALSE, { @@ -58,7 +59,7 @@ table_modal_w_download_UI <- function(id, title, download_title, n_rows, result) ) } -#' Table Modal with Download Server +#' Table Modal with Download Server (Preview) #' @noRd #' #' @importFrom shiny moduleServer @@ -88,3 +89,97 @@ table_modal_w_download_Server <- function(id, result) { } ) } + + +#' Table Modal with Download Server (Full Table) +#' @noRd +#' +#' @importFrom shiny moduleServer +#' @importFrom shiny downloadHandler +#' @importFrom shiny removeModal +#' @importFrom shiny withProgress +#' @importFrom shiny setProgress +#' @importFrom shiny showNotification +#' @importFrom DBI dbQuoteIdentifier +#' @importFrom DBI dbSendQuery +#' @importFrom DBI dbFetch +#' @importFrom DBI dbHasCompleted +#' @importFrom DBI Id +#' @importFrom glue glue +#' @importFrom data.table fwrite +#' @importFrom data.table rbindlist +#' +#' @param id A namespace id. +#' @param con A database connection object. +#' @param schema The schema name. +#' @param table The table name. +#' @param n_rows The number of rows of the table. +#' @param step_size The number of rows to retrieve at each step. +#' +#' @return A model server. +table_modal_w_download_full_Server <- function(id, con, schema, table, n_rows, step_size = floor(n_rows / 100)) { + shiny::moduleServer( + id, + function(input, output, session) { + + tryCatch({ + + output$downloadQuery <- + shiny::downloadHandler( + filename = function(){ + + glue::glue( + "{schema}_{table}.csv" + ) + }, + + content = function(file) { + + shiny::withProgress( + min = 0, + max = n_rows, + value = 0, + message = "Initializing", + expr = { + query <- paste( + "SELECT * FROM", + DBI::dbQuoteIdentifier( + con, + DBI::Id(schema = schema, table = table) + ) + ) + + res <- DBI::dbSendQuery(con, query) + table_pieces <- vector("list", length = 101) + i <- 1 + + while (!DBI::dbHasCompleted(res)){ + table_pieces[[i]] <- DBI::dbFetch(res, n = step_size) + + shiny::setProgress( + value = step_size * i, + message = paste("Rows Retrieved:", step_size * i), + session = session + ) + + i <- i + 1 + } + + full_table <- data.frame( + data.table::rbindlist(table_pieces) + ) + + } + ) + + data.table::fwrite(full_table, file) + } + ) + + }, error = function(error){ + shiny::showNotification(error$message) + }) + + } + ) +} diff --git a/R/view_database.R b/R/view_database.R index 8ea1ab3..5a62c9d 100644 --- a/R/view_database.R +++ b/R/view_database.R @@ -4,6 +4,7 @@ #' #' @param con A database connection object. The result of DBI::dbConnect(). #' @param options A named list of options to be passed along to shinyApp(). +#' @param max_file_upload_size An integer. The max number of bits allowed in file uploads. #' #' @importFrom shiny shinyApp #' @importFrom shiny showNotification @@ -49,7 +50,7 @@ #' @export #' view_database <- - function(con, options = list()){ + function(con, options = list(), max_file_upload_size = 2000 * 1024^2){ ui <- shiny::bootstrapPage( theme = bslib::bs_theme( version = 5 @@ -177,6 +178,8 @@ view_database <- showPrintMargin = FALSE, fontSize = 16, highlightActiveLine = FALSE, + autoComplete = "enabled", + autoCompleters = c("static"), hotkeys = list( run_key = list( win = "Ctrl-Shift-Enter", @@ -239,6 +242,7 @@ view_database <- server = TRUE ) + # Update table select on schema change shinyjs::onevent("change", "schema", { shiny::showNotification( @@ -260,6 +264,39 @@ view_database <- "loading-notification" ) }) + + # Update column name suggestions + shinyjs::onevent("change", "tables",{ + + if (input$tables %in% current_tables){ + + # Set Ace Editors Auto Complete Suggestions + current_schema <- input$schema + current_table <- input$tables + current_table_preview <- tryCatch({ + get_preview( + con, + current_schema, + current_table + ) + }, error = function(error){ + data.frame() + }) + auto_complete_suggestions <- list() + auto_complete_suggestions[["Schema"]] <- current_schema + auto_complete_suggestions[["Table"]] <- current_table + auto_complete_suggestions[["Column"]] <- colnames(current_table_preview) + + + shinyAce::updateAceEditor( + session = session, + editorId = "query", + autoCompleteList = auto_complete_suggestions + ) + } + + }) + }, error = function(error){ shiny::showNotification(error$message) }) @@ -291,15 +328,18 @@ view_database <- input$tables ) - table_modal_w_download_Server( + table_modal_w_download_full_Server( id = "preview", - result = result + con = con, + schema = input$schema, + table = input$tables, + n_rows = n_rows ) table_modal_w_download_UI( id = "preview", title = glue::glue("Preview Table: {input$tables}"), - download_title = "Download Preview", + download_title = "Download Table", n_rows = n_rows, result = result ) @@ -380,7 +420,7 @@ view_database <- # Upload Table ----------------------------------------------------------- # Increase file upload limit - options(shiny.maxRequestSize = 2000 * 1024^2) + options(shiny.maxRequestSize = max_file_upload_size) # Upload file to DB shiny::observeEvent(input$newTableUpload, { @@ -440,6 +480,10 @@ view_database <- if (input$cleanColumnNames == "Yes") { new_table <- janitor::clean_names(new_table) + if (driver == "Snowflake"){ + colnames(new_table) <- toupper(colnames(new_table)) + } + } tryCatch({ @@ -475,7 +519,7 @@ view_database <- session, "tables", choices = current_tables, - selected = current_tables[1], + selected = input$newTableName, server = TRUE ) diff --git a/inst/doc/octopus.html b/inst/doc/octopus.html index 9d13502..03903e3 100644 --- a/inst/doc/octopus.html +++ b/inst/doc/octopus.html @@ -359,59 +359,59 @@

Multilingual Support

Take, for example, the following two ways of querying data from a table. In Postgres, you would specify the exact table name using quotations.

-
SELECT * FROM "public"."mtcars"
+
SELECT * FROM "public"."mtcars"

However, in MySQL you would specify the exact table name using back ticks.

-
SELECT * FROM `public`.`mtcars`
+
SELECT * FROM `public`.`mtcars`

Subtle differences like this can be annoying to remember, but you can simply click the View button in the octopus interface and you will get a preview of the table regardless of the SQL dialect.

Here is a list of the currently supported databases.

-
octopus::list_drivers()
-#> [1] "PqConnection"         "Snowflake"            "Vertica Database"    
-#> [4] "duckdb_connection"    "MySQLConnection"      "SQLiteConnection"    
-#> [7] "Microsoft SQL Server"
+
octopus::list_drivers()
+#> [1] "PqConnection"         "Snowflake"            "Vertica Database"    
+#> [4] "duckdb_connection"    "MySQLConnection"      "SQLiteConnection"    
+#> [7] "Microsoft SQL Server"

Comparison to DBI

Let’s demonstrate how to explore a database using the DBI package alone versus using the octopus package. Let’s begin by connecting to a MySQL database that we want to explore.

-
con <- DBI::dbConnect(
-  RMySQL::MySQL(),
-  host = "localhost",
-  user = "sakila",
-  password = "p_ssW0rd",
-  dbname = "sakila",
-  port = 3306
-)
+
con <- DBI::dbConnect(
+  RMySQL::MySQL(),
+  host = "localhost",
+  user = "sakila",
+  password = "p_ssW0rd",
+  dbname = "sakila",
+  port = 3306
+)

We can list all the tables available in the database by using DBI::dbListTables().

-
DBI::dbListTables(con)
-
#  [1] "actor"                      "actor_info"                
-#  [3] "address"                    "category"                  
-#  [5] "city"                       "country"                   
-#  [7] "customer"                   "customer_list"             
-#  [9] "film"                       "film_actor"                
-# [11] "film_category"              "film_list"                 
-# [13] "film_text"                  "inventory"                 
-# [15] "language"                   "nicer_but_slower_film_list"
-# [17] "payment"                    "rental"                    
-# [19] "sales_by_film_category"     "sales_by_store"            
-# [21] "staff"                      "staff_list"                
-# [23] "store"
+
DBI::dbListTables(con)
+
#  [1] "actor"                      "actor_info"                
+#  [3] "address"                    "category"                  
+#  [5] "city"                       "country"                   
+#  [7] "customer"                   "customer_list"             
+#  [9] "film"                       "film_actor"                
+# [11] "film_category"              "film_list"                 
+# [13] "film_text"                  "inventory"                 
+# [15] "language"                   "nicer_but_slower_film_list"
+# [17] "payment"                    "rental"                    
+# [19] "sales_by_film_category"     "sales_by_store"            
+# [21] "staff"                      "staff_list"                
+# [23] "store"

To see the details of a table we can use DBI::dbReadTable() to read the entire table into our R session.

-
DBI::dbReadTable(con, "actor") |> head()
-
#   actor_id first_name    last_name         last_update
-# 1        1   PENELOPE      GUINESS 2006-02-15 04:34:33
-# 2        2       NICK     WAHLBERG 2006-02-15 04:34:33
-# 3        3         ED        CHASE 2006-02-15 04:34:33
-# 4        4   JENNIFER        DAVIS 2006-02-15 04:34:33
-# 5        5     JOHNNY LOLLOBRIGIDA 2006-02-15 04:34:33
-# 6        6      BETTE    NICHOLSON 2006-02-15 04:34:33
+
DBI::dbReadTable(con, "actor") |> head()
+
#   actor_id first_name    last_name         last_update
+# 1        1   PENELOPE      GUINESS 2006-02-15 04:34:33
+# 2        2       NICK     WAHLBERG 2006-02-15 04:34:33
+# 3        3         ED        CHASE 2006-02-15 04:34:33
+# 4        4   JENNIFER        DAVIS 2006-02-15 04:34:33
+# 5        5     JOHNNY LOLLOBRIGIDA 2006-02-15 04:34:33
+# 6        6      BETTE    NICHOLSON 2006-02-15 04:34:33

Using this approach works fine for a single table, but what if we want to view many tables, or even different schemas? We would need to copy and paste our code and change the table name each time.

@@ -422,8 +422,8 @@

Comparison to DBI

Octopus Approach

Let’s try a different approach. Let’s use octopus to interact with the database through the application interface.

-
# Start the octopus app
-octopus::view_database(con)
+
# Start the octopus app
+octopus::view_database(con)

diff --git a/man/view_database.Rd b/man/view_database.Rd index c3913f4..7928304 100644 --- a/man/view_database.Rd +++ b/man/view_database.Rd @@ -4,12 +4,14 @@ \alias{view_database} \title{View Database Connection with Octopus} \usage{ -view_database(con, options = list()) +view_database(con, options = list(), max_file_upload_size = 2000 * 1024^2) } \arguments{ \item{con}{A database connection object. The result of DBI::dbConnect().} \item{options}{A named list of options to be passed along to shinyApp().} + +\item{max_file_upload_size}{An integer. The max number of bits allowed in file uploads.} } \value{ An R Shiny instance. diff --git a/tests/testthat/test-duckdb_connection.R b/tests/testthat/test-duckdb_connection.R index 4579ad1..74cc8b3 100644 --- a/tests/testthat/test-duckdb_connection.R +++ b/tests/testthat/test-duckdb_connection.R @@ -21,7 +21,7 @@ if (duckdb_installed){ "get_schemas retrieves schemas correctly", { - expect_equal( + expect_setequal( get_schemas_duckdb(con), c("information_schema", "main", "pg_catalog") ) diff --git a/tests/testthat/test-table_modal_w_download.R b/tests/testthat/test-table_modal_w_download.R new file mode 100644 index 0000000..5c19705 --- /dev/null +++ b/tests/testthat/test-table_modal_w_download.R @@ -0,0 +1,90 @@ +# Interactively Test the Modal +if (interactive()){ + drv <- duckdb::duckdb() + con <- DBI::dbConnect(drv) + + n <- 100000 + example <- data.frame( + x = runif(n), + y = runif(n), + z = runif(n) + ) + + DBI::dbWriteTable( + con, + "example", + example, + temporary = FALSE, + overwrite = TRUE + ) + + ui <- shiny::basicPage() + + server <- function(input, output, session){ + + table_modal_w_download_full_Server( + id = "preview", + con = con, + schema = "main", + table = "example", + n_rows = n, + step_size = 10 + ) + + table_modal_w_download_UI( + id = "preview", + title = "Preview Table Example", + download_title = "Download Table", + n_rows = n, + result = example[1:1000,] + ) + } + + shiny::shinyApp(ui = ui, server = server) +} + +# Same as Above but Using Larger Database +if (interactive()){ + con <- DBI::dbConnect( + odbc::odbc(), + dsn = "Snowflake_BBR", + pwd = keyring::key_get("Upstart") + ) + + res <- DBI::dbSendQuery( + con, + "CREATE TEMPORARY TABLE SDW_ECDW_ATT_VIEWS.EXAMPLE AS + SELECT * FROM SDW_ECDW_ATT_VIEWS.PRSD_ADDR LIMIT 1000000" + ) + + DBI::dbClearResult(res) + + ui <- shiny::basicPage() + + server <- function(input, output, session){ + + table_modal_w_download_full_Server( + id = "preview", + con = con, + schema = "SDW_ECDW_ATT_VIEWS", + table = "EXAMPLE", + n_rows = 1000000 + ) + + result <- get_preview_snowflake( + con, + schema = "SDW_ECDW_ATT_VIEWS", + table = "EXAMPLE" + ) + + table_modal_w_download_UI( + id = "preview", + title = "Preview Table EXAMPLE", + download_title = "Download Table", + n_rows = 1000000, + result = result + ) + } + + shiny::shinyApp(ui = ui, server = server) +} diff --git a/tests/testthat/test-view_database.R b/tests/testthat/test-view_database.R index 5bfc8b3..8dbcfcf 100644 --- a/tests/testthat/test-view_database.R +++ b/tests/testthat/test-view_database.R @@ -20,3 +20,16 @@ test_that( expect_true(TRUE) } ) + +# Interactively Test the App +if (interactive()){ + + con <- DBI::dbConnect( + odbc::odbc(), + dsn = "Snowflake_BBR", + pwd = keyring::key_get("Upstart") + ) + + octopus::view_database(con) + +}