Merge pull request #2541 from infotroph/bety-parms-from-env

Read Postgres settings from environment variables

CHANGELOG.md

@@ -35,6 +35,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha
 - Models monitoring container for Docker now shows a webpage with models it has seen
 - Added small container to check if certain services are up, used as initi container for kubernetes
 - Documentation how to run ED using singularity
+- PEcAn.DB gains new function `get_postgres_envvars`, which tries to look up connection parameters from Postgres environment variables (if they are set) and return them as a list ready to be passed to `db.open`. It should be especially useful when writing tests that need to run on systems with many different database configurations (#2541).
 
 ## [1.7.1] - 2018-09-12
 

base/db/NAMESPACE

@@ -30,6 +30,7 @@ export(fancy_scientific)
 export(get.id)
 export(get.trait.data)
 export(get.trait.data.pft)
+export(get_postgres_envvars)
 export(get_run_ids)
 export(get_users)
 export(get_var_names)

base/db/R/get_postgres_envvars.R

@@ -0,0 +1,75 @@
+#' Look up Postgres connection parameters from environment variables
+#'
+#' Retrieves database connection parameters stored in any of the
+#'  environment variables known by Postgres,
+#'  using defaults from `...` for parameters not set in the environment.
+#'  In a standard PEcAn installation only a few of these parameters
+#'  will ever be set, but we check all of them anyway in case you need to do
+#'  anything unusual.
+#'
+#' The list of environment variables we check is taken from the
+#'  [Postgres 12 manual](https://postgresql.org/docs/12/libpq-envars.html),
+#'  but it should apply to older Postgres versions as well.
+#'  Note that this function only looks for environment variables that control
+#'  connection parameters; it does not retrieve any of the variables related to
+#'  per-session behavior (e.g. PGTZ, PGSYSCONFDIR).
+#'
+#' @param ... defaults for parameters not found in the environment,
+#'  in `name = value` form
+#' @return list of connection parameters suitable for passing on to `db.open`
+#'
+#' @examples
+#'  host <- Sys.getenv("PGHOST") # to restore environment after demo
+#'
+#'  Sys.unsetenv("PGHOST")
+#'  get_postgres_envvars()$host # NULL
+#'  get_postgres_envvars(host = "default", port = 5432)$host # "default"
+
+#'  # defaults are ignored for a variable that exists
+#'  Sys.setenv(PGHOST = "localhost")
+#'  get_postgres_envvars()$host # "localhost"
+#'  get_postgres_envvars(host = "postgres")$host # still "localhost"
+#'
+#'  # To override a set variable, edit the returned list before using it
+#'  con_parms <- get_postgres_envvars()
+#'  con_parms$host # "localhost"
+#'  con_parms$host <- "postgres"
+#'  # db.open(con_parms)
+#'
+#'  Sys.setenv(PGHOST = host)
+#' @export
+get_postgres_envvars <- function(...) {
+  pg_vars <- list(
+    host = "PGHOST",
+    hostaddr = "PGHOSTADDR",
+    port = "PGPORT",
+    dbname = "PGDATABASE",
+    user = "PGUSER",
+    password = "PGPASSWORD",
+    passfile = "PGPASSFILE",
+    service = "PGSERVICE",
+    options = "PGOPTIONS",
+    application_name = "PGAPPNAME",
+    ssl_mode = "PGSSLMODE",
+    requiressl = "PGREQUIRESSL",
+    sslcompression = "PGSSLCOMPRESSION",
+    sslcert = "PGSSLCERT",
+    sslkey = "PGSSLKEY",
+    sslrootcert = "PGSSLROOTCERT",
+    sslcrl = "PGSSLCRL",
+    requirepeer = "PGREQUIREPEER",
+    krbsrvname = "PGKRBSRVNAME",
+    gsslib = "PGGSSLIB",
+    connect_timeout = "PGCONNECT_TIMEOUT",
+    client_encoding = "PGCLIENTENCODING",
+    target_session_attrs = "PGTARGETSESSIONATTRS")
+
+  vals <- Sys.getenv(pg_vars)
+  names(vals) <- names(pg_vars)
+  vals <- vals[vals != ""]
+
+  defaults <- list(...)
+  defaults <- defaults[!(names(defaults) %in% names(vals))]
+
+  append(vals, defaults)
+}

base/db/tests/testthat.R

@@ -10,7 +10,12 @@ library(testthat)
 
 library(PEcAn.DB)
 library(RPostgreSQL)
-dbparms <- list(host = "localhost", driver = "PostgreSQL", user = "bety", dbname = "bety", password = "bety")
+dbparms <- get_postgres_envvars(
+	host = "localhost",
+	driver = "PostgreSQL",
+	user = "bety",
+	dbname = "bety",
+	password = "bety")
 
 if(db.exists(dbparms)){
   con <- db.open(dbparms)

base/db/tests/testthat/helper-db-setup.R

@@ -10,22 +10,35 @@ get_db_params <- function() {
   #                                host = "localhost",
   #                                port = 5432))
   # ```
+  # OR by setting Postgres environment parameters in your shell:
+  # ```
+  # export PGHOST=localhost
+  # export PGUSER=bety
+  # [etc]
+  # ```
   option_params <- getOption("pecan.db.params")
   # Check if running on continuous integration (CI)
   # If yes, skip this test
-  is_ci <- Sys.getenv('CI') != ''
+  is_ci <- Sys.getenv("CI") != ""
   if (!is.null(option_params)) {
     return(option_params)
   } else if (is_ci) {
-    return(list(host = "localhost", user = "bety", password = "bety",
-                driver = "Postgres"))
+    return(get_postgres_envvars(
+      host = "localhost",
+      user = "bety",
+      password = "bety",
+      driver = "Postgres"))
   } else {
     if (PEcAn.remote::fqdn() == "pecan2.bu.edu") {
       return(list(host = "psql-pecan.bu.edu", driver = "PostgreSQL",
                   dbname = "bety", user = "bety", password = "bety"))
     } else {
-      return(list(host = "localhost", driver = "PostgreSQL",
-                  user = "bety", dbname = "bety", password = "bety"))
+      return(get_postgres_envvars(
+        host = "localhost",
+        driver = "PostgreSQL",
+        user = "bety",
+        dbname = "bety",
+        password = "bety"))
     }
   }
 }

base/settings/tests/testthat/helper-get.test.settings.R

@@ -9,6 +9,14 @@
   },
   silent = TRUE)
 
+  # NB environment variables override values in XML here!
+  # This is opposite of usual PEcAn rule that XML values always win,
+  # but useful here because it allows testing on systems where we
+  # don't know the database configuration in advance
+  settings$database$bety <- do.call(
+    PEcAn.DB::get_postgres_envvars,
+    settings$database$bety)
+
   if (is.null(settings)) {
     skip("Can't get a valid test Settings right now. Skipping test. ")
   }

models/biocro/tests/testthat/test-run.biocro.R

@@ -10,6 +10,9 @@ ref_mat <- mean(ref_met$Temp)
 # run setup
 metpath <- "data/US-Bo1"
 settings <- PEcAn.settings::read.settings("data/pecan.biocro.xml")
+settings$database$bety <- do.call(
+  PEcAn.DB::get_postgres_envvars,
+  settings$database$bety)
 config <- PEcAn.settings::prepare.settings(settings)
 config$pft$type$genus <- "Salix"
 config$run$start.date <- as.POSIXct("2004-01-01")

models/biocro/tests/testthat/test.model2netcdf.BIOCRO.R

@@ -6,6 +6,9 @@ dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 file.copy(from = "data/result.RData", to = outdir)
 
 settings <- PEcAn.settings::read.settings("data/pecan.biocro.xml")
+settings$database$bety <- do.call(
+  PEcAn.DB::get_postgres_envvars,
+  settings$database$bety)
 
 start_date <- settings$run$start.date
 

models/biocro/tests/testthat/test.write.configs.BIOCRO.R

@@ -2,6 +2,9 @@ context("checking write.configs.BIOCRO")
 
 settings.xml <- file.path("data", "pecan.biocro.xml")
 settings <- PEcAn.settings::read.settings(settings.xml)
+settings$database$bety <- do.call(
+  PEcAn.DB::get_postgres_envvars,
+  settings$database$bety)
 
 testthat::skip_if_not(PEcAn.DB::db.exists(settings[[c("database", "bety")]]))
 

modules/benchmark/tests/testthat/test-align_pft.R

@@ -1,8 +1,10 @@
 
 context("align_pft")
 
-con <- PEcAn.DB::db.open(
-  list(host = "localhost", user = "bety", password = "bety"))
+con <- PEcAn.DB::db.open(PEcAn.DB::get_postgres_envvars(
+    host = "localhost",
+    user = "bety",
+    password = "bety"))
 teardown(PEcAn.DB::db.close(con))
 
 observation_one <- c("AMCA3", "AMCA3", "AMCA3", "AMCA3")

modules/data.land/tests/testthat/test.PFT_consistency.R

@@ -8,11 +8,16 @@ library(PEcAn.settings)
 library(PEcAn.DB)
 library(RPostgreSQL)
 
-betyparms <- list(host = 'localhost', dbname = 'bety', 
-                user = 'bety', password = 'bety', driver = 'PostgreSQL', write = FALSE)
-fiaparms <- list(host = 'localhost', dbname = 'fia5data', 
-                user = 'bety', password = 'bety', driver = 'PostgreSQL')
-if(db.exists(params = betyparms) & db.exists(fiaparms)){
+betyparms <- PEcAn.DB::get_postgres_envvars(
+  host = "localhost",
+  dbname = "bety",
+  user = "bety",
+  password = "bety",
+  driver = "PostgreSQL",
+  write = FALSE)
+fiaparms <- betyparms
+fiaparms$dbname <- "fia5data"
+if (db.exists(params = betyparms) & db.exists(fiaparms)) {
 
 
   context("Testing consistency of FIA PFTs")

scripts/add.util.sh

@@ -7,9 +7,14 @@ if [ -z "$FQDN" ]; then
 	FQDN=$( hostname -f )
 fi
 
+# postgres hostname
+if [ -z "$PGHOST" ]; then
+  PGHOST="localhost"
+fi
+
 # command to execute to add items to BETY database
 if [ -z "$PSQL" ]; then
-	PSQL="psql -h localhost -U bety bety -q -t -c"
+	PSQL="psql -h ${PGHOST} -U bety bety -q -t -c"
 fi
 
 # folder to data, this is assumed to be installed at the same level