feat: allow connection via uri and addtional config parameters

feat: allow connection URI and add additional config parameters fix: dependencies fix: dont return empty objects fix: errors fix: return if both Msg and Err are empty chore: add deprecation warning, set DefaultSSLMode to prefer chore: add TODO reminder to remove BuildURI when host/port connection is deprecated docs: update with connection string docs: add a comment on top to state the context the file is used within docs: fix versions
go-spatial · Mar 24, 2022 · 75498b9 · 75498b9
1 parent 905eed6
commit 75498b9
Show file tree

Hide file tree

Showing 4 changed files with 450 additions and 79 deletions.
diff --git a/provider/postgis/README.md b/provider/postgis/README.md
@@ -6,24 +6,60 @@ The PostGIS provider manages querying for tile requests against a Postgres datab
 [[providers]]
 name = "test_postgis"       # provider name is referenced from map layers (required)
 type = "postgis"            # the type of data provider must be "postgis" for this data provider (required)
-host = "localhost"          # PostGIS database host (required)
-port = 5432                 # PostGIS database port (required)
-database = "tegola"         # PostGIS database name (required)
-user = "tegola"             # PostGIS database user (required)
-password = ""               # PostGIS database password (required)
+
+uri = "postgres://tegola:supersecret@localhost:5432/tegola?sslmode=prefer" # PostGIS connection string (required)
+
+host = "localhost"                # PostGIS database host (deprecated)
+port = 5432                       # PostGIS database port (deprecated)
+database = "tegola"               # PostGIS database name (deprecated)
+user = "tegola"                   # PostGIS database user (deprecated)
+password = "supersecret"          # PostGIS database password (deprecated)
+max_connections = 10              # PostGIS max connections (deprecated)
+max_connection_idle_time = "30m"  # PostGIS max connection idle time (deprecated)
+max_connection_lifetime = "1h"    # PostGIS max connection life time (deprecated)
 ```
 
 ### Connection Properties
 
+Establishing a connection via connection string (`uri`) will become the default connection method as of v0.16.0.
+Connecting via host/port/database is flagged for deprecation as of v0.15.0 but will be possible until v0.16.0 still.
+
+- `uri` (string): [Required] PostGIS connection string
+- `name` (string): [Required] provider name is referenced from map layers
+- `type` (string): [Required] the type of data provider. must be "postgis" to use this data provider
+- `srid` (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326)
+
+#### Connection string properties
+
+**Example**
+
+```
+# {protocol}://{user}:{password}@{host}:{port}/{database}?{options}=
+postgres://tegola:supersecret@localhost:5432/tegola?sslmode=prefer&pool_max_conns=10
+```
+
+**Options**
+
+- `sslmode`: [Optional] PostGIS SSL mode. Default: "prefer"
+- `pool_max_conns`: [Optional] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max.
+- `pool_max_conn_idle_time`: [Optional] The maximum time an idle connection is kept alive. Defaults to "30m".
+- `max_connection_lifetime` [Optional] The maximum time a connection lives before it is terminated and recreated. Defaults to "1h".
+
+### [DEPRECATED] Connection Properties
+
+- `uri` (string): [Required] PostGIS connection string
 - `name` (string): [Required] provider name is referenced from map layers
 - `type` (string): [Required] the type of data provider. must be "postgis" to use this data provider
-- `host` (string): [Required] PostGIS database host
-- `port` (int): [Required] PostGIS database port (required)
-- `database` (string): [Required] PostGIS database name
-- `user` (string): [Required] PostGIS database user
-- `password` (string): [Required] PostGIS database password
+- `host` (string): [deprecated] PostGIS database host
+- `port` (int): [deprecated] PostGIS database port (required)
+- `database` (string): [deprecated] PostGIS database name
+- `user` (string): [deprecated] PostGIS database user
+- `password` (string): [deprecated] PostGIS database password
 - `srid` (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326)
-- `max_connections` (int): [Optional] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max.
+- `ssl_mode`: (string): [Optional]. PostGIS SSL mode. Default is "prefer".
+- `max_connections` (int): [deprecated] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max.
+- `max_connection_idle_time` (duration string): [deprecated] The maximum time an idle connection is kept alive.
+- `max_connection_lifetime` (duration string): [deprecated] The maximum time a connection lives before it is terminated and recreated.
 
 ## Provider Layers
 In addition to the connection configuration above, Provider Layers need to be configured. A Provider Layer tells tegola how to query PostGIS for a certain layer. An example minimum config:

diff --git a/provider/postgis/error.go b/provider/postgis/error.go
@@ -37,3 +37,24 @@ type ErrGeomFieldNotFound struct {
 func (e ErrGeomFieldNotFound) Error() string {
 	return fmt.Sprintf("postgis: geom fieldname (%v) not found for layer (%v)", e.GeomFieldName, e.LayerName)
 }
+
+type ErrInvalidURI struct {
+	Err error
+	Msg string
+}
+
+func (e ErrInvalidURI) Error() string {
+	if e.Msg == "" {
+		if e.Err != nil {
+			return fmt.Sprintf("postgis: %v", e.Err.Error())
+		} else {
+			return "postgis: invalid uri"
+		}
+	}
+
+	return fmt.Sprintf("postgis: invalid uri (%v)", e.Msg)
+}
+
+func (e ErrInvalidURI) Unwrap() error {
+	return e.Err
+}
diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go
@@ -7,6 +7,8 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
@@ -156,34 +158,40 @@ const (
 )
 
 const (
-	DefaultPort    = 5432
-	DefaultSRID    = tegola.WebMercator
-	DefaultMaxConn = 100
-	DefaultSSLMode = "disable"
-	DefaultSSLKey  = ""
-	DefaultSSLCert = ""
+	DefaultURI             = ""
+	DefaultPort            = 5432
+	DefaultSRID            = tegola.WebMercator
+	DefaultMaxConn         = 100
+	DefaultMaxConnIdleTime = "30m"
+	DefaultMaxConnLifetime = "1h"
+	DefaultSSLMode         = "prefer"
+	DefaultSSLKey          = ""
+	DefaultSSLCert         = ""
 )
 
 const (
-	ConfigKeyHost        = "host"
-	ConfigKeyPort        = "port"
-	ConfigKeyDB          = "database"
-	ConfigKeyUser        = "user"
-	ConfigKeyPassword    = "password"
-	ConfigKeySSLMode     = "ssl_mode"
-	ConfigKeySSLKey      = "ssl_key"
-	ConfigKeySSLCert     = "ssl_cert"
-	ConfigKeySSLRootCert = "ssl_root_cert"
-	ConfigKeyMaxConn     = "max_connections"
-	ConfigKeySRID        = "srid"
-	ConfigKeyLayers      = "layers"
-	ConfigKeyLayerName   = "name"
-	ConfigKeyTablename   = "tablename"
-	ConfigKeySQL         = "sql"
-	ConfigKeyFields      = "fields"
-	ConfigKeyGeomField   = "geometry_fieldname"
-	ConfigKeyGeomIDField = "id_fieldname"
-	ConfigKeyGeomType    = "geometry_type"
+	ConfigKeyURI             = "uri"
+	ConfigKeyHost            = "host"
+	ConfigKeyPort            = "port"
+	ConfigKeyDB              = "database"
+	ConfigKeyUser            = "user"
+	ConfigKeyPassword        = "password"
+	ConfigKeySSLMode         = "ssl_mode"
+	ConfigKeySSLKey          = "ssl_key"
+	ConfigKeySSLCert         = "ssl_cert"
+	ConfigKeySSLRootCert     = "ssl_root_cert"
+	ConfigKeyMaxConn         = "max_connections"
+	ConfigKeyMaxConnIdleTime = "max_connection_idle_time"
+	ConfigKeyMaxConnLifetime = "max_connection_lifetime"
+	ConfigKeySRID            = "srid"
+	ConfigKeyLayers          = "layers"
+	ConfigKeyLayerName       = "name"
+	ConfigKeyTablename       = "tablename"
+	ConfigKeySQL             = "sql"
+	ConfigKeyFields          = "fields"
+	ConfigKeyGeomField       = "geometry_fieldname"
+	ConfigKeyGeomIDField     = "id_fieldname"
+	ConfigKeyGeomType        = "geometry_type"
 )
 
 // isSelectQuery is a regexp to check if a query starts with `SELECT`,
@@ -195,9 +203,147 @@ type hstoreOID struct {
 	hasInit bool
 }
 
+// validateURI validates for minimum requirements for a valid postgresql uri
+func validateURI(u string) error {
+	uri, err := url.Parse(u)
+	if err != nil {
+		return ErrInvalidURI{Err: err}
+	}
+
+	if uri.Scheme != "postgres" && uri.Scheme != "postgresql" {
+		return ErrInvalidURI{
+			Msg: fmt.Sprintf("invalid connection scheme (%v)", uri.Scheme),
+		}
+	}
+
+	if uri.User == nil {
+		return ErrInvalidURI{Msg: "auth credentials missing"}
+	}
+
+	host, port, err := net.SplitHostPort(uri.Host)
+	if err != nil {
+		return ErrInvalidURI{
+			Err: fmt.Errorf("splitting host port error: %w", err),
+		}
+	}
+
+	if host == "" {
+		return ErrInvalidURI{
+			Msg: fmt.Sprintf("address %v:%v: missing host in address", host, port),
+		}
+	}
+
+	if uri.Path == "" {
+		return ErrInvalidURI{Msg: "missing database"}
+	}
+
+	return nil
+}
+
+// TODO: (iwpnd) to be removed/refactored in v0.17.0
+// BuildURI creates a database URI from config
+func BuildURI(config dict.Dicter) (*url.URL, *url.Values, error) {
+
+	sslmode := DefaultSSLMode
+	sslmode, err := config.String(ConfigKeySSLMode, &sslmode)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	uri := DefaultURI
+	uri, err = config.String(ConfigKeyURI, &uri)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// if uri is set in the config, we add sslmode and return early
+	if uri != "" {
+		log.Warn("Connecting to PostGIS with host/port combination is deprecated. Please use connection string instead.")
+
+		if err := validateURI(uri); err != nil {
+			return nil, nil, err
+		}
+
+		parsedUri, err := url.Parse(uri)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// parse query to make sure sslmode is attached
+		parsedQuery, err := url.ParseQuery(parsedUri.RawQuery)
+		if err != nil {
+			return &url.URL{}, nil, err
+		}
+
+		if ok := parsedQuery.Get("sslmode"); ok == "" {
+			parsedQuery.Add("sslmode", sslmode)
+		}
+
+		parsedUri.RawQuery = parsedQuery.Encode()
+
+		return parsedUri, &parsedQuery, nil
+	}
+
+	host, err := config.String(ConfigKeyHost, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	port := DefaultPort
+	if port, err = config.Int(ConfigKeyPort, &port); err != nil {
+		return nil, nil, err
+	}
+
+	db, err := config.String(ConfigKeyDB, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	user, err := config.String(ConfigKeyUser, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	password, err := config.String(ConfigKeyPassword, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	maxcon := DefaultMaxConn
+	if maxcon, err = config.Int(ConfigKeyMaxConn, &maxcon); err != nil {
+		return nil, nil, err
+	}
+
+	idletime := DefaultMaxConnIdleTime
+	if idletime, err = config.String(ConfigKeyMaxConnIdleTime, &idletime); err != nil {
+		return nil, nil, err
+	}
+
+	lifetime := DefaultMaxConnLifetime
+	if lifetime, err = config.String(ConfigKeyMaxConnLifetime, &lifetime); err != nil {
+		return nil, nil, err
+	}
+
+	params := &url.Values{}
+	params.Add("sslmode", sslmode)
+	params.Add("pool_max_conns", fmt.Sprintf("%v", maxcon))
+	params.Add("pool_max_conn_lifetime", lifetime)
+	params.Add("pool_max_conn_idle_time", idletime)
+
+	u := &url.URL{
+		Scheme:   "postgres",
+		Host:     fmt.Sprintf("%v:%v", host, port),
+		User:     url.UserPassword(user, password),
+		Path:     db,
+		RawQuery: params.Encode(),
+	}
+
+	return u, params, nil
+}
+
 // BuildDBConfig build db config with defaults
-func BuildDBConfig(cs string) (*pgxpool.Config, error) {
-	dbconfig, err := pgxpool.ParseConfig(cs)
+func BuildDBConfig(uri string) (*pgxpool.Config, error) {
+	dbconfig, err := pgxpool.ParseConfig(uri)
 	if err != nil {
 		return nil, err
 	}
@@ -278,32 +424,12 @@ func BuildDBConfig(cs string) (*pgxpool.Config, error) {
 // 			!ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile.
 //
 func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) {
-
-	host, err := config.String(ConfigKeyHost, nil)
+	uri, params, err := BuildURI(config)
 	if err != nil {
 		return nil, err
 	}
 
-	db, err := config.String(ConfigKeyDB, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	user, err := config.String(ConfigKeyUser, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	password, err := config.String(ConfigKeyPassword, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	sslmode := DefaultSSLMode
-	sslmode, err = config.String(ConfigKeySSLMode, &sslmode)
-	if err != nil {
-		return nil, err
-	}
+	sslmode := params.Get("sslmode")
 
 	sslkey := DefaultSSLKey
 	sslkey, err = config.String(ConfigKeySSLKey, &sslkey)
@@ -323,30 +449,16 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error)
 		return nil, err
 	}
 
-	port := DefaultPort
-	if port, err = config.Int(ConfigKeyPort, &port); err != nil {
-		return nil, err
-	}
-
-	maxcon := DefaultMaxConn
-	if maxcon, err = config.Int(ConfigKeyMaxConn, &maxcon); err != nil {
-		return nil, err
+	dbconfig, err := BuildDBConfig(uri.String())
+	if err != nil {
+		return nil, fmt.Errorf("Failed while building db config: %w", err)
 	}
 
 	srid := DefaultSRID
 	if srid, err = config.Int(ConfigKeySRID, &srid); err != nil {
 		return nil, err
 	}
 
-	// TODO: allow connection string option in config
-	cs := fmt.Sprintf("postgres://%v:%v@%v:%v/%v?sslmode=%v&pool_max_conns=%v",
-		user, password, host, port, db, sslmode, maxcon)
-
-	dbconfig, err := BuildDBConfig(cs)
-	if err != nil {
-		return nil, fmt.Errorf("Failed while building db config: %w", err)
-	}
-
 	if err = ConfigTLS(sslmode, sslkey, sslcert, sslrootcert, dbconfig); err != nil {
 		return nil, err
 	}