Handle database URLs a bit better (#1575)

* Add url-splitting pseudo option handlers to config.

* Update whats_new.rst

* Mypyage

* Lintage

* Fix tests for new behaviour.

datacube/cfg/opt.py

@@ -207,18 +207,50 @@ def validate_and_normalise(self, value: Any) -> Any:
 
     def handle_dependent_options(self, value: Any) -> None:
         if value is None:
-            for handler in (
-                    ODCOptionHandler("db_username", self.env, legacy_env_aliases=['DB_USERNAME'],
-                                     default=_DEFAULT_DB_USER),
-                    ODCOptionHandler("db_password", self.env, legacy_env_aliases=['DB_PASSWORD']),
-                    ODCOptionHandler("db_hostname", self.env, legacy_env_aliases=['DB_HOSTNAME'],
-                                     default=_DEFAULT_HOSTNAME),
-                    IntOptionHandler("db_port", self.env, default=5432, legacy_env_aliases=['DB_PORT'],
-                                     minval=1, maxval=49151),
-                    ODCOptionHandler("db_database", self.env, legacy_env_aliases=['DB_DATABASE'],
-                                     default=_DEFAULT_DATABASE),
-            ):
-                self.env._option_handlers.append(handler)
+            handlers: tuple[ODCOptionHandler, ...] = (
+                ODCOptionHandler("db_username", self.env, legacy_env_aliases=['DB_USERNAME'],
+                                 default=_DEFAULT_DB_USER),
+                ODCOptionHandler("db_password", self.env, legacy_env_aliases=['DB_PASSWORD']),
+                ODCOptionHandler("db_hostname", self.env, legacy_env_aliases=['DB_HOSTNAME'],
+                                 default=_DEFAULT_HOSTNAME),
+                IntOptionHandler("db_port", self.env, default=5432, legacy_env_aliases=['DB_PORT'],
+                                 minval=1, maxval=49151),
+                ODCOptionHandler("db_database", self.env, legacy_env_aliases=['DB_DATABASE'],
+                                 default=_DEFAULT_DATABASE),
+            )
+        else:
+            # These pseudo-handlers extract the equivalent config from the url returned by this handler.
+            handlers = (
+                PostgresURLPartHandler(self, "username", "db_username", self.env),
+                PostgresURLPartHandler(self, "password", "db_password", self.env),
+                PostgresURLPartHandler(self, "hostname", "db_hostname", self.env),
+                PostgresURLPartHandler(self, "port", "db_port", self.env),
+                PostgresURLPartHandler(self, "path", "db_database", self.env),
+            )
+
+        for handler in handlers:
+            self.env._option_handlers.append(handler)
+
+
+class PostgresURLPartHandler(ODCOptionHandler):
+    def __init__(self, urlhandler: PostgresURLOptionHandler, urlpart: str, name: str, env: "ODCEnvironment"):
+        self.urlhandler = urlhandler
+        self.urlpart = urlpart
+        super().__init__(name, env)
+
+    def validate_and_normalise(self, value: Any) -> Any:
+        url = self.env._normalised[self.urlhandler.name]
+        purl = urlparse(url)
+        part = getattr(purl, self.urlpart)
+        if self.urlpart == "path" and part.startswith('/'):
+            # Remove leading slash
+            return str(part)[1:]
+        else:
+            return part
+
+    def get_val_from_environment(self) -> str | None:
+        # Never read from environment - take from URL, wherever it came from
+        return None
 
 
 def config_options_for_psql_driver(env: "ODCEnvironment"):

docs/about/whats_new.rst

@@ -8,9 +8,14 @@ What's New
 v1.9.next
 =========
 
+v1.9.0-rc4 (15th April 2024)
+============================
+
 - Standardize resampling input supported to `odc.geo.warp.Resampling` (:pull:`1571`)
 - Refine default behaviour for config engine to support easier migration from 1.8 (:pull:`1573`)
 - Convert legacy GeoBoxes to odc.geo GeoBoxes in the core API (:pull:`1574`)
+- Add URL component pseudo to config layer to expose components to the api when configured as a URL,
+  and reformat whats_new for 1.9.0-rc4 release. (:pull:`1575`)
 
 
 v1.9.0-rc3 (27th March 2024)

tests/test_cfg.py

@@ -213,8 +213,9 @@ def test_single_env(single_env_config):
 
     assert cfg['experimental'].index_driver == "postgis"
     assert cfg['experimental'].db_url == "postgresql://foo:bar@server.subdomain.domain/mytestdb"
+    assert cfg['experimental'].db_username == "foo"
     with pytest.raises(AttributeError):
-        assert cfg['experimental'].db_username
+        assert cfg['experimental'].not_an_option
     assert cfg['experimental']['db_iam_authentication']
     assert cfg['experimental'].db_iam_timeout == 600
     assert cfg['experimental']['db_connection_timeout'] == 60
@@ -257,8 +258,9 @@ def assert_simple_options(cfg):
         assert cfg['default']["db_iam_timeout"]
 
     assert cfg['exp2'].db_url == "postgresql://foo:bar@server.subdomain.domain/mytestdb"
+    assert cfg['exp2'].db_username == "foo"
     with pytest.raises(AttributeError):
-        assert cfg['exp2'].db_username
+        assert cfg['exp2'].not_an_option
     assert cfg['exp2']['db_iam_authentication']
     assert cfg['exp2'].db_iam_timeout == 300
     assert cfg['exp2']['db_connection_timeout'] == 60
@@ -284,8 +286,7 @@ def test_noenv_overrides_in_text(simple_config, monkeypatch):
     cfg = ODCConfig(text=simple_config)
 
     assert cfg["legacy"].db_username != 'bar'
-    with pytest.raises(AttributeError):
-        cfg["experimental"].db_username
+    assert cfg["experimental"].db_username != "bar"
 
 
 @pytest.fixture
@@ -389,8 +390,7 @@ def test_envvar_overrides(path_to_yaml_config, monkeypatch):
     assert cfg["experimental"].db_iam_authentication
     assert cfg["exp2"].db_iam_authentication
     assert cfg["exp2"].db_connection_timeout == 20
-    with pytest.raises(AttributeError):
-        assert cfg["experimental"].db_username == 'bar'
+    assert cfg["experimental"].db_username != 'bar'
 
 
 def test_intopt_validation():
@@ -467,6 +467,12 @@ def test_pgurl_from_config(simple_dict):
         psql_url_from_config(
             cfg["memory"]
         )
+    assert cfg["exp2"].db_url == "postgresql://foo:bar@server.subdomain.domain/mytestdb"
+    assert cfg["exp2"].db_username == "foo"
+    assert cfg["exp2"].db_password == "bar"
+    assert cfg["exp2"].db_hostname == "server.subdomain.domain"
+    assert not cfg["exp2"].db_port
+    assert cfg["exp2"].db_database == "mytestdb"
 
     cfg = ODCConfig(raw_dict={
         "foo": {