[DPE-1086] Add new extensions/plugins (#363)

* Add new extensions/plugins

* fix plugins ordering

* increment lib patch

* update rock image hash

config.yaml

@@ -219,6 +219,82 @@ options:
     default: false
     type: boolean
     description: Enable spi extension.
+  plugin_bool_plperl_enable:
+    default: false
+    type: boolean
+    description: Enable bool_plperl extension.
+  plugin_hll_enable:
+    default: false
+    type: boolean
+    description: Enable hll extension.
+  plugin_hypopg_enable:
+    default: false
+    type: boolean
+    description: Enable hypopg extension.
+  plugin_ip4r_enable:
+    default: false
+    type: boolean
+    description: Enable ip4r extension.
+  plugin_plperl_enable:
+    default: false
+    type: boolean
+    description: Enable plperl extension.
+  plugin_jsonb_plperl_enable:
+    default: false
+    type: boolean
+    description: Enable jsonb_plperl extension.
+  plugin_orafce_enable:
+    default: false
+    type: boolean
+    description: Enable orafce extension.
+  plugin_pg_similarity_enable:
+    default: false
+    type: boolean
+    description: Enable pg_similarity extension.
+  plugin_prefix_enable:
+    default: false
+    type: boolean
+    description: Enable prefix extension.
+  plugin_rdkit_enable:
+    default: false
+    type: boolean
+    description: Enable rdkit extension.
+  plugin_tds_fdw_enable:
+    default: false
+    type: boolean
+    description: Enable tds_fdw extension.
+  plugin_icu_ext_enable:
+    default: false
+    type: boolean
+    description: Enable icu_ext extension.
+  plugin_pltcl_enable:
+    default: false
+    type: boolean
+    description: Enable pltcl extension.
+  plugin_postgis_enable:
+    default: false
+    type: boolean
+    description: Enable postgis extension.
+  plugin_address_standardizer_enable:
+    default: false
+    type: boolean
+    description: Enable address_standardizer extension.
+  plugin_postgis_raster_enable:
+    default: false
+    type: boolean
+    description: Enable postgis_raster extension.
+  plugin_address_standardizer_data_us_enable:
+    default: false
+    type: boolean
+    description: Enable address_standardizer_data_us extension.
+  plugin_postgis_tiger_geocoder_enable:
+    default: false
+    type: boolean
+    description: Enable postgis_tiger_geocoder extension.
+  plugin_postgis_topology_enable:
+    default: false
+    type: boolean
+    description: Enable postgis_topology extension.
   plugin_vector_enable:
     default: false
     type: boolean

lib/charms/postgresql_k8s/v0/postgresql.py

@@ -19,6 +19,7 @@
 Any charm using this library should import the `psycopg2` or `psycopg2-binary` dependency.
 """
 import logging
+from collections import OrderedDict
 from typing import Dict, List, Optional, Set, Tuple
 
 import psycopg2
@@ -34,10 +35,21 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 21
+LIBPATCH = 22
 
 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles"
 
+REQUIRED_PLUGINS = {
+    "address_standardizer": ["postgis"],
+    "address_standardizer_data_us": ["postgis"],
+    "jsonb_plperl": ["plperl"],
+    "postgis_raster": ["postgis"],
+    "postgis_tiger_geocoder": ["postgis", "fuzzystrmatch"],
+    "postgis_topology": ["postgis"],
+}
+DEPENDENCY_PLUGINS = set()
+for dependencies in REQUIRED_PLUGINS.values():
+    DEPENDENCY_PLUGINS |= set(dependencies)
 
 logger = logging.getLogger(__name__)
 
@@ -289,12 +301,18 @@ def enable_disable_extensions(self, extensions: Dict[str, bool], database: str =
                     cursor.execute("SELECT datname FROM pg_database WHERE NOT datistemplate;")
                     databases = {database[0] for database in cursor.fetchall()}
 
+            ordered_extensions = OrderedDict()
+            for plugin in DEPENDENCY_PLUGINS:
+                ordered_extensions[plugin] = extensions.get(plugin, False)
+            for extension, enable in extensions.items():
+                ordered_extensions[extension] = enable
+
             # Enable/disabled the extension in each database.
             for database in databases:
                 with self._connect_to_database(
                     database=database
                 ) as connection, connection.cursor() as cursor:
-                    for extension, enable in extensions.items():
+                    for extension, enable in ordered_extensions.items():
                         cursor.execute(
                             f"CREATE EXTENSION IF NOT EXISTS {extension};"
                             if enable

metadata.yaml

@@ -28,7 +28,7 @@ resources:
   postgresql-image:
     type: oci-image
     description: OCI image for PostgreSQL
-    upstream-source: ghcr.io/canonical/charmed-postgresql@sha256:899f5455fa4557b7060990880fb0ae1fa3b21c0ab2e72ad863cc16d0b3f25fee
+    upstream-source: ghcr.io/canonical/charmed-postgresql@sha256:ff74e3e8fc0e08e7b952cb69477f1a39e94e7caa156f06bbb844e752bec42f0b
 
 peers:
   database-peers:

src/charm.py

@@ -15,6 +15,7 @@
 from charms.loki_k8s.v0.loki_push_api import LogProxyConsumer
 from charms.observability_libs.v1.kubernetes_service_patch import KubernetesServicePatch
 from charms.postgresql_k8s.v0.postgresql import (
+    REQUIRED_PLUGINS,
     PostgreSQL,
     PostgreSQLEnableDisableExtensionError,
     PostgreSQLUpdateUserPasswordError,
@@ -83,6 +84,8 @@
 
 logger = logging.getLogger(__name__)
 
+EXTENSIONS_DEPENDENCY_MESSAGE = "Unsatisfied plugin dependencies. Please check the logs"
+
 # http{x,core} clutter the logs with debug messages
 logging.getLogger("httpcore").setLevel(logging.ERROR)
 logging.getLogger("httpx").setLevel(logging.ERROR)
@@ -459,10 +462,10 @@ def enable_disable_extensions(self, database: str = None) -> None:
             database: optional database where to enable/disable the extension.
         """
         spi_module = ["refint", "autoinc", "insert_username", "moddatetime"]
+        plugins_exception = {"uuid_ossp": '"uuid-ossp"'}
         original_status = self.unit.status
         extensions = {}
         # collect extensions
-        plugins_exception = {"uuid_ossp": '"uuid-ossp"'}
         for plugin in self.config.plugin_keys():
             enable = self.config[plugin]
 
@@ -473,7 +476,12 @@ def enable_disable_extensions(self, database: str = None) -> None:
                     extensions[ext] = enable
                 continue
             extension = plugins_exception.get(extension, extension)
+            if self._check_extension_dependencies(extension, enable):
+                self.unit.status = BlockedStatus(EXTENSIONS_DEPENDENCY_MESSAGE)
+                return
             extensions[extension] = enable
+        if self.is_blocked and self.unit.status.message == EXTENSIONS_DEPENDENCY_MESSAGE:
+            self.unit.status = ActiveStatus()
         if not isinstance(original_status, UnknownStatus):
             self.unit.status = WaitingStatus("Updating extensions")
         try:
@@ -483,6 +491,19 @@ def enable_disable_extensions(self, database: str = None) -> None:
         if not isinstance(original_status, UnknownStatus):
             self.unit.status = original_status
 
+    def _check_extension_dependencies(self, extension: str, enable: bool) -> bool:
+        skip = False
+        if enable and extension in REQUIRED_PLUGINS:
+            for ext in REQUIRED_PLUGINS[extension]:
+                if not self.config[f"plugin_{ext}_enable"]:
+                    skip = True
+                    logger.exception(
+                        "cannot enable %s, extension required %s to be enabled before",
+                        extension,
+                        ext,
+                    )
+        return skip
+
     def _add_members(self, event) -> None:
         """Add new cluster members.
 

src/config.py

@@ -62,8 +62,27 @@ class CharmConfig(BaseConfigModel):
     plugin_tsm_system_rows_enable: bool
     plugin_tsm_system_time_enable: bool
     plugin_uuid_ossp_enable: bool
-    plugin_vector_enable: bool
     plugin_spi_enable: bool
+    plugin_bool_plperl_enable: bool
+    plugin_hll_enable: bool
+    plugin_hypopg_enable: bool
+    plugin_ip4r_enable: bool
+    plugin_plperl_enable: bool
+    plugin_jsonb_plperl_enable: bool
+    plugin_orafce_enable: bool
+    plugin_pg_similarity_enable: bool
+    plugin_prefix_enable: bool
+    plugin_rdkit_enable: bool
+    plugin_tds_fdw_enable: bool
+    plugin_icu_ext_enable: bool
+    plugin_pltcl_enable: bool
+    plugin_postgis_enable: bool
+    plugin_address_standardizer_enable: bool
+    plugin_address_standardizer_data_us_enable: bool
+    plugin_postgis_tiger_geocoder_enable: bool
+    plugin_postgis_topology_enable: bool
+    plugin_postgis_raster_enable: bool
+    plugin_vector_enable: bool
     request_date_style: Optional[str]
     request_standard_conforming_strings: Optional[bool]
     request_time_zone: Optional[str]

tests/integration/test_plugins.py

@@ -57,6 +57,29 @@
 AUTOINC_EXTENSION_STATEMENT = "CREATE TABLE ids (id int4, idesc text);CREATE TRIGGER ids_nextid BEFORE INSERT OR UPDATE ON ids FOR EACH ROW EXECUTE PROCEDURE autoinc (id, next_id);"
 INSERT_USERNAME_EXTENSION_STATEMENT = "CREATE TABLE username_test (name text, username text not null);CREATE TRIGGER insert_usernames BEFORE INSERT OR UPDATE ON username_test FOR EACH ROW EXECUTE PROCEDURE insert_username (username);"
 MODDATETIME_EXTENSION_STATEMENT = "CREATE TABLE mdt (moddate timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL);CREATE TRIGGER mdt_moddatetime BEFORE UPDATE ON mdt FOR EACH ROW EXECUTE PROCEDURE moddatetime (moddate);"
+BOOL_PLPERL_EXTENSION_STATEMENT = "CREATE FUNCTION hello_bool(bool) RETURNS TEXT TRANSFORM FOR TYPE bool LANGUAGE plperl AS $$ my $with_world = shift; return sprintf('hello%s', $with_world ? ' world' : ''); $$;"
+HLL_EXTENSION_STATEMENT = "CREATE TABLE hll_test (users hll);"
+HYPOPG_EXTENSION_STATEMENT = "CREATE TABLE hypopg_test (id integer, val text); SELECT hypopg_create_index('CREATE INDEX ON hypopg_test (id)');"
+IP4R_EXTENSION_STATEMENT = "CREATE TABLE ip4r_test (ip ip4);"
+JSONB_PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION jsonb_plperl_test(val jsonb) RETURNS jsonb TRANSFORM FOR TYPE jsonb LANGUAGE plperl as $$ return $_[0]; $$;"
+ORAFCE_EXTENSION_STATEMENT = "SELECT add_months(date '2005-05-31',1);"
+PG_SIMILARITY_EXTENSION_STATEMENT = "SHOW pg_similarity.levenshtein_threshold;"
+PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION plperl_test(name text) RETURNS text AS $$ return $_SHARED{$_[0]}; $$ LANGUAGE plperl;"
+PREFIX_EXTENSION_STATEMENT = "SELECT '123'::prefix_range @> '123456';"
+RDKIT_EXTENSION_STATEMENT = "SELECT is_valid_smiles('CCC');"
+TDS_FDW_EXTENSION_STATEMENT = "CREATE SERVER mssql_svr FOREIGN DATA WRAPPER tds_fdw OPTIONS (servername 'tds_fdw_test', port '3306', database 'tds_fdw_test', tds_version '7.1');"
+ICU_EXT_EXTENSION_STATEMENT = (
+    'CREATE COLLATION "vat-lat" (provider = icu, locale = "la-VA-u-kn-true")'
+)
+PLTCL_EXTENSION_STATEMENT = (
+    "CREATE FUNCTION pltcl_test(integer) RETURNS integer AS $$ return $1 $$ LANGUAGE pltcl STRICT;"
+)
+POSTGIS_EXTENSION_STATEMENT = "SELECT PostGIS_Full_Version();"
+ADDRESS_STANDARDIZER_EXTENSION_STATEMENT = "SELECT num, street, city, zip, zipplus FROM parse_address('1 Devonshire Place, Boston, MA 02109-1234');"
+ADDRESS_STANDARDIZER_DATA_US_EXTENSION_STATEMENT = "SELECT house_num, name, suftype, city, country, state, unit  FROM standardize_address('us_lex', 'us_gaz', 'us_rules', 'One Devonshire Place, PH 301, Boston, MA 02109');"
+POSTGIS_TIGER_GEOCODER_EXTENSION_STATEMENT = "SELECT *  FROM standardize_address('tiger.pagc_lex', 'tiger.pagc_gaz', 'tiger.pagc_rules', 'One Devonshire Place, PH 301, Boston, MA 02109-1234');"
+POSTGIS_TOPOLOGY_STATEMENT = "SELECT topology.CreateTopology('nyc_topo', 26918, 0.5);"
+POSTGIS_RASTER_STATEMENT = "CREATE TABLE test_postgis_raster (name varchar, rast raster);"
 VECTOR_EXTENSION_STATEMENT = (
     "CREATE TABLE vector_test (id bigserial PRIMARY KEY, embedding vector(3));"
 )
@@ -106,6 +129,25 @@ async def test_plugins(ops_test: OpsTest) -> None:
             INSERT_USERNAME_EXTENSION_STATEMENT,
             MODDATETIME_EXTENSION_STATEMENT,
         ],
+        "plugin_bool_plperl_enable": BOOL_PLPERL_EXTENSION_STATEMENT,
+        "plugin_hll_enable": HLL_EXTENSION_STATEMENT,
+        "plugin_postgis_enable": POSTGIS_EXTENSION_STATEMENT,
+        "plugin_hypopg_enable": HYPOPG_EXTENSION_STATEMENT,
+        "plugin_ip4r_enable": IP4R_EXTENSION_STATEMENT,
+        "plugin_plperl_enable": PLPERL_EXTENSION_STATEMENT,
+        "plugin_jsonb_plperl_enable": JSONB_PLPERL_EXTENSION_STATEMENT,
+        "plugin_orafce_enable": ORAFCE_EXTENSION_STATEMENT,
+        "plugin_pg_similarity_enable": ORAFCE_EXTENSION_STATEMENT,
+        "plugin_prefix_enable": PREFIX_EXTENSION_STATEMENT,
+        "plugin_rdkit_enable": RDKIT_EXTENSION_STATEMENT,
+        "plugin_tds_fdw_enable": TDS_FDW_EXTENSION_STATEMENT,
+        "plugin_icu_ext_enable": ICU_EXT_EXTENSION_STATEMENT,
+        "plugin_pltcl_enable": PLTCL_EXTENSION_STATEMENT,
+        "plugin_address_standardizer_enable": ADDRESS_STANDARDIZER_EXTENSION_STATEMENT,
+        "plugin_address_standardizer_data_us_enable": ADDRESS_STANDARDIZER_DATA_US_EXTENSION_STATEMENT,
+        "plugin_postgis_tiger_geocoder_enable": POSTGIS_TIGER_GEOCODER_EXTENSION_STATEMENT,
+        "plugin_postgis_raster_enable": POSTGIS_RASTER_STATEMENT,
+        "plugin_postgis_topology_enable": POSTGIS_TOPOLOGY_STATEMENT,
         "plugin_vector_enable": VECTOR_EXTENSION_STATEMENT,
     }