Scale MySQL depending on the scale of the cluster

mysql-k8s-operator needs to be scaled depending on the number of
connections services need.
MySQL takes two config options:
- profile-limit-memory: how much memory to configure MySQL with
- experimental-max-connections: how many max connections to configure

If experimental-max-connections is set, the needed memory will be taken
from the profile-limit-memory. What remains will be used to configure
the respecting buffers.

Signed-off-by: Guillaume Boutry <guillaume.boutry@canonical.com>

sunbeam-python/sunbeam/commands/openstack.py

@@ -15,6 +15,7 @@
 
 import asyncio
 import logging
+import math
 from typing import Optional
 
 from lightkube.core import exceptions
@@ -56,9 +57,33 @@
 
 CONFIG_KEY = "TerraformVarsOpenstack"
 TOPOLOGY_KEY = "Topology"
+DATABASE_MEMORY_KEY = "DatabaseMemory"
 REGION_CONFIG_KEY = "Region"
 DEFAULT_REGION = "RegionOne"
 
+DATABASE_MAX_POOL_SIZE = 2
+DATABASE_ADDITIONAL_BUFFER_SIZE = 600
+DATABASE_OVERSIZE_FACTOR = 1.2
+MB_BYTES_PER_CONNECTION = 12
+
+
+# This dict maps every databases that can be used by OpenStack services
+# to the number of processes each service needs to the database.
+CONNECTIONS = {
+    "cinder": {
+        "cinder-ceph-k8s": 4,
+        "cinder-k8s": 5,
+    },
+    "glance": {"glance-k8s": 5},
+    # Horizon does not pool connections, actual value is 40
+    # but set it to 20 because of max_pool_size multiplier
+    "horizon": {"horizon-k8s": 20},
+    "keystone": {"keystone-k8s": 4},
+    "neutron": {"neutron-k8s": 4},
+    "nova": {"nova-k8s": 4 * 3},
+    "placement": {"placement-k8s": 4},
+}
+
 
 def determine_target_topology(client: Client) -> str:
     """Determines the target topology.
@@ -102,6 +127,81 @@ def compute_ingress_scale(topology: str, control_nodes: int) -> int:
     return min(control_nodes, 3)
 
 
+def compute_resources_for_service(
+    database: dict[str, int], max_pool_size: int
+) -> tuple[int, int]:
+    """Compute resources needed for a single unit service."""
+    memory_needed = 0
+    total_connections = 0
+    for connection in database.values():
+        nb_connections = max_pool_size * connection
+        total_connections += nb_connections
+        memory_needed += nb_connections * MB_BYTES_PER_CONNECTION
+    return total_connections, memory_needed
+
+
+def get_database_resource_dict(client: Client) -> dict[str, tuple[int, int]]:
+    """Returns a dict containing the resource allocation for each database service.
+
+    Resource allocation is only for a single unit service.
+    """
+    try:
+        memory_dict = read_config(client, DATABASE_MEMORY_KEY)
+    except ConfigItemNotFoundException:
+        memory_dict = {}
+    memory_dict.update(
+        {
+            service: compute_resources_for_service(connection, DATABASE_MAX_POOL_SIZE)
+            for service, connection in CONNECTIONS.items()
+        }
+    )
+
+    return memory_dict
+
+
+def write_database_resource_dict(
+    client: Client, resource_dict: dict[str, tuple[int, int]]
+):
+    """Write the resource allocation for each database service."""
+    update_config(client, DATABASE_MEMORY_KEY, resource_dict)
+
+
+def get_database_tfvars(
+    many_mysql: bool, resource_dict: dict[str, tuple[int, int]], service_scale: int
+) -> dict[str, bool | dict]:
+    """Create terraform variables related to database."""
+    tfvars: dict[str, bool | dict] = {"many-mysql": many_mysql}
+
+    if many_mysql:
+        tfvars["mysql-config-map"] = {
+            service: {
+                "profile-limit-memory": (
+                    math.ceil(memory * DATABASE_OVERSIZE_FACTOR) * service_scale
+                )
+                + DATABASE_ADDITIONAL_BUFFER_SIZE,
+                "experimental-max-connections": math.floor(
+                    connections * DATABASE_OVERSIZE_FACTOR
+                )
+                * service_scale,
+            }
+            for service, (connections, memory) in resource_dict.items()
+        }
+    else:
+        connections, memories = list(zip(*resource_dict.values()))
+        total_memory = (
+            math.ceil(sum(memories) * DATABASE_OVERSIZE_FACTOR)
+            + DATABASE_ADDITIONAL_BUFFER_SIZE
+        )
+        tfvars["mysql-config"] = {
+            "profile-limit-memory": (total_memory),
+            "experimental-max-connections": math.floor(
+                sum(connections) * DATABASE_OVERSIZE_FACTOR
+            ),
+        }
+
+    return tfvars
+
+
 class DeployControlPlaneStep(BaseStep, JujuStepHelper):
     """Deploy OpenStack using Terraform cloud."""
 
@@ -151,6 +251,13 @@ def get_storage_tfvars(self, storage_nodes: list[dict]) -> dict:
 
         return tfvars
 
+    def get_database_tfvars(self, service_scale: int) -> dict:
+        """Create terraform variables related to database."""
+        many_mysql = self.database == "multi"
+        resource_dict = get_database_resource_dict(self.client)
+        write_database_resource_dict(self.client, resource_dict)
+        return get_database_tfvars(many_mysql, resource_dict, service_scale)
+
     def get_region_tfvars(self) -> dict:
         """Create terraform variables related to region."""
         return {"region": read_config(self.client, REGION_CONFIG_KEY)["region"]}
@@ -216,17 +323,18 @@ def run(self, status: Optional[Status] = None) -> Result:
         self.update_status(status, "computing deployment sizing")
         model_config = convert_proxy_to_model_configs(self.proxy_settings)
         model_config.update({"workload-storage": K8SHelper.get_default_storageclass()})
+        service_scale = compute_os_api_scale(self.topology, len(control_nodes))
         extra_tfvars = self.get_storage_tfvars(storage_nodes)
         extra_tfvars.update(self.get_region_tfvars())
+        extra_tfvars.update(self.get_database_tfvars(service_scale))
         extra_tfvars.update(
             {
                 "model": self.model,
                 "cloud": self.cloud,
                 "credential": f"{self.cloud}{CREDENTIAL_SUFFIX}",
                 "config": model_config,
-                "many-mysql": self.database == "multi",
                 "ha-scale": compute_ha_scale(self.topology, len(control_nodes)),
-                "os-api-scale": compute_os_api_scale(self.topology, len(control_nodes)),
+                "os-api-scale": service_scale,
                 "ingress-scale": compute_ingress_scale(
                     self.topology, len(control_nodes)
                 ),

sunbeam-python/sunbeam/commands/terraform.py

@@ -362,7 +362,6 @@ def update_tfvars_and_apply_tf(
         :param tf_apply_extra_args: Extra args to terraform apply command
         :type tf_apply_extra_args: list or None
         """
-        current_tfvars = None
         updated_tfvars = {}
         if tfvar_config:
             try:

sunbeam-python/sunbeam/features/caas/feature.py

@@ -207,6 +207,12 @@ def set_tfvars_on_resize(self) -> dict:
         """Set terraform variables to resize the application."""
         return {}
 
+    def get_database_charm_processes(self) -> dict[str, dict[str, int]]:
+        """Returns the database processes accessing this service."""
+        return {
+            "magnum": {"magnum-k8s": 9},
+        }
+
     @click.command()
     def enable_feature(self) -> None:
         """Enable Container as a Service feature."""

sunbeam-python/sunbeam/features/dns/feature.py

@@ -129,6 +129,12 @@ def set_tfvars_on_resize(self) -> dict:
         """Set terraform variables to resize the application."""
         return {}
 
+    def get_database_charm_processes(self) -> dict[str, dict[str, int]]:
+        """Returns the database processes accessing this service."""
+        return {
+            "designate": {"designate-k8s": 8},
+        }
+
     @click.command()
     @click.option(
         "--nameservers",

sunbeam-python/sunbeam/features/interface/v1/openstack.py

@@ -29,7 +29,15 @@
 
 from sunbeam.clusterd.service import ConfigItemNotFoundException
 from sunbeam.commands.juju import JujuStepHelper
-from sunbeam.commands.openstack import OPENSTACK_MODEL, TOPOLOGY_KEY
+from sunbeam.commands.openstack import (
+    DATABASE_MAX_POOL_SIZE,
+    OPENSTACK_MODEL,
+    TOPOLOGY_KEY,
+    compute_resources_for_service,
+    get_database_resource_dict,
+    get_database_tfvars,
+    write_database_resource_dict,
+)
 from sunbeam.commands.terraform import (
     TerraformException,
     TerraformHelper,
@@ -210,6 +218,47 @@ def get_database_topology(self) -> str:
         topology = read_config(client, TOPOLOGY_KEY)
         return topology["database"]
 
+    def get_database_charm_processes(self) -> dict[str, dict[str, int]]:
+        """Returns the database processes accessing this service.
+
+        Example:
+        {
+            "cinder": {
+              "cinder-k8s": 4,
+              "cinder-ceph-k8s": 4,
+            }
+        }
+        """
+        return {}
+
+    def get_database_resource_tfvars(self, *, enable: bool) -> dict:
+        """Return tfvars for configuring memory for database."""
+        client = self.deployment.get_client()
+        try:
+            config = read_config(client, self.get_tfvar_config_key())
+        except ConfigItemNotFoundException:
+            config = {}
+        database_processes = self.get_database_charm_processes()
+        resource_dict = get_database_resource_dict(client)
+        if enable:
+            resource_dict.update(
+                {
+                    service: compute_resources_for_service(
+                        connection, DATABASE_MAX_POOL_SIZE
+                    )
+                    for service, connection in database_processes.items()
+                }
+            )
+        else:
+            for service in database_processes:
+                resource_dict.pop(service, None)
+        write_database_resource_dict(client, resource_dict)
+        return get_database_tfvars(
+            config.get("many-mysql", False),
+            resource_dict,
+            config.get("os-api-scale", 1),
+        )
+
     def set_application_timeout_on_enable(self) -> int:
         """Set Application Timeout on enabling the feature.
 
@@ -426,6 +475,7 @@ def run(self, status: Optional[Status] = None) -> Result:
         """Apply terraform configuration to deploy openstack application."""
         config_key = self.feature.get_tfvar_config_key()
         extra_tfvars = self.feature.set_tfvars_on_enable()
+        extra_tfvars.update(self.feature.get_database_resource_tfvars(enable=True))
 
         try:
             self.tfhelper.update_tfvars_and_apply_tf(
@@ -496,6 +546,9 @@ def run(self, status: Optional[Status] = None) -> Result:
             else:
                 # Update terraform variables to disable the application
                 extra_tfvars = self.feature.set_tfvars_on_disable()
+                extra_tfvars.update(
+                    self.feature.get_database_resource_tfvars(enable=False)
+                )
                 self.tfhelper.update_tfvars_and_apply_tf(
                     self.feature.deployment.get_client(),
                     self.feature.manifest,

sunbeam-python/sunbeam/features/loadbalancer/feature.py

@@ -85,6 +85,12 @@ def set_tfvars_on_resize(self) -> dict:
         """Set terraform variables to resize the application."""
         return {}
 
+    def get_database_charm_processes(self) -> dict[str, dict[str, int]]:
+        """Returns the database processes accessing this service."""
+        return {
+            "octavia": {"octavia-k8s": 6},
+        }
+
     @click.command()
     def enable_feature(self) -> None:
         """Enable Loadbalancer service."""

sunbeam-python/sunbeam/features/orchestration/feature.py

@@ -85,6 +85,12 @@ def set_tfvars_on_resize(self) -> dict:
         """Set terraform variables to resize the application."""
         return {}
 
+    def get_database_charm_processes(self) -> dict[str, dict[str, int]]:
+        """Returns the database processes accessing this service."""
+        return {
+            "heat": {"heat-k8s": 4},
+        }
+
     @click.command()
     def enable_feature(self) -> None:
         """Enable Orchestration service."""

sunbeam-python/sunbeam/features/secrets/feature.py

@@ -80,6 +80,12 @@ def set_tfvars_on_resize(self) -> dict:
         """Set terraform variables to resize the application."""
         return {}
 
+    def get_database_charm_processes(self) -> dict[str, dict[str, int]]:
+        """Returns the database processes accessing this service."""
+        return {
+            "barbican": {"barbican-k8s": 9},
+        }
+
     @click.command()
     def enable_feature(self) -> None:
         """Enable OpenStack Secrets service."""

sunbeam-python/sunbeam/features/telemetry/feature.py

@@ -164,6 +164,13 @@ def set_tfvars_on_resize(self) -> dict:
         """Set terraform variables to resize the application."""
         return {}
 
+    def get_database_charm_processes(self) -> dict[str, dict[str, int]]:
+        """Returns the database processes accessing this service."""
+        return {
+            "aodh": {"aodh-k8s": 4},
+            "gnocchi": {"gnocchi-k8s": 4},
+        }
+
     @click.command()
     def enable_feature(self) -> None:
         """Enable OpenStack Telemetry applications."""

sunbeam-python/sunbeam/provider/local/commands.py

@@ -87,7 +87,6 @@
     RemoveMicrok8sUnitStep,
     StoreMicrok8sConfigStep,
 )
-from sunbeam.commands.mysql import ConfigureMySQLStep
 from sunbeam.commands.openstack import (
     OPENSTACK_MODEL,
     DeployControlPlaneStep,
@@ -503,7 +502,6 @@ def bootstrap(
     plan5 = []
 
     if is_control_node:
-        plan5.append(ConfigureMySQLStep(jhelper))
         plan5.append(PatchLoadBalancerServicesStep(client))
 
     # NOTE(jamespage):

sunbeam-python/sunbeam/provider/maas/commands.py

@@ -69,7 +69,6 @@
     AddMicrok8sUnitsStep,
     StoreMicrok8sConfigStep,
 )
-from sunbeam.commands.mysql import ConfigureMySQLStep
 from sunbeam.commands.openstack import (
     OPENSTACK_MODEL,
     DeployControlPlaneStep,
@@ -652,7 +651,6 @@ def deploy(
             refresh=True,
         )
     )
-    plan2.append(ConfigureMySQLStep(jhelper))
     plan2.append(PatchLoadBalancerServicesStep(client))
     plan2.append(TerraformInitStep(tfhelper_hypervisor_deploy))
     plan2.append(