Add some leeway for k8s reserved resources

galaxyproject · Aug 24, 2020 · ec2be20 · ec2be20
1 parent d43ad57
commit ec2be20
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 10 deletions.
diff --git a/cloudman/clusterman/cluster_templates.py b/cloudman/clusterman/cluster_templates.py
@@ -86,19 +86,24 @@ def _find_matching_vm_type(self, zone_model=None, default_vm_type=None,
         """
         vm_type = default_vm_type or self.cluster.default_vm_type
         if min_vcpus > 0 or min_ram > 0 or not vm_type.startswith(vm_family):
+            # Add some accommodation for rancher and k8s reserved resources
+            # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
+            min_vcpus += 1.0
+            min_ram *= 1.1
+
             cloud = self.context.cloudlaunch_client.infrastructure.clouds.get(
                 zone_model.region.cloud.id)
             region = cloud.regions.get(zone_model.region.region_id)
             zone = region.zones.get(zone_model.zone_id)
             default_matches = zone.vm_types.list(vm_type_prefix=vm_type)
             if default_matches:
                 default_match = default_matches[0]
-                min_vcpus = min_vcpus if min_vcpus > int(default_match.vcpus) else default_match.vcpus
+                min_vcpus = min_vcpus if min_vcpus > float(default_match.vcpus) else default_match.vcpus
                 min_ram = min_ram if min_ram > float(default_match.ram) else default_match.ram
             candidates = zone.vm_types.list(min_vcpus=min_vcpus, min_ram=min_ram,
                                             vm_type_prefix=vm_family)
             if candidates:
-                candidate_type = sorted(candidates, key=lambda x: int(x.vcpus) * float(x.ram))[0]
+                candidate_type = sorted(candidates, key=lambda x: float(x.vcpus) * float(x.ram))[0]
                 return candidate_type.name
         return vm_type
 

diff --git a/cloudman/clusterman/tests/test_cluster_api.py b/cloudman/clusterman/tests/test_cluster_api.py
@@ -565,16 +565,16 @@ class CMClusterScaleSignalTests(CMClusterNodeTestBase):
                 "labels": {
                     "alertname": "PodNotSchedulable",
                     "container": "job-container",
-                    "cpus": "93.5",
+                    "cpus": "92.5",
                     "instance": "10.42.0.19:8080",
                     "job": "kube-state-metrics",
-                    "memory": "768000000000.5",
+                    "memory": "691100000000.5",
                     "pod": "galaxy-galaxy-1596991139-245-lc4mg",
                     "severity": "warning"
                 },
                 "annotations": {
-                    "cpus": "93.5",
-                    "memory": "768000000000.5",
+                    "cpus": "92.5",
+                    "memory": "691100000000.5",
                     "message": "Cluster has unschedulable pods due to insufficient CPU or memory"
                 },
                 "startsAt": "2020-08-21T11:47:31.470370261Z",
@@ -588,16 +588,16 @@ class CMClusterScaleSignalTests(CMClusterNodeTestBase):
         "commonLabels": {
             "alertname": "PodNotSchedulable",
             "container": "job-container",
-            "cpus": "93.5",
+            "cpus": "92.5",
             "instance": "10.42.0.19:8080",
             "job": "kube-state-metrics",
-            "memory": "768000000000.5",
+            "memory": "691100000000.5",
             "pod": "galaxy-galaxy-1596991139-245-lc4mg",
             "severity": "warning"
         },
         "commonAnnotations": {
-            "cpus": "93.5",
-            "memory": "768000000000.5",
+            "cpus": "92.5",
+            "memory": "691100000000.5",
             "message": "Cluster has unschedulable pods due to insufficient CPU or memory"
         },
         "externalURL": "http://cloudman-prometheus-alertmanager.cloudman:9093",