fix(gce): remove libcloud usage from sdcm.utils.common

since we have feature we want to use that aren't supported
by libcloud for gce, we want to remove all usages of it.
this commit remove the usage from `sdcm.utils.common` and all
the code depending on it.

sct.py

@@ -101,7 +101,7 @@
     send_perf_email
 from sdcm.parallel_timeline_report.generate_pt_report import ParallelTimelinesReportGenerator
 from sdcm.utils.aws_utils import AwsArchType
-from sdcm.utils.gce_utils import SUPPORTED_PROJECTS
+from sdcm.utils.gce_utils import SUPPORTED_PROJECTS, gce_public_addresses
 from sdcm.utils.context_managers import environment
 from sdcm.cluster_k8s import mini_k8s
 from sdcm.utils.es_index import create_index, get_mapping
@@ -440,7 +440,7 @@ def list_resources(ctx, user, test_id, get_all, get_all_running, verbose):
         gke_table.align = "l"
         gke_table.sortby = 'CreateTime'
         for cluster in gke_clusters:
-            tags = gce_meta_to_dict(cluster.extra['metadata'])
+            tags = gce_meta_to_dict(cluster.metadata)
             gke_table.add_row([cluster.name,
                                cluster.zone,
                                tags.get('TestId', 'N/A') if tags else "N/A",
@@ -460,14 +460,15 @@ def list_resources(ctx, user, test_id, get_all, get_all_running, verbose):
                 gce_table.align = "l"
                 gce_table.sortby = 'LaunchTime'
                 for instance in gce_instances:
-                    tags = gce_meta_to_dict(instance.extra['metadata'])
-                    public_ips = ", ".join(instance.public_ips) if None not in instance.public_ips else "N/A"
+                    tags = gce_meta_to_dict(instance.metadata)
+                    public_ips = gce_public_addresses(instance)
+                    public_ips = ", ".join(public_ips) if None not in public_ips else "N/A"
                     gce_table.add_row([instance.name,
-                                       instance.extra["zone"].name,
-                                       public_ips if get_all_running else instance.state,
+                                       instance.zone.split('/')[-1],
+                                       public_ips if get_all_running else instance.status,
                                        tags.get('TestId', 'N/A') if tags else "N/A",
                                        tags.get('RunByUser', 'N/A') if tags else "N/A",
-                                       instance.extra['creationTimestamp'],
+                                       instance.creation_timestamp,
                                        ])
                 click.echo(gce_table.get_string(title="Resources used on GCE"))
             else:
@@ -480,7 +481,7 @@ def list_resources(ctx, user, test_id, get_all, get_all_running, verbose):
         eks_table.align = "l"
         eks_table.sortby = 'CreateTime'
         for cluster in eks_clusters:
-            tags = gce_meta_to_dict(cluster.extra['metadata'])
+            tags = gce_meta_to_dict(cluster.metadata)
             eks_table.add_row([cluster.name,
                                tags.get('TestId', 'N/A') if tags else "N/A",
                                cluster.region_name,

sdcm/cluster.py

@@ -3247,7 +3247,7 @@ def get_backtraces(self):
     def node_setup(self, node, verbose=False, timeout=3600):
         raise NotImplementedError("Derived class must implement 'node_setup' method!")
 
-    def get_node_ips_param(self, public_ip=True):
+    def public_ips(self, public_ip=True):
         raise NotImplementedError("Derived class must implement 'get_node_ips_param' method!")
 
     def wait_for_init(self):

sdcm/logcollector.py

@@ -57,6 +57,7 @@
 from sdcm.utils.get_username import get_username
 from sdcm.utils.remotewebbrowser import RemoteBrowser, WebDriverContainerMixin
 from sdcm.utils.s3_remote_uploader import upload_remote_files_directly_to_s3
+from sdcm.utils.gce_utils import gce_public_addresses
 
 LOGGER = logging.getLogger(__name__)
 
@@ -1417,44 +1418,48 @@ def get_aws_instances_by_testid(self):
                 global_ip=self.get_aws_ip_address(instance),
                 tags={**self.tags, "NodeType": "loader", }))
 
+    @staticmethod
+    def gce_first_public_ip(instance):
+        return gce_public_addresses(instance)[0]
+
     def get_gce_instances_by_testid(self):
         instances = list_instances_gce({"TestId": self.test_id}, running=True)
         filtered_instances = filter_gce_instances_by_type(instances)
         for instance in filtered_instances['db_nodes']:
             self.db_cluster.append(CollectingNode(name=instance.name,
                                                   ssh_login_info={
-                                                      "hostname": instance.public_ips[0],
+                                                      "hostname": self.gce_first_public_ip(instance),
                                                       "user": self.params.get('gce_image_username'),
                                                       "key_file": self.params.get('user_credentials_path')},
                                                   instance=instance,
-                                                  global_ip=instance.public_ips[0],
+                                                  global_ip=self.gce_first_public_ip(instance),
                                                   tags={**self.tags, "NodeType": "scylla-db", }))
         for instance in filtered_instances['monitor_nodes']:
             self.monitor_set.append(CollectingNode(name=instance.name,
                                                    ssh_login_info={
-                                                       "hostname": instance.public_ips[0],
+                                                       "hostname": self.gce_first_public_ip(instance),
                                                        "user": self.params.get('gce_image_username'),
                                                        "key_file": self.params.get('user_credentials_path')},
                                                    instance=instance,
-                                                   global_ip=instance.public_ips[0],
+                                                   global_ip=self.gce_first_public_ip(instance),
                                                    tags={**self.tags, "NodeType": "monitor", }))
         for instance in filtered_instances['loader_nodes']:
             self.loader_set.append(CollectingNode(name=instance.name,
                                                   ssh_login_info={
-                                                      "hostname": instance.public_ips[0],
+                                                      "hostname": self.gce_first_public_ip(instance),
                                                       "user": self.params.get('gce_image_username'),
                                                       "key_file": self.params.get('user_credentials_path')},
                                                   instance=instance,
-                                                  global_ip=instance.public_ips[0],
+                                                  global_ip=self.gce_first_public_ip(instance),
                                                   tags={**self.tags, "NodeType": "loader", }))
         for instance in filtered_instances['kubernetes_nodes']:
             self.kubernetes_set.append(CollectingNode(name=instance.name,
                                                       ssh_login_info={
-                                                          "hostname": instance.public_ips[0],
+                                                          "hostname": self.gce_first_public_ip(instance),
                                                           "user": self.params.get('gce_image_username'),
                                                           "key_file": self.params.get('user_credentials_path')},
                                                       instance=instance,
-                                                      global_ip=instance.public_ips[0],
+                                                      global_ip=self.gce_first_public_ip(instance),
                                                       tags={**self.tags, "NodeType": "loader", }))
         if self.params.get("use_cloud_manager"):
             self.find_and_append_cloud_manager_instance_to_collecting_nodes()
@@ -1490,11 +1495,11 @@ def get_docker_instances_by_testid(self):
         for instance in filtered_instances['db_nodes']:
             self.db_cluster.append(CollectingNode(name=instance.name,
                                                   ssh_login_info={
-                                                      "hostname": instance.public_ips[0],
+                                                      "hostname": self.gce_first_public_ip(instance),
                                                       "user": 'scylla-test',
                                                       "key_file": self.params.get('user_credentials_path')},
                                                   instance=instance,
-                                                  global_ip=instance.public_ips[0],
+                                                  global_ip=self.gce_first_public_ip(instance),
                                                   tags={**self.tags, "NodeType": "scylla-db", }))
         self.monitor_set.append(CollectingNode(name=f"monitor-node-{self.test_id}-0",
                                                global_ip='127.0.0.1',
@@ -1503,11 +1508,11 @@ def get_docker_instances_by_testid(self):
         for instance in filtered_instances['loader_nodes']:
             self.loader_set.append(CollectingNode(name=instance.name,
                                                   ssh_login_info={
-                                                      "hostname": instance.public_ips[0],
+                                                      "hostname": self.gce_first_public_ip(instance),
                                                       "user": 'scylla-test',
                                                       "key_file": self.params.get('user_credentials_path')},
                                                   instance=instance,
-                                                  global_ip=instance.public_ips[0],
+                                                  global_ip=self.gce_first_public_ip(instance),
                                                   tags={**self.tags, "NodeType": "loader", }))
 
     def get_running_cluster_sets(self, backend):

sdcm/sct_config.py

@@ -1681,7 +1681,7 @@ def __init__(self):
                     # gce_image.name format examples: scylla-4-3-6 or scylla-enterprise-2021-1-2
                     gce_image = get_scylla_gce_images_versions(version=scylla_version)[0]
                 self.log.debug("Found GCE image %s for scylla_version='%s'", gce_image.name, scylla_version)
-                self["gce_image_db"] = gce_image.extra["selfLink"]
+                self["gce_image_db"] = gce_image.self_link
             elif not self.get("azure_image_db") and self.get("cluster_backend") == "azure":
                 scylla_azure_images = []
                 if isinstance(self.get('azure_region_name'), list):

sdcm/sct_runner.py

@@ -800,16 +800,16 @@ def list_sct_runners(cls, verbose: bool = True) -> list[SctRunnerInfo]:
             with environment(SCT_GCE_PROJECT=project):
                 instances += list_instances_gce(tags_dict={"NodeType": cls.NODE_TYPE}, verbose=verbose)
         for instance in instances:
-            tags = gce_meta_to_dict(instance.extra["metadata"])
-            region = instance.extra["zone"].name
+            tags = gce_meta_to_dict(instance.metadata)
+            region = instance.zone.split('/')[-1]
             if launch_time := tags.get("launch_time"):
                 try:
                     launch_time = datetime_from_formatted(date_string=launch_time)
                 except ValueError as exc:
                     LOGGER.warning("Value of `launch_time' tag is invalid: %s", exc)
                     launch_time = None
             if not launch_time:
-                create_time = instance.extra["creationTimestamp"]
+                create_time = instance.creation_timestamp
                 LOGGER.info("`launch_time' tag is empty or invalid, fallback to creation time: %s", create_time)
                 launch_time = datetime.datetime.fromisoformat(create_time)
             sct_runners.append(SctRunnerInfo(
@@ -818,7 +818,7 @@ def list_sct_runners(cls, verbose: bool = True) -> list[SctRunnerInfo]:
                 region_az=region,
                 instance=instance,
                 instance_name=instance.name,
-                public_ips=instance.public_ips,
+                public_ips=instance.network_interfaces[0].access_configs[0].nat_i_p,
                 test_id=tags.get("TestId"),
                 launch_time=launch_time,
                 keep=tags.get("keep"),

sdcm/send_email.py

@@ -29,6 +29,7 @@
 
 from sdcm.keystore import KeyStore
 from sdcm.utils.common import list_instances_gce, list_instances_aws, list_resources_docker, format_timestamp
+from sdcm.utils.gce_utils import gce_public_addresses
 
 LOGGER = logging.getLogger(__name__)
 
@@ -646,13 +647,13 @@ def get_running_instances_for_email_report(test_id: str, ip_filter: str = None):
                               region])
     instances = list_instances_gce(tags_dict=tags, running=True)
     for instance in instances:
-        public_ips = instance.public_ips
+        public_ips = gce_public_addresses(instance)
         if ip_filter not in public_ips:
             nodes.append([instance.name,
-                          ", ".join(public_ips) if None not in instance.public_ips else "N/A",
-                          instance.state,
+                          ", ".join(public_ips) if public_ips else "N/A",
+                          instance.status.lower(),
                           "gce",
-                          instance.extra["zone"].name])
+                          instance.zone.split('/')[-1]])
     resources = list_resources_docker(tags_dict=tags, running=True, group_as_builder=True)
     for builder_name, containers in resources.get("containers", {}).items():
         for container in containers:

sdcm/utils/cloud_monitor/resources/instances.py

@@ -3,6 +3,7 @@
 
 from boto3 import client as boto3_client
 from azure.mgmt.compute.models import VirtualMachine
+from google.cloud.compute_v1.types import Instance as GceInstance
 
 from sdcm.utils.azure_utils import AzureService
 from sdcm.utils.cloud_monitor.common import InstanceLifecycle, NA
@@ -64,36 +65,36 @@ def get_owner(self):
 class GCEInstance(CloudInstance):
     pricing = GCEPricing()
 
-    def __init__(self, instance):
-        tags = gce_meta_to_dict(instance.extra['metadata'])
-        is_preemptible = instance.extra["scheduling"]["preemptible"]
+    def __init__(self, instance: GceInstance):
+        tags = gce_meta_to_dict(instance.metadata)
+        is_preemptible = instance.scheduling.preemptible
         super().__init__(
             cloud="gce",
             name=instance.name,
             instance_id=instance.id,
-            region_az=instance.extra["zone"].name,
-            state=str(instance.state),
+            region_az=instance.zone.split('/')[-1],
+            state=str(instance.status.lower()),
             lifecycle=InstanceLifecycle.SPOT if is_preemptible else InstanceLifecycle.ON_DEMAND,
-            instance_type=instance.size,
+            instance_type=instance.machine_type.split('/')[-1],
             owner=tags.get("RunByUser", NA) if tags else NA,
-            create_time=datetime.fromisoformat(instance.extra['creationTimestamp']),
+            create_time=datetime.fromisoformat(instance.creation_timestamp),
             keep=self.get_keep_alive_gce_instance(instance),
-            project=instance.driver.project
+            project=instance.self_link.split('/')[6]
         )
 
     @property
     def region(self):
         return self.region_az[:-2]
 
     @staticmethod
-    def get_keep_alive_gce_instance(instance):
+    def get_keep_alive_gce_instance(instance: GceInstance):
         # same logic as in cloud instance stopper
         # checking labels
-        labels = instance.extra["labels"]
+        labels = instance.labels
         if labels:
             return labels.get("keep", labels.get("keep-alive", ""))
         # checking tags
-        tags = instance.extra["tags"]
+        tags = instance.tags.items
         if tags:
             return "alive" if 'alive' in tags or 'keep-alive' in tags or 'keep' in tags else ""
         return ""

sdcm/utils/cloud_monitor/resources/static_ips.py

@@ -1,5 +1,5 @@
 from logging import getLogger
-from libcloud.compute.drivers.gce import GCEAddress
+from google.cloud import compute_v1
 
 from sdcm.utils.azure_utils import AzureService
 from sdcm.utils.cloud_monitor.common import NA
@@ -43,13 +43,13 @@ def __init__(self, eip, region):
 
 
 class GceStaticIP(StaticIP):  # pylint: disable=too-few-public-methods
-    def __init__(self, static_ip: GCEAddress):
-        used_by = static_ip.extra.get("users")
+    def __init__(self, static_ip: compute_v1.Address):
+        used_by = static_ip.users
         super().__init__(
             cloud="gce",
             name=static_ip.name,
             address=static_ip.address,
-            region=static_ip.region if isinstance(static_ip.region, str) else static_ip.region.name,
+            region=static_ip.region,
             used_by=used_by[0].rsplit("/", maxsplit=1)[-1] if used_by else NA,
             owner=NA  # currently unsupported, maybe we can store it in description in future
         )