ENT-14394/ENT-14395: Fixed issues with resolving AMIs spawn & infinite hanging on unreachable hosts

cf_remote/cloud_data.py

@@ -40,12 +40,17 @@
     },
     "ubuntu-24": {
         "owner_id": "099720109477",
-        "name_pattern": "ubuntu/images/hvm-ssd-gp3/ubuntu-*-{version}.*",
+        "name_pattern": "*ubuntu/images/hvm-ssd-gp3/ubuntu-*-{version}*",
+        "user": "ubuntu",
+    },
+    "ubuntu-26": {
+        "owner_id": "099720109477",
+        "name_pattern": "*ubuntu/images/hvm-ssd-gp3/ubuntu-*-{version}*",
         "user": "ubuntu",
     },
     "ubuntu": {
         "owner_id": "099720109477",
-        "name_pattern": "ubuntu/images/hvm-ssd/ubuntu-*-{version}.*",
+        "name_pattern": "*ubuntu/images/hvm-ssd/ubuntu-*-{version}*",
         "user": "ubuntu",
     },
     "centos": {

cf_remote/spawn.py

@@ -351,12 +351,10 @@ def _get_image_criteria(platform_name):
     platform_parts = platform_name.split("-")
     platform = platform_parts[0]
     if platform == "ubuntu":
-        if len(platform_parts) == 2:
-            platform_version = platform_parts[1]
-        elif len(platform_parts) > 2:
+        if platform_parts[-1] in ("x64", "arm64"):
             platform_version = ".".join(platform_parts[1:-1])
         else:
-            platform_version = ""
+            platform_version = ".".join(platform_parts[1:3])
     else:
         platform_version = platform_name.count("-") > 0 and platform_parts[1] or "*"
     log.debug(

cf_remote/ssh.py

@@ -1,19 +1,45 @@
 import os
-import sys
 import pwd
 import shutil
 import signal
+import socket
 import subprocess
 from typing import Union
 from urllib.parse import urlparse
 
 from cf_remote import aramid
 from cf_remote import log
 from cf_remote import paths
-from cf_remote.utils import whoami, read_json
+from cf_remote.utils import whoami, read_json, CFRUserError
 from cf_remote.aramid import ExecutionResult
 from cf_remote.paths import SSH_CONFIG_FPATH, SSH_CONFIGS_JSON_FPATH, CLOUD_STATE_FPATH
 
+_PREFLIGHT_TIMEOUT = 5  # seconds
+_PREFLIGHT_MAX_RETRIES = 5
+
+
+class UnreachableHostError(aramid.AramidError):
+    pass
+
+
+def _check_reachable(
+    host, port, timeout=_PREFLIGHT_TIMEOUT, max_retries=_PREFLIGHT_MAX_RETRIES
+):
+    tries = 0
+    err = ""
+    while tries < max_retries:
+        try:
+            with socket.create_connection((host, port), timeout=timeout):
+                return
+        except OSError as e:  # timeout/no-route-to-host
+            tries += 1
+            err = e
+            pass
+
+    raise UnreachableHostError(
+        "Host '%s' is unreachable on port %s: %s" % (host, port, err)
+    )
+
 
 class LocalConnection:
     is_local = True
@@ -59,6 +85,11 @@ def __init__(self, host, user, connect_kwargs=None, port=aramid._DEFAULT_SSH_POR
         self.ssh_port = port
         self.ssh_user = user
         self._connect_kwargs = connect_kwargs
+        self._ssh_control_master = None
+
+        # Fail fast, before starting the Control Master or entering run()'s retry loop.
+        log.debug("Checking that '%s:%s' is reachable" % (host, port))
+        _check_reachable(host, port)
 
         # Create an SSH Control Master process (man:ssh_config(5)) so that
         # commands run on this host can reuse the same SSH connection.
@@ -99,7 +130,10 @@ def run(self, command, hide=False):
 
         # If the Control Master process is running (poll() returns None), let's
         # reuse its connection.
-        if self._ssh_control_master.poll() is None:
+        if (
+            self._ssh_control_master is not None
+            and self._ssh_control_master.poll() is None
+        ):
             log.debug("Control Master is running, using it")
             extra_ssh_args.extend(["-oControlPath=%s" % self._control_path])
 
@@ -199,9 +233,15 @@ def connect(host, users=None):
             c.ssh_port = port
             c.run("whoami", hide=True)
             return c
+        except UnreachableHostError as e:
+            # Host is down, trying other usernames won't help. Must raise
+            # rather than sys.exit(): install() calls connect() inside a
+            # multiprocessing.dummy.Pool worker thread, where a SystemExit
+            # is silently swallowed and pool.map() hangs forever instead.
+            raise CFRUserError(str(e)) from e
         except aramid.ExecutionError:
             continue
-    sys.exit("Could not ssh into '%s'" % host)
+    raise CFRUserError("Could not ssh into '%s'" % host)
 
 
 # Decorator to make a function automatically connect