diff --git a/docs/changelog/123272.yaml b/docs/changelog/123272.yaml
new file mode 100644
index 0000000000000..18db2437f85c1
--- /dev/null
+++ b/docs/changelog/123272.yaml
@@ -0,0 +1,5 @@
+pr: 123272
+summary: Set Connect Timeout to 5s
+area: Machine Learning
+type: bug
+issues: []
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
index f0102d01b37a1..f890b6fce13d0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.external.http;
 
 import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.concurrent.FutureCallback;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
@@ -56,14 +57,18 @@ public static HttpClient create(
         PoolingNHttpClientConnectionManager connectionManager,
         ThrottlerManager throttlerManager
     ) {
-        CloseableHttpAsyncClient client = createAsyncClient(Objects.requireNonNull(connectionManager));
+        var client = createAsyncClient(Objects.requireNonNull(connectionManager), Objects.requireNonNull(settings));
 
         return new HttpClient(settings, client, threadPool, throttlerManager);
     }
 
-    private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConnectionManager connectionManager) {
-        HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create();
-        clientBuilder.setConnectionManager(connectionManager);
+    private static CloseableHttpAsyncClient createAsyncClient(
+        PoolingNHttpClientConnectionManager connectionManager,
+        HttpSettings settings
+    ) {
+        var requestConfig = RequestConfig.custom().setConnectTimeout(settings.connectionTimeout()).build();
+
+        var clientBuilder = HttpAsyncClientBuilder.create().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig);
         // The apache client will be shared across all connections because it can be expensive to create it
         // so we don't want to support cookies to avoid accidental authentication for unauthorized users
         clientBuilder.disableCookieManagement();
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java
index b2825d1b79cbf..c8d5b36a319ec 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.TimeValue;
 
 import java.util.List;
 import java.util.Objects;
@@ -27,12 +28,21 @@ public class HttpSettings {
         Setting.Property.Dynamic
     );
 
+    // The time we wait for a connection to establish
+    public static final Setting<TimeValue> CONNECTION_TIMEOUT = Setting.timeSetting(
+        "xpack.inference.http.connect_timeout",
+        TimeValue.timeValueSeconds(5),
+        Setting.Property.NodeScope
+    );
+
     private volatile ByteSizeValue maxResponseSize;
+    private final int connectionTimeout;
 
     public HttpSettings(Settings settings, ClusterService clusterService) {
         Objects.requireNonNull(clusterService);
         Objects.requireNonNull(settings);
         maxResponseSize = MAX_HTTP_RESPONSE_SIZE.get(settings);
+        connectionTimeout = Math.toIntExact(CONNECTION_TIMEOUT.get(settings).getMillis());
 
         clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_HTTP_RESPONSE_SIZE, this::setMaxResponseSize);
     }
@@ -41,11 +51,15 @@ public ByteSizeValue getMaxResponseSize() {
         return maxResponseSize;
     }
 
+    public int connectionTimeout() {
+        return connectionTimeout;
+    }
+
     private void setMaxResponseSize(ByteSizeValue maxResponseSize) {
         this.maxResponseSize = maxResponseSize;
     }
 
     public static List<Setting<?>> getSettingsDefinitions() {
-        return List.of(MAX_HTTP_RESPONSE_SIZE);
+        return List.of(MAX_HTTP_RESPONSE_SIZE, CONNECTION_TIMEOUT);
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java
index 4061e78c31dc4..b55ae9f76cad1 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java
@@ -29,7 +29,6 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.xpack.core.inference.action.InferenceAction.Request.DEFAULT_TIMEOUT;
 import static org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService.ELASTIC_INFERENCE_SERVICE_IDENTIFIER;
 
 /**
@@ -39,6 +38,7 @@ public class ElasticInferenceServiceAuthorizationHandler {
 
     private static final String FAILED_TO_RETRIEVE_MESSAGE =
         "Failed to retrieve the authorization information from the Elastic Inference Service.";
+    private static final TimeValue DEFAULT_AUTH_TIMEOUT = TimeValue.timeValueMinutes(1);
     private static final ResponseHandler AUTH_RESPONSE_HANDLER = createAuthResponseHandler();
 
     private static ResponseHandler createAuthResponseHandler() {
@@ -110,7 +110,7 @@ public void getAuthorization(ActionListener<ElasticInferenceServiceAuthorization
 
             var request = new ElasticInferenceServiceAuthorizationRequest(baseUrl, getCurrentTraceInfo());
 
-            sender.sendWithoutQueuing(logger, request, AUTH_RESPONSE_HANDLER, DEFAULT_TIMEOUT, newListener);
+            sender.sendWithoutQueuing(logger, request, AUTH_RESPONSE_HANDLER, DEFAULT_AUTH_TIMEOUT, newListener);
         } catch (Exception e) {
             logger.warn(Strings.format("Retrieving the authorization information encountered an exception: %s", e));
             requestCompleteLatch.countDown();