control-service: add timeouts to shedlock's database operations (#693)

We have experienced cases when operations against the database appear to hang indefinitely (usually after a connectivity issue with the database). As a result tasks that attempt to obtain a distributed lock, fail to start. This is particularly problematic for important tasks, such as the data job watching. This commit attempts to resolve this by specifying a query timeout to the Jdbc template used by shedlock to operate with the database. Link: https://dzone.com/articles/threads-stuck-in-javanetsocketinputstreamsocketrea Testing done: started the service locally and veried that the locks are operating normally with thevtimeout. Signed-off-by: Tsvetomir Palashki <tpalashki@vmware.com>
vmware · Feb 7, 2022 · 15b6ab7 · 15b6ab7
1 parent ff6ff10
commit 15b6ab7
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 4 deletions.
diff --git a/...nes_control_service/src/main/java/com/vmware/taurus/service/locks/CustomLockProvider.java b/...nes_control_service/src/main/java/com/vmware/taurus/service/locks/CustomLockProvider.java
@@ -10,8 +10,8 @@
 import net.javacrumbs.shedlock.core.SimpleLock;
 import net.javacrumbs.shedlock.provider.jdbctemplate.JdbcTemplateLockProvider;
 import org.jetbrains.annotations.NotNull;
+import org.springframework.jdbc.core.JdbcTemplate;
 
-import javax.sql.DataSource;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
@@ -24,8 +24,8 @@
 class CustomLockProvider extends JdbcTemplateLockProvider {
     private final Map<String, SimpleLock> activeLocks = new ConcurrentHashMap<>();
 
-    public CustomLockProvider(DataSource dataSource, String tableName) {
-        super(dataSource, tableName);
+    public CustomLockProvider(JdbcTemplate jdbcTemplate, String tableName) {
+        super(jdbcTemplate, tableName);
     }
 
     @NotNull

diff --git a/...ines_control_service/src/main/java/com/vmware/taurus/service/locks/LockConfiguration.java b/...ines_control_service/src/main/java/com/vmware/taurus/service/locks/LockConfiguration.java
@@ -10,6 +10,7 @@
 import net.javacrumbs.shedlock.spring.annotation.EnableSchedulerLock;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.scheduling.annotation.EnableScheduling;
 
 import javax.annotation.PreDestroy;
@@ -21,6 +22,18 @@
 @Slf4j
 public class LockConfiguration {
 
+    /**
+     * We have experienced cases when operations against the database appear to hang
+     * indefinitely (usually after a connectivity issue with the database). As a result
+     * tasks that attempt to obtain a distributed lock, fail to start.
+     * This value is set directly on the JdbcTemplate as a query timeout in an attempt
+     * to force the database operation to time out and prevent such cases.
+     *
+     * @see   <a href="https://dzone.com/articles/threads-stuck-in-javanetsocketinputstreamsocketrea">
+     *     Threads Stuck in java.net.SocketInputStream.socketRead0 API</a>
+     */
+    private static final int LOCK_PROVIDER_QUERY_TIMEOUT_SECONDS = 60;
+
     private CustomLockProvider customLockProvider;
 
     /**
@@ -37,7 +50,9 @@ public class LockConfiguration {
      */
     @Bean
     public LockProvider lockProvider(DataSource dataSource) {
-        customLockProvider = new CustomLockProvider(dataSource, "shedlock");
+        JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
+        jdbcTemplate.setQueryTimeout(LOCK_PROVIDER_QUERY_TIMEOUT_SECONDS);
+        customLockProvider = new CustomLockProvider(jdbcTemplate, "shedlock");
         return customLockProvider;
     }