[DEVOPS-2664] Fix cannot sync resources when there are not resources …

…exists
jenkinsci · Oct 24, 2019 · 0a791e8 · 0a791e8
1 parent 469bb80
commit 0a791e8
Show file tree

Hide file tree

Showing 7 changed files with 143 additions and 5 deletions.
diff --git a/src/main/java/io/alauda/jenkins/devops/sync/ConnectionAliveDetectTask.java b/src/main/java/io/alauda/jenkins/devops/sync/ConnectionAliveDetectTask.java
@@ -6,6 +6,7 @@
 import hudson.model.AsyncPeriodicWork;
 import hudson.model.TaskListener;
 import io.alauda.jenkins.devops.sync.controller.ResourceSyncManager;
+import io.kubernetes.client.ApiException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,11 +40,22 @@ protected void execute(TaskListener listener) throws IOException, InterruptedExc
         HeartbeatResourceDetector.all()
                 .forEach(detector -> {
                     logger.debug("Starting to check if the watch connection of resource {} is alive", detector.resourceName());
+                    heartbeatLostCount.putIfAbsent(detector, new AtomicInteger(0));
 
                     LocalDateTime lastEventComingTime = detector.lastEventComingTime();
                     // controller might not be initialized, or no resource exist in k8s so that we cannot receive event
                     if (lastEventComingTime == null) {
-                        logger.debug("The controller of resource {} seems not start or no resource exists in k8s, will skip check for it", detector.resourceName());
+                        try {
+                            // if there has resource exists but we didn't receive any event, the watch connection might be broken
+                            if (detector.hasResourceExists()) {
+                                int count = heartbeatLostCount.get(detector).incrementAndGet();
+                                logger.warn("The watch connection of resource {} seems broken, retry count {}", detector.resourceName(), count);
+                            } else {
+                                logger.debug("There are no resource {} exists in k8s, will skip this check for it", detector.resourceName());
+                            }
+                        } catch (ApiException e) {
+                            logger.warn("Unable to check if resource {} exists in k8s, will skip this check for it, reason: {}", detector.resourceName(), e);
+                        }
                         return;
                     }
 
@@ -91,6 +103,8 @@ public interface HeartbeatResourceDetector extends ExtensionPoint {
 
         String resourceName();
 
+        boolean hasResourceExists() throws ApiException;
+
         static ExtensionList<HeartbeatResourceDetector> all() {
             return ExtensionList.lookup(HeartbeatResourceDetector.class);
         }

diff --git a/src/main/java/io/alauda/jenkins/devops/sync/controller/CodeRepositoryController.java b/src/main/java/io/alauda/jenkins/devops/sync/controller/CodeRepositoryController.java
@@ -9,20 +9,24 @@
 import io.alauda.jenkins.devops.sync.client.Clients;
 import io.alauda.jenkins.devops.sync.client.CodeRepositoryClient;
 import io.alauda.jenkins.devops.sync.controller.util.InformerUtils;
+import io.kubernetes.client.ApiException;
 import io.kubernetes.client.extended.controller.Controller;
 import io.kubernetes.client.extended.controller.builder.ControllerBuilder;
 import io.kubernetes.client.extended.controller.builder.ControllerManagerBuilder;
 import io.kubernetes.client.extended.controller.reconciler.Request;
 import io.kubernetes.client.extended.controller.reconciler.Result;
 import io.kubernetes.client.informer.SharedIndexInformer;
 import io.kubernetes.client.informer.SharedInformerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.time.LocalDateTime;
 import java.util.concurrent.TimeUnit;
 
 @Extension
 public class CodeRepositoryController implements ResourceSyncController, ConnectionAliveDetectTask.HeartbeatResourceDetector {
 
+    private static final Logger logger = LoggerFactory.getLogger(NamespaceController.class);
     private LocalDateTime lastEventComingTime;
 
     @Override
@@ -82,4 +86,24 @@ public LocalDateTime lastEventComingTime() {
     public String resourceName() {
         return "CodeRepository";
     }
+
+    @Override
+    public boolean hasResourceExists() throws ApiException {
+        DevopsAlaudaIoV1alpha1Api api = new DevopsAlaudaIoV1alpha1Api();
+        V1alpha1CodeRepositoryList repositoryList = api.listCodeRepositoryForAllNamespaces(null,
+                null,
+                null,
+                null,
+                1,
+                null,
+                "0",
+                null,
+                null);
+
+        if (repositoryList == null || repositoryList.getItems() == null || repositoryList.getItems().size() == 0) {
+            return false;
+        }
+
+        return true;
+    }
 }
diff --git a/src/main/java/io/alauda/jenkins/devops/sync/controller/JenkinsBindingController.java b/src/main/java/io/alauda/jenkins/devops/sync/controller/JenkinsBindingController.java
@@ -88,4 +88,24 @@ public LocalDateTime lastEventComingTime() {
     public String resourceName() {
         return "JenkinsBinding";
     }
+
+    @Override
+    public boolean hasResourceExists() throws ApiException {
+        DevopsAlaudaIoV1alpha1Api api = new DevopsAlaudaIoV1alpha1Api();
+        V1alpha1JenkinsBindingList bindingList = api.listJenkinsBindingForAllNamespaces(null,
+                null,
+                null,
+                null,
+                1,
+                null,
+                "0",
+                null,
+                null);
+
+        if (bindingList == null || bindingList.getItems() == null || bindingList.getItems().size() == 0) {
+            return false;
+        }
+
+        return true;
+    }
 }
diff --git a/src/main/java/io/alauda/jenkins/devops/sync/controller/JenkinsController.java b/src/main/java/io/alauda/jenkins/devops/sync/controller/JenkinsController.java
@@ -7,15 +7,13 @@
 import hudson.model.Node;
 import hudson.model.UpdateSite;
 import io.alauda.devops.java.client.apis.DevopsAlaudaIoV1alpha1Api;
-import io.alauda.devops.java.client.models.V1alpha1BindingCondition;
-import io.alauda.devops.java.client.models.V1alpha1Jenkins;
-import io.alauda.devops.java.client.models.V1alpha1JenkinsList;
-import io.alauda.devops.java.client.models.V1alpha1JenkinsStatus;
+import io.alauda.devops.java.client.models.*;
 import io.alauda.devops.java.client.utils.DeepCopyUtils;
 import io.alauda.jenkins.devops.sync.AlaudaSyncGlobalConfiguration;
 import io.alauda.jenkins.devops.sync.ConnectionAliveDetectTask;
 import io.alauda.jenkins.devops.sync.client.JenkinsClient;
 import io.alauda.jenkins.devops.sync.constants.Constants;
+import io.kubernetes.client.ApiException;
 import io.kubernetes.client.JSON;
 import io.kubernetes.client.extended.controller.Controller;
 import io.kubernetes.client.extended.controller.builder.ControllerBuilder;
@@ -95,6 +93,26 @@ public String resourceName() {
         return "Jenkins";
     }
 
+    @Override
+    public boolean hasResourceExists() throws ApiException {
+        DevopsAlaudaIoV1alpha1Api api = new DevopsAlaudaIoV1alpha1Api();
+        V1alpha1JenkinsList jenkinsList = api.listJenkins(null,
+                null,
+                null,
+                null,
+                null,
+                1,
+                "0",
+                null,
+                null);
+
+        if (jenkinsList == null || jenkinsList.getItems() == null || jenkinsList.getItems().size() == 0) {
+            return false;
+        }
+
+        return true;
+    }
+
     private class JenkinsReconciler implements Reconciler {
 
         private Lister<V1alpha1Jenkins> lister;

diff --git a/src/main/java/io/alauda/jenkins/devops/sync/controller/NamespaceController.java b/src/main/java/io/alauda/jenkins/devops/sync/controller/NamespaceController.java
@@ -10,6 +10,7 @@
 import io.alauda.jenkins.devops.sync.client.Clients;
 import io.alauda.jenkins.devops.sync.client.NamespaceClient;
 import io.alauda.jenkins.devops.sync.controller.util.InformerUtils;
+import io.kubernetes.client.ApiException;
 import io.kubernetes.client.apis.CoreV1Api;
 import io.kubernetes.client.extended.controller.Controller;
 import io.kubernetes.client.extended.controller.builder.ControllerBuilder;
@@ -94,6 +95,25 @@ public String resourceName() {
         return "Namespace";
     }
 
+    @Override
+    public boolean hasResourceExists() throws ApiException {
+        CoreV1Api api = new CoreV1Api();
+        V1NamespaceList namespaceList = api.listNamespace(null,
+                null,
+                null,
+                null,
+                1,
+                "0",
+                null,
+                null);
+
+        if (namespaceList == null || namespaceList.getItems() == null || namespaceList.getItems().size() == 0) {
+            return false;
+        }
+
+        return true;
+    }
+
 
     static class NamespaceReconciler implements Reconciler {
         private Lister<V1Namespace> namespaceLister;

diff --git a/src/main/java/io/alauda/jenkins/devops/sync/controller/PipelineConfigController.java b/src/main/java/io/alauda/jenkins/devops/sync/controller/PipelineConfigController.java
@@ -18,6 +18,7 @@
 import io.alauda.jenkins.devops.sync.exception.PipelineConfigConvertException;
 import io.alauda.jenkins.devops.sync.util.NamespaceName;
 import io.alauda.jenkins.devops.sync.util.PipelineConfigUtils;
+import io.kubernetes.client.ApiException;
 import io.kubernetes.client.extended.controller.Controller;
 import io.kubernetes.client.extended.controller.builder.ControllerBuilder;
 import io.kubernetes.client.extended.controller.builder.ControllerManagerBuilder;
@@ -136,6 +137,26 @@ public String resourceName() {
         return "PipelineConfig";
     }
 
+    @Override
+    public boolean hasResourceExists() throws ApiException {
+        DevopsAlaudaIoV1alpha1Api api = new DevopsAlaudaIoV1alpha1Api();
+        V1alpha1PipelineConfigList pipelineConfigList = api.listPipelineConfigForAllNamespaces(null,
+                null,
+                null,
+                null,
+                1,
+                null,
+                "0",
+                null,
+                null);
+
+        if (pipelineConfigList == null || pipelineConfigList.getItems() == null || pipelineConfigList.getItems().size() == 0) {
+            return false;
+        }
+
+        return true;
+    }
+
     static class PipelineConfigReconciler implements Reconciler {
         private Lister<V1alpha1PipelineConfig> lister;
         private JenkinsClient jenkinsClient;

diff --git a/src/main/java/io/alauda/jenkins/devops/sync/controller/PipelineController.java b/src/main/java/io/alauda/jenkins/devops/sync/controller/PipelineController.java
@@ -18,6 +18,7 @@
 import io.alauda.jenkins.devops.sync.util.AlaudaUtils;
 import io.alauda.jenkins.devops.sync.util.JenkinsUtils;
 import io.alauda.jenkins.devops.sync.util.NamespaceName;
+import io.kubernetes.client.ApiException;
 import io.kubernetes.client.extended.controller.Controller;
 import io.kubernetes.client.extended.controller.builder.ControllerBuilder;
 import io.kubernetes.client.extended.controller.builder.ControllerManagerBuilder;
@@ -120,6 +121,26 @@ public String resourceName() {
         return "Pipeline";
     }
 
+    @Override
+    public boolean hasResourceExists() throws ApiException {
+        DevopsAlaudaIoV1alpha1Api api = new DevopsAlaudaIoV1alpha1Api();
+        V1alpha1PipelineList pipelineList = api.listPipelineForAllNamespaces(null,
+                null,
+                null,
+                null,
+                1,
+                null,
+                "0",
+                null,
+                null);
+
+        if (pipelineList == null || pipelineList.getItems() == null || pipelineList.getItems().size() == 0) {
+            return false;
+        }
+
+        return true;
+    }
+
 
     static class PipelineReconciler implements Reconciler {