SONAR-9041 add job canceling worn out tasks from CE_QUEUE regularly

server/sonar-ce-api/src/main/java/org/sonar/ce/configuration/CeConfiguration.java

@@ -31,4 +31,14 @@ public interface CeConfiguration {
    * when previous one had nothing to do.
    */
   long getQueuePollingDelay();
+
+  /**
+   * Delay before running job that cancels worn out tasks for the first time (in minutes).
+   */
+  long getCancelWornOutsInitialDelay();
+
+  /**
+   * Delay between the end of a run and the start of the next one of the job that cancels worn out CE tasks (in minutes).
+   */
+  long getCancelWornOutsDelay();
 }

server/sonar-ce-api/src/main/java/org/sonar/ce/configuration/CeConfigurationImpl.java

@@ -19,7 +19,6 @@
  */
 package org.sonar.ce.configuration;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.picocontainer.Startable;
 import org.sonar.api.config.Settings;
 import org.sonar.api.utils.MessageException;
@@ -38,11 +37,13 @@ public class CeConfigurationImpl implements CeConfiguration, Startable {
 
   private static final Logger LOG = Loggers.get(CeConfigurationImpl.class);
 
-  @VisibleForTesting
-  protected static final int DEFAULT_WORKER_COUNT = 1;
+  private static final int DEFAULT_WORKER_COUNT = 1;
   // 2 seconds
-  @VisibleForTesting
-  protected static final long DEFAULT_QUEUE_POLLING_DELAY = 2 * 1000L;
+  private static final long DEFAULT_QUEUE_POLLING_DELAY = 2 * 1000L;
+  // 1 minute
+  private static final long CANCEL_WORN_OUTS_INITIAL_DELAY = 1;
+  // 10 minutes
+  private static final long CANCEL_WORN_OUTS_DELAY = 10;
 
   private final int workerCount;
 
@@ -96,4 +97,14 @@ public int getWorkerCount() {
   public long getQueuePollingDelay() {
     return DEFAULT_QUEUE_POLLING_DELAY;
   }
+
+  @Override
+  public long getCancelWornOutsInitialDelay() {
+    return CANCEL_WORN_OUTS_INITIAL_DELAY;
+  }
+
+  @Override
+  public long getCancelWornOutsDelay() {
+    return CANCEL_WORN_OUTS_DELAY;
+  }
 }

server/sonar-ce-api/src/test/java/org/sonar/ce/configuration/CeConfigurationImplTest.java

@@ -39,33 +39,33 @@ public class CeConfigurationImplTest {
   private Settings settings = new MapSettings();
 
   @Test
-  public void getWorkCount_returns_1_when_worker_property_is_not_defined() {
+  public void getWorkerCount_returns_1_when_worker_property_is_not_defined() {
     assertThat(new CeConfigurationImpl(settings).getWorkerCount()).isEqualTo(1);
   }
 
   @Test
-  public void getWorkCount_returns_1_when_worker_property_is_empty() {
+  public void getWorkerCount_returns_1_when_worker_property_is_empty() {
     settings.setProperty(CE_WORKERS_COUNT_PROPERTY, "");
 
     assertThat(new CeConfigurationImpl(settings).getWorkerCount()).isEqualTo(1);
   }
 
   @Test
-  public void getWorkCount_returns_1_when_worker_property_is_space_chars() {
+  public void getWorkerCount_returns_1_when_worker_property_is_space_chars() {
     settings.setProperty(CE_WORKERS_COUNT_PROPERTY, "  \n  ");
 
     assertThat(new CeConfigurationImpl(settings).getWorkerCount()).isEqualTo(1);
   }
 
   @Test
-  public void getWorkCount_returns_1_when_worker_property_is_1() {
+  public void getWorkerCount_returns_1_when_worker_property_is_1() {
     settings.setProperty(CE_WORKERS_COUNT_PROPERTY, 1);
 
     assertThat(new CeConfigurationImpl(settings).getWorkerCount()).isEqualTo(1);
   }
 
   @Test
-  public void getWorkCount_returns_value_when_worker_property_is_integer_greater_than_1() {
+  public void getWorkerCount_returns_value_when_worker_property_is_integer_greater_than_1() {
     int value = abs(new Random().nextInt()) + 2;
     settings.setProperty(CE_WORKERS_COUNT_PROPERTY, value);
 
@@ -107,6 +107,18 @@ public void constructor_throws_MessageException_when_worker_property_is_not_an_d
   private void expectMessageException(int value) {
     expectedException.expect(MessageException.class);
     expectedException.expectMessage("value '" + value + "' of property " + CE_WORKERS_COUNT_PROPERTY + " is invalid. " +
-        "It must an integer strictly greater than 0");
+      "It must an integer strictly greater than 0");
+  }
+
+  @Test
+  public void getCancelWornOutsInitialDelay_returns_1() {
+    assertThat(new CeConfigurationImpl(settings).getCancelWornOutsInitialDelay())
+      .isEqualTo(1L);
+  }
+
+  @Test
+  public void getCancelWornOutsDelay_returns_10() {
+    assertThat(new CeConfigurationImpl(settings).getCancelWornOutsDelay())
+      .isEqualTo(10L);
   }
 }

server/sonar-ce/src/main/java/org/sonar/ce/cleaning/CeCleaningExecutorService.java

@@ -0,0 +1,25 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.cleaning;
+
+import java.util.concurrent.ScheduledExecutorService;
+
+public interface CeCleaningExecutorService extends ScheduledExecutorService {
+}

server/sonar-ce/src/main/java/org/sonar/ce/cleaning/CeCleaningExecutorServiceImpl.java

@@ -0,0 +1,38 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.cleaning;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import org.sonar.server.util.AbstractStoppableScheduledExecutorServiceImpl;
+
+public class CeCleaningExecutorServiceImpl
+  extends AbstractStoppableScheduledExecutorServiceImpl<ScheduledExecutorService>
+  implements CeCleaningExecutorService {
+
+  public CeCleaningExecutorServiceImpl() {
+    super(Executors.newSingleThreadScheduledExecutor(
+      new ThreadFactoryBuilder()
+        .setDaemon(false)
+        .setNameFormat("CE_cleaning-%d")
+        .build()));
+  }
+}

server/sonar-ce/src/main/java/org/sonar/ce/cleaning/CeCleaningModule.java

@@ -0,0 +1,31 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.cleaning;
+
+import org.sonar.core.platform.Module;
+
+public class CeCleaningModule extends Module {
+  @Override
+  protected void configureModule() {
+    add(
+      CeCleaningExecutorServiceImpl.class,
+      CeCleaningSchedulerImpl.class);
+  }
+}

server/sonar-ce/src/main/java/org/sonar/ce/cleaning/CeCleaningScheduler.java

@@ -0,0 +1,24 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.cleaning;
+
+public interface CeCleaningScheduler {
+  void startScheduling();
+}

server/sonar-ce/src/main/java/org/sonar/ce/cleaning/CeCleaningSchedulerImpl.java

@@ -0,0 +1,58 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.cleaning;
+
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.ce.configuration.CeConfiguration;
+import org.sonar.ce.queue.InternalCeQueue;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+
+public class CeCleaningSchedulerImpl implements CeCleaningScheduler {
+  private static final Logger LOG = Loggers.get(CeCleaningSchedulerImpl.class);
+
+  private final CeCleaningExecutorService executorService;
+  private final CeConfiguration ceConfiguration;
+  private final InternalCeQueue internalCeQueue;
+
+  public CeCleaningSchedulerImpl(CeCleaningExecutorService executorService, CeConfiguration ceConfiguration, InternalCeQueue internalCeQueue) {
+    this.executorService = executorService;
+    this.internalCeQueue = internalCeQueue;
+    this.ceConfiguration = ceConfiguration;
+  }
+
+  @Override
+  public void startScheduling() {
+    executorService.scheduleWithFixedDelay(this::cancelWornOuts,
+      ceConfiguration.getCancelWornOutsInitialDelay(),
+      ceConfiguration.getCancelWornOutsDelay(),
+      MINUTES);
+  }
+
+  private void cancelWornOuts() {
+    try {
+      LOG.info("Deleting any worn out task");
+      internalCeQueue.cancelWornOuts();
+    } catch (Exception e) {
+      LOG.warn("Failed to cancel worn out tasks", e);
+    }
+  }
+}

server/sonar-ce/src/main/java/org/sonar/ce/cleaning/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.ce.cleaning;
+
+import javax.annotation.ParametersAreNonnullByDefault;

server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java

@@ -43,6 +43,7 @@
 import org.sonar.ce.CeHttpModule;
 import org.sonar.ce.CeQueueModule;
 import org.sonar.ce.CeTaskCommonsModule;
+import org.sonar.ce.cleaning.CeCleaningModule;
 import org.sonar.ce.db.ReadOnlyPropertiesDao;
 import org.sonar.ce.log.CeProcessLogging;
 import org.sonar.ce.platform.ComputeEngineExtensionInstaller;
@@ -393,6 +394,9 @@ private static Object[] level4Components() {
 
       InternalPropertiesImpl.class,
       ProjectSettingsFactory.class,
+
+      // cleaning
+      CeCleaningModule.class
     };
   }
 

server/sonar-ce/src/main/java/org/sonar/ce/queue/CeQueueInitializer.java

@@ -22,6 +22,7 @@
 import org.sonar.api.ce.ComputeEngineSide;
 import org.sonar.api.platform.Server;
 import org.sonar.api.platform.ServerStartHandler;
+import org.sonar.ce.cleaning.CeCleaningScheduler;
 import org.sonar.ce.taskprocessor.CeProcessingScheduler;
 
 /**
@@ -32,11 +33,13 @@
 @ComputeEngineSide
 public class CeQueueInitializer implements ServerStartHandler {
 
-  private final CeProcessingScheduler scheduler;
+  private final CeProcessingScheduler processingScheduler;
+  private final CeCleaningScheduler cleaningScheduler;
   private boolean done = false;
 
-  public CeQueueInitializer(CeProcessingScheduler scheduler) {
-    this.scheduler = scheduler;
+  public CeQueueInitializer(CeProcessingScheduler processingScheduler, CeCleaningScheduler cleaningScheduler) {
+    this.processingScheduler = processingScheduler;
+    this.cleaningScheduler = cleaningScheduler;
   }
 
   @Override
@@ -48,6 +51,7 @@ public void onServerStart(Server server) {
   }
 
   private void initCe() {
-    scheduler.startScheduling();
+    processingScheduler.startScheduling();
+    cleaningScheduler.startScheduling();
   }
 }