KYLIN-3997: Add a health check job of Kylin

core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java

@@ -787,7 +787,7 @@ public int getJobRetry() {
     }
 
     // retry interval in milliseconds
-    public int getJobRetryInterval(){
+    public int getJobRetryInterval() {
         return Integer.parseInt(getOptional("kylin.job.retry-interval", "30000"));
     }
 
@@ -873,11 +873,11 @@ public Map<Integer, String> getSourceEngines() {
         return r;
     }
 
-    public boolean enableHiveDdlQuote(){
+    public boolean enableHiveDdlQuote() {
         return Boolean.parseBoolean(getOptional("kylin.source.hive.quote-enabled", TRUE));
     }
 
-    public String getQuoteCharacter(){
+    public String getQuoteCharacter() {
         return getOptional("kylin.source.quote.character", "`");
     }
 
@@ -1386,12 +1386,11 @@ public boolean isUseLocalClasspathEnabled() {
         return Boolean.parseBoolean(getOptional("kylin.engine.mr.use-local-classpath", TRUE));
     }
 
-
     /**
      * different version hive use different UNION style
      * https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Union
      */
-    public String getHiveUnionStyle(){
+    public String getHiveUnionStyle() {
         return getOptional("kylin.hive.union.style", "UNION");
     }
 
@@ -1430,7 +1429,7 @@ public boolean isSparkSanityCheckEnabled() {
     public boolean isSparkFactDistinctEnable() {
         return Boolean.parseBoolean(getOptional("kylin.engine.spark-fact-distinct", "false"));
     }
-    
+
     // ============================================================================
     // ENGINE.LIVY
     // ============================================================================
@@ -2192,4 +2191,24 @@ public boolean isStreamingStandAloneMode() {
     public String getLocalStorageImpl() {
         return getOptional("kylin.stream.settled.storage", null);
     }
+
+    public int getWarningSegmentNum() {
+        return Integer.parseInt(getOptional("kylin.tool.health-check.warning-segment-num", "-1"));
+    }
+
+    public int getWarningCubeExpansionRate() {
+        return Integer.parseInt(getOptional("kylin.tool.health-check.warning-cube-expansion-rate", "5"));
+    }
+
+    public int getExpansionCheckMinCubeSizeInGb() {
+        return Integer.parseInt(getOptional("kylin.tool.health-check.expansion-check.min-cube-size-gb", "500"));
+    }
+
+    public int getStaleCubeThresholdInDays() {
+        return Integer.parseInt(getOptional("kylin.tool.health-check.stale-cube-threshold-days", "100"));
+    }
+
+    public int getStaleJobThresholdInDays() {
+        return Integer.parseInt(getOptional("kylin.tool.health-check.stale-job-threshold-days", "30"));
+    }
 }

core-common/src/main/java/org/apache/kylin/common/util/BufferedLogger.java

@@ -18,6 +18,9 @@
 
 package org.apache.kylin.common.util;
 
+import org.slf4j.helpers.FormattingTuple;
+import org.slf4j.helpers.MessageFormatter;
+
 /**
  * A Logger that remembers all the logged message.
  *
@@ -41,6 +44,12 @@ public void log(String message) {
         }
     }
 
+    @Override
+    public void log(String message, Object... arguments) {
+        FormattingTuple ft = MessageFormatter.arrayFormat(message, arguments);
+        log(ft.getMessage());
+    }
+
     public String getBufferedLog() {
         return buffer.toString();
     }

core-common/src/main/java/org/apache/kylin/common/util/Logger.java

@@ -24,4 +24,6 @@
  */
 public interface Logger {
     public void log(String message);
+
+    public void log(String message, Object... var2);
 }

core-common/src/main/java/org/apache/kylin/common/util/SoutLogger.java

@@ -17,6 +17,9 @@
 */
 package org.apache.kylin.common.util;
 
+import org.slf4j.helpers.FormattingTuple;
+import org.slf4j.helpers.MessageFormatter;
+
 /**
  */
 public class SoutLogger implements Logger {
@@ -25,4 +28,10 @@ public class SoutLogger implements Logger {
     public void log(String message) {
         System.out.println(message);
     }
+
+    @Override
+    public void log(String message, Object... arguments) {
+        FormattingTuple ft = MessageFormatter.arrayFormat(message, arguments);
+        log(ft.getMessage());
+    }
 }

core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java

@@ -220,6 +220,10 @@ public CubeInstance getCubeByUuid(String uuid) {
         }
     }
 
+    public List<String> getErrorCubes() {
+        return crud.getLoadFailedEntities();
+    }
+
     /**
      * Get related Cubes by cubedesc name. By default, the desc name will be
      * translated into upper case.

core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java

@@ -23,6 +23,7 @@
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.kylin.common.persistence.JsonSerializer;
@@ -48,6 +49,7 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> {
     final private SingleValueCache<String, T> cache;
 
     private boolean checkCopyOnWrite;
+    final private List<String> loadErrors = new ArrayList<>();
 
     public CachedCrudAssist(ResourceStore store, String resourceRootPath, Class<T> entityType,
             SingleValueCache<String, T> cache) {
@@ -118,14 +120,15 @@ public void reloadAll() throws IOException {
         logger.debug("Reloading " + entityType.getSimpleName() + " from " + store.getReadableResourcePath(resRootPath));
 
         cache.clear();
+        loadErrors.clear();
 
         List<String> paths = store.collectResourceRecursively(resRootPath, resPathSuffix);
         for (String path : paths) {
             reloadQuietlyAt(path);
         }
 
         logger.debug("Loaded " + cache.size() + " " + entityType.getSimpleName() + "(s) out of " + paths.size()
-                + " resource");
+                + " resource with " + loadErrors.size() + " errors");
     }
 
     public T reload(String resourceName) {
@@ -141,10 +144,15 @@ private T reloadQuietlyAt(String path) {
             return reloadAt(path);
         } catch (Exception ex) {
             logger.error("Error loading " + entityType.getSimpleName() + " at " + path, ex);
+            loadErrors.add(path);
             return null;
         }
     }
 
+    public List<String> getLoadFailedEntities() {
+        return loadErrors;
+    }
+
     public T reloadAt(String path) {
         try {
             T entity = store.getResource(path, serializer);

core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java

@@ -106,7 +106,7 @@ private class DataModelSyncListener extends Broadcaster.Listener {
         public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
             //clean up the current project's table desc
             TableMetadataManager.getInstance(config).resetProjectSpecificTableDesc(project);
-
+            logger.info("Update models in project: " + project);
             try (AutoLock lock = modelMapLock.lockForWrite()) {
                 for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) {
                     crud.reloadQuietly(model);
@@ -130,6 +130,10 @@ public void onEntityChange(Broadcaster broadcaster, String entity, Event event,
         }
     }
 
+    public List<String> getErrorModels() {
+        return crud.getLoadFailedEntities();
+    }
+
     private Class<DataModelDesc> getDataModelImplClass() {
         try {
             String cls = StringUtil.noBlank(config.getDataModelImpl(), DataModelDesc.class.getName());