[TE] Added support for Vertica as a data source (#4404)

Added support for Vertica as a data source.
apache · Jul 10, 2019 · 52f69d1 · 52f69d1
1 parent 07e2c72
commit 52f69d1
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 0 deletions.
diff --git a/...-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java b/...-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
@@ -60,19 +60,23 @@ public class SqlResponseCacheLoader extends CacheLoader<SqlQuery, ThirdEyeResult
   private static int MAX_CONNECTIONS = 50;
   private static final String PRESTO = "Presto";
   private static final String MYSQL = "MySQL";
+  private static final String VERTICA = "Vertica";
   private static final String DATASETS = "datasets";
   private static final String H2 = "H2";
   private static final String USER = "user";
   private static final String DB = "db";
   private static final String PASSWORD = "password";
+  private static final String DRIVER = "driver";
   private static final DateTime MIN_DATETIME = DateTime.parse("1970-01-01");
   private static final int ABANDONED_TIMEOUT = 60000;
 
   private Map<String, DataSource> prestoDBNameToDataSourceMap = new HashMap<>();
   private Map<String, DataSource> mysqlDBNameToDataSourceMap = new HashMap<>();
+  private Map<String, DataSource> verticaDBNameToDataSourceMap = new HashMap<>();
 
   private static Map<String, String> prestoDBNameToURLMap = new HashMap<>();
   private static Map<String, String> mysqlDBNameToURLMap = new HashMap<>();
+  private static Map<String, String> verticaDBNameToURLMap = new HashMap<>();
 
   private static String h2Url;
   DataSource h2DataSource;
@@ -133,6 +137,34 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
       }
     }
 
+    // Init Vertica datasources
+    if (properties.containsKey(VERTICA)) {
+      List<Map<String, Object>> verticaMapList = ConfigUtils.getList(properties.get(VERTICA));
+      for (Map<String, Object> objMap: verticaMapList) {
+        Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
+        String verticaUser = (String)objMap.get(USER);
+        String verticaPassword = getPassword(objMap);
+        String verticaDriver = (String)objMap.get(DRIVER);
+
+        for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
+          DataSource dataSource = new DataSource();
+          dataSource.setInitialSize(INIT_CONNECTIONS);
+          dataSource.setMaxActive(MAX_CONNECTIONS);
+          dataSource.setUsername(verticaUser);
+          dataSource.setPassword(verticaPassword);
+          dataSource.setDriverClassName(verticaDriver);
+          dataSource.setUrl(entry.getValue());
+
+          // Timeout before an abandoned(in use) connection can be removed.
+          dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
+          dataSource.setRemoveAbandoned(true);
+
+          verticaDBNameToDataSourceMap.put(entry.getKey(), dataSource);
+          verticaDBNameToURLMap.putAll(dbNameToURLMap);
+        }
+      }
+    }
+
     // Init H2 datasource
     if (properties.containsKey(H2)) {
       h2DataSource = new DataSource();
@@ -293,6 +325,8 @@ public ThirdEyeResultSetGroup load(SqlQuery SQLQuery) throws Exception {
       dataSource = prestoDBNameToDataSourceMap.get(SQLQuery.getDbName());
     } else if (sourceName.equals(MYSQL)) {
       dataSource = mysqlDBNameToDataSourceMap.get(SQLQuery.getDbName());
+    } else if (sourceName.equals(VERTICA)) {
+      dataSource = verticaDBNameToDataSourceMap.get(SQLQuery.getDbName());
     } else {
       dataSource = h2DataSource;
     }
@@ -323,6 +357,7 @@ public static Map<String, Map<String,String>> getDBNameToURLMap() {
     Map<String, Map<String,String>> dbNameToURLMap = new LinkedHashMap<>();
     dbNameToURLMap.put(PRESTO, prestoDBNameToURLMap);
     dbNameToURLMap.put(MYSQL, mysqlDBNameToURLMap);
+    dbNameToURLMap.put(VERTICA, verticaDBNameToURLMap);
 
     Map<String, String> h2ToURLMap = new HashMap<>();
     h2ToURLMap.put(H2, h2Url);
@@ -348,6 +383,8 @@ private DataSource getDataSourceFromDataset(String dataset) {
       return prestoDBNameToDataSourceMap.get(dbName);
     } else if (sourceName.equals(MYSQL)) {
       return mysqlDBNameToDataSourceMap.get(dbName);
+    } else if (sourceName.equals(VERTICA)) {
+      return verticaDBNameToDataSourceMap.get(dbName);
     } else {
       return h2DataSource;
     }

diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
@@ -87,6 +87,7 @@ public class SqlUtils {
   private static final String PRESTO = "Presto";
   private static final String MYSQL = "MySQL";
   private static final String H2 = "H2";
+  private static final String VERTICA = "Vertica";
 
   /**
    * Insert a table to SQL database, currently only used by H2, that can be read by ThirdEye
@@ -595,6 +596,8 @@ private static String getToUnixTimeClause(String timeFormat, String timeColumn,
       return "UNIX_TIMESTAMP(STR_TO_DATE(CAST(" + timeColumn + " AS CHAR), '" + timeFormatToMySQLFormat(timeFormat) + "'))";
     } else if (sourceName.equals(H2)){
       return "TO_UNIXTIME(PARSEDATETIME(CAST(" + timeColumn + " AS VARCHAR), '" + timeFormat + "'))";
+    } else if (sourceName.equals(VERTICA)) {
+      return "EXTRACT(EPOCH FROM to_timestamp(to_char(" + timeColumn + "), '" + timeFormat + "'))";
     }
     return "";
   }