Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
[AMBARI-23685] Rename metric tables to support schema upgrade (dsen) …
…(#1091)

* [AMBARI-23685] Rename metric tables to support schema upgrade (dsen)

* [AMBARI-23685] Rename metric tables to support schema upgrade (dsen)

* [AMBARI-23685] Rename metric tables to support schema upgrade (dsen) - removed whitespaces

* [AMBARI-23685] Rename metric tables to support schema upgrade (dsen) - changed _V2 to _UUID

* [AMBARI-23685] Rename metric tables to support schema upgrade (dsen) - regex change
  • Loading branch information
d0zen1 committed May 3, 2018
1 parent 3e7eed2 commit be1571e5423a9dfcd4a4230d6996393b22006a94
Showing 7 changed files with 82 additions and 65 deletions.
@@ -32,8 +32,8 @@ HBASE_CONF_DIR=/etc/ams-hbase/conf

HBASE_CMD=${HBASE_DIR}/bin/hbase

METRIC_TABLES=(METRIC_AGGREGATE_DAILY METRIC_AGGREGATE_HOURLY METRIC_AGGREGATE_MINUTE METRIC_AGGREGATE METRIC_RECORD METRIC_RECORD_DAILY METRIC_RECORD_HOURLY METRIC_RECORD_MINUTE)
METRIC_FIFO_COMPACTION_TABLES=(METRIC_AGGREGATE METRIC_RECORD METRIC_RECORD_MINUTE)
METRIC_TABLES=(METRIC_AGGREGATE_DAILY_UUID METRIC_AGGREGATE_HOURLY_UUID METRIC_AGGREGATE_MINUTE_UUID METRIC_AGGREGATE_UUID METRIC_RECORD_UUID METRIC_RECORD_DAILY_UUID METRIC_RECORD_HOURLY_UUID METRIC_RECORD_MINUTE_UUID)
METRIC_FIFO_COMPACTION_TABLES=(METRIC_AGGREGATE_UUID METRIC_RECORD_UUID METRIC_RECORD_MINUTE_UUID)
METRIC_COLLECTOR=ambari-metrics-collector

NORMALIZER_ENABLED_STUB_FILE=/var/run/ambari-metrics-collector/normalizer_enabled
@@ -44,7 +44,7 @@ public class PhoenixTransactSQL {
* Create table to store individual metric records.
*/
public static final String CREATE_METRICS_TABLE_SQL = "CREATE TABLE IF NOT " +
"EXISTS METRIC_RECORD (UUID BINARY(32) NOT NULL, " +
"EXISTS METRIC_RECORD_UUID (UUID BINARY(32) NOT NULL, " +
"SERVER_TIME BIGINT NOT NULL, " +
"METRIC_SUM DOUBLE, " +
"METRIC_COUNT UNSIGNED_INT, " +
@@ -140,7 +140,7 @@ public class PhoenixTransactSQL {

//////// METATDATA TABLES ////////
public static final String CREATE_METRICS_METADATA_TABLE_SQL =
"CREATE TABLE IF NOT EXISTS METRICS_METADATA " +
"CREATE TABLE IF NOT EXISTS METRICS_METADATA_UUID " +
"(METRIC_NAME VARCHAR, " +
"APP_ID VARCHAR, " +
"INSTANCE_ID VARCHAR, " +
@@ -154,7 +154,7 @@ public class PhoenixTransactSQL {
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";

public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
"CREATE TABLE IF NOT EXISTS HOSTED_APPS_METADATA " +
"CREATE TABLE IF NOT EXISTS HOSTED_APPS_METADATA_UUID " +
"(HOSTNAME VARCHAR, UUID BINARY(16), APP_IDS VARCHAR, " +
"CONSTRAINT pk PRIMARY KEY (HOSTNAME))" +
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
@@ -166,7 +166,7 @@ public class PhoenixTransactSQL {
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";

public static final String ALTER_METRICS_METADATA_TABLE =
"ALTER TABLE METRICS_METADATA ADD IF NOT EXISTS IS_WHITELISTED BOOLEAN";
"ALTER TABLE METRICS_METADATA_UUID ADD IF NOT EXISTS IS_WHITELISTED BOOLEAN";

////////////////////////////////

@@ -253,12 +253,12 @@ public class PhoenixTransactSQL {
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";

public static final String UPSERT_METADATA_SQL =
"UPSERT INTO METRICS_METADATA (METRIC_NAME, APP_ID, INSTANCE_ID, UUID, UNITS, TYPE, " +
"UPSERT INTO METRICS_METADATA_UUID (METRIC_NAME, APP_ID, INSTANCE_ID, UUID, UNITS, TYPE, " +
"START_TIME, SUPPORTS_AGGREGATION, IS_WHITELISTED) " +
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";

public static final String UPSERT_HOSTED_APPS_METADATA_SQL =
"UPSERT INTO HOSTED_APPS_METADATA (HOSTNAME, UUID, APP_IDS) VALUES (?, ?, ?)";
"UPSERT INTO HOSTED_APPS_METADATA_UUID (HOSTNAME, UUID, APP_IDS) VALUES (?, ?, ?)";

public static final String UPSERT_INSTANCE_HOST_METADATA_SQL =
"UPSERT INTO INSTANCE_HOST_METADATA (INSTANCE_ID, HOSTNAME) VALUES (?, ?)";
@@ -364,10 +364,10 @@ public class PhoenixTransactSQL {

public static final String GET_METRIC_METADATA_SQL = "SELECT " +
"METRIC_NAME, APP_ID, INSTANCE_ID, UUID, UNITS, TYPE, START_TIME, " +
"SUPPORTS_AGGREGATION, IS_WHITELISTED FROM METRICS_METADATA";
"SUPPORTS_AGGREGATION, IS_WHITELISTED FROM METRICS_METADATA_UUID";

public static final String GET_HOSTED_APPS_METADATA_SQL = "SELECT " +
"HOSTNAME, UUID, APP_IDS FROM HOSTED_APPS_METADATA";
"HOSTNAME, UUID, APP_IDS FROM HOSTED_APPS_METADATA_UUID";

public static final String GET_INSTANCE_HOST_METADATA_SQL = "SELECT " +
"INSTANCE_ID, HOSTNAME FROM INSTANCE_HOST_METADATA";
@@ -423,26 +423,43 @@ public class PhoenixTransactSQL {
"MAX(METRIC_MAX), MIN(METRIC_MIN) FROM %s WHERE METRIC_NAME LIKE %s AND SERVER_TIME > %s AND " +
"SERVER_TIME <= %s GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS";

public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD";

public static final String CONTAINER_METRICS_TABLE_NAME = "CONTAINER_METRICS";

public static final String METRICS_AGGREGATE_MINUTE_TABLE_NAME =
public static final String METRICS_RECORD_V1_TABLE_NAME = "METRIC_RECORD";

public static final String METRICS_AGGREGATE_MINUTE_V1_TABLE_NAME =
"METRIC_RECORD_MINUTE";
public static final String METRICS_AGGREGATE_HOURLY_TABLE_NAME =
public static final String METRICS_AGGREGATE_HOURLY_V1_TABLE_NAME =
"METRIC_RECORD_HOURLY";
public static final String METRICS_AGGREGATE_DAILY_TABLE_NAME =
public static final String METRICS_AGGREGATE_DAILY_V1_TABLE_NAME =
"METRIC_RECORD_DAILY";
public static final String METRICS_CLUSTER_AGGREGATE_TABLE_NAME =
public static final String METRICS_CLUSTER_AGGREGATE_V1_TABLE_NAME =
"METRIC_AGGREGATE";
public static final String METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME =
public static final String METRICS_CLUSTER_AGGREGATE_MINUTE_V1_TABLE_NAME =
"METRIC_AGGREGATE_MINUTE";
public static final String METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME =
public static final String METRICS_CLUSTER_AGGREGATE_HOURLY_V1_TABLE_NAME =
"METRIC_AGGREGATE_HOURLY";
public static final String METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME =
public static final String METRICS_CLUSTER_AGGREGATE_DAILY_V1_TABLE_NAME =
"METRIC_AGGREGATE_DAILY";

public static final Pattern PHOENIX_TABLES_REGEX_PATTERN = Pattern.compile("METRIC_");
public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD_UUID";

public static final String METRICS_AGGREGATE_MINUTE_TABLE_NAME =
"METRIC_RECORD_MINUTE_UUID";
public static final String METRICS_AGGREGATE_HOURLY_TABLE_NAME =
"METRIC_RECORD_HOURLY_UUID";
public static final String METRICS_AGGREGATE_DAILY_TABLE_NAME =
"METRIC_RECORD_DAILY_UUID";
public static final String METRICS_CLUSTER_AGGREGATE_TABLE_NAME =
"METRIC_AGGREGATE_UUID";
public static final String METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME =
"METRIC_AGGREGATE_MINUTE_UUID";
public static final String METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME =
"METRIC_AGGREGATE_HOURLY_UUID";
public static final String METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME =
"METRIC_AGGREGATE_DAILY_UUID";

public static final Pattern PHOENIX_TABLES_REGEX_PATTERN = Pattern.compile("METRIC_.*_UUID");

public static final String[] PHOENIX_TABLES = {
METRICS_RECORD_TABLE_NAME,
@@ -121,16 +121,16 @@ public void tearDown() {
conn = getConnection(getUrl());
stmt = conn.createStatement();

deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE");
deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_MINUTE");
deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_HOURLY");
deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_DAILY");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_MINUTE");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_HOURLY");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_DAILY");
deleteTableIgnoringExceptions(stmt, "METRICS_METADATA");
deleteTableIgnoringExceptions(stmt, "HOSTED_APPS_METADATA");
deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_UUID");
deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_MINUTE_UUID");
deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_HOURLY_UUID");
deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_DAILY_UUID");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_UUID");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_MINUTE_UUID");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_HOURLY_UUID");
deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_DAILY_UUID");
deleteTableIgnoringExceptions(stmt, "METRICS_METADATA_UUID");
deleteTableIgnoringExceptions(stmt, "HOSTED_APPS_METADATA_UUID");

conn.commit();
} catch (Exception e) {
@@ -409,7 +409,7 @@ public void testInitPoliciesAndTTL() throws Exception {

Assert.assertFalse("Normalizer disabled.", normalizerEnabled);
Assert.assertTrue("Durability Set.", tableDurabilitySet);
Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD.", FIFO_COMPACTION_POLICY_CLASS, precisionTableCompactionPolicy);
Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD_UUID.", FIFO_COMPACTION_POLICY_CLASS, precisionTableCompactionPolicy);
Assert.assertEquals("FIFO compaction policy is set for aggregate tables", DATE_TIERED_COMPACTION_POLICY, aggregateTableCompactionPolicy);
Assert.assertEquals("Precision TTL value as expected.", 86400, precisionTtl);

@@ -441,7 +441,7 @@ public void testInsertContainerMetrics() throws Exception {
metric.setExitCode(0);
List<ContainerMetric> list = Arrays.asList(metric);
hdb.insertContainerMetrics(list);
PreparedStatement stmt = conn.prepareStatement("SELECT * FROM CONTAINER_METRICS");
PreparedStatement stmt = conn.prepareStatement("SELECT * FROM CONTAINER_METRICS_UUID");
ResultSet set = stmt.executeQuery();
// check each filed is set properly when read back.
boolean foundRecord = false;
@@ -154,7 +154,7 @@ public void testPrepareGetAggregatePrecisionMINUTES() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_MINUTE_UUID"));
verify(connection, preparedStatement);
}

@@ -183,7 +183,7 @@ public void testPrepareGetAggregateNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_UUID"));
Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
verify(connection, preparedStatement);

@@ -201,7 +201,7 @@ public void testPrepareGetAggregateNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_MINUTE_UUID"));
Assert.assertEquals(Precision.MINUTES, condition.getPrecision());
verify(connection, preparedStatement);

@@ -219,7 +219,7 @@ public void testPrepareGetAggregateNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY"));
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY_UUID"));
Assert.assertEquals(Precision.HOURS, condition.getPrecision());
verify(connection, preparedStatement);

@@ -237,7 +237,7 @@ public void testPrepareGetAggregateNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_DAILY"));
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_DAILY_UUID"));
Assert.assertEquals(Precision.DAYS, condition.getPrecision());
verify(connection, preparedStatement);
}
@@ -256,7 +256,7 @@ public void testPrepareGetAggregatePrecisionHours() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY"));
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY_UUID"));
verify(connection, preparedStatement);
}

@@ -274,7 +274,7 @@ public void testPrepareGetMetricsPrecisionMinutes() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE_UUID"));
verify(connection, preparedStatement);
}

@@ -301,7 +301,7 @@ public void testPrepareGetMetricsNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_UUID"));
Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
verify(connection, preparedStatement);
reset(connection, preparedStatement);
@@ -320,7 +320,7 @@ public void testPrepareGetMetricsNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_UUID"));
Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
verify(connection, preparedStatement);

@@ -338,7 +338,7 @@ public void testPrepareGetMetricsNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE_UUID"));
Assert.assertEquals(Precision.MINUTES, condition.getPrecision());
verify(connection, preparedStatement);

@@ -355,7 +355,7 @@ public void testPrepareGetMetricsNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY_UUID"));
Assert.assertEquals(Precision.HOURS, condition.getPrecision());
verify(connection, preparedStatement);

@@ -372,7 +372,7 @@ public void testPrepareGetMetricsNoPrecision() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_DAILY"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_DAILY_UUID"));
Assert.assertEquals(Precision.DAYS, condition.getPrecision());
verify(connection, preparedStatement);

@@ -398,7 +398,7 @@ public void testPrepareGetLatestMetricSqlStmtMultipleHostNames() throws SQLExcep
replay(connection, preparedStatement, parameterMetaData);
PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_UUID"));
Assert.assertTrue(stmt.contains("JOIN"));
verify(connection, preparedStatement, parameterMetaData);
}
@@ -441,7 +441,7 @@ public void testPrepareGetMetricsPrecisionHours() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY_UUID"));
verify(connection, preparedStatement);
}

@@ -476,7 +476,7 @@ public void testResultSetLimitCheck() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
String stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_UUID"));
verify(connection, preparedStatement);

//Check without passing precision. Should be OK!
@@ -492,7 +492,7 @@ public void testResultSetLimitCheck() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_UUID"));
verify(connection, preparedStatement);

//Check with more hosts and lesser metrics for 1 day data = 11520 points Should be OK!
@@ -518,7 +518,7 @@ public void testResultSetLimitCheck() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE_UUID"));
verify(connection, preparedStatement);

//Check with 20 metrics, NO hosts and 1 day data = 5760 points. Should be OK!
@@ -540,7 +540,7 @@ public void testResultSetLimitCheck() throws SQLException {
replay(connection, preparedStatement);
PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
stmt = stmtCapture.getValue();
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_MINUTE"));
Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_MINUTE_UUID"));
verify(connection, preparedStatement);

//Check with 5 hosts and 10 metrics for 1 hour data = 18000 points. Should throw out Exception!
@@ -584,7 +584,7 @@ public void testTopNHostsConditionClause() throws Exception {

String conditionClause = condition.getConditionClause().toString();
String expectedClause = " UUID IN (" +
"SELECT UUID FROM METRIC_RECORD WHERE " +
"SELECT UUID FROM METRIC_RECORD_UUID WHERE " +
"(UUID IN (?, ?)) AND " +
"SERVER_TIME >= ? AND SERVER_TIME < ? " +
"GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?";
@@ -602,7 +602,7 @@ public void testTopNMetricsConditionClause() throws Exception {

String conditionClause = condition.getConditionClause().toString();
String expectedClause = " UUID IN (" +
"SELECT UUID FROM METRIC_RECORD WHERE " +
"SELECT UUID FROM METRIC_RECORD_UUID WHERE " +
"(UUID IN (?, ?, ?)) AND " +
"SERVER_TIME >= ? AND SERVER_TIME < ? " +
"GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?";

0 comments on commit be1571e

Please sign in to comment.