Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.ConnectionQueryServices;
Expand All @@ -49,10 +50,12 @@ public class IndexToolTableUtil extends Configured {
public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL";
public static String OUTPUT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME);
public static byte[] OUTPUT_TABLE_FULL_NAME_BYTES = Bytes.toBytes(OUTPUT_TABLE_FULL_NAME);

public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT";
public static final String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT";
public static String RESULT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME);
public static byte[] RESULT_TABLE_FULL_NAME_BYTES = Bytes.toBytes(RESULT_TABLE_FULL_NAME);

public static void setIndexToolTableName(Connection connection) throws Exception {
ConnectionQueryServices queryServices =
Expand All @@ -66,6 +69,8 @@ public static void setIndexToolTableName(Connection connection) throws Exception
OUTPUT_TABLE_FULL_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME);
RESULT_TABLE_FULL_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME);
}
OUTPUT_TABLE_FULL_NAME_BYTES = Bytes.toBytes(OUTPUT_TABLE_FULL_NAME);
RESULT_TABLE_FULL_NAME_BYTES = Bytes.toBytes(RESULT_TABLE_FULL_NAME);
}

public static Table createResultTable(Connection connection) throws IOException, SQLException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,7 @@ public int validateLastVerifyTime() throws Exception {
public boolean isValidLastVerifyTime(Long lastVerifyTime) throws Exception {
try (Connection conn = getConnection(configuration);
Table hIndexToolTable = conn.unwrap(PhoenixConnection.class).getQueryServices()
.getTable(IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES)) {
.getTable(IndexVerificationResultRepository.getResultTableNameBytes())) {
Scan s = new Scan();
ConnectionQueryServices cqs = conn.unwrap(PhoenixConnection.class).getQueryServices();
boolean isNamespaceMapped = SchemaUtil.isNamespaceMappingEnabled(null, cqs.getProps());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,17 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
IndexTool.IndexDisableLoggingType.NONE;
private boolean shouldLogBeyondMaxLookback = true;

public final static String OUTPUT_TABLE_NAME = IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME;
public final static byte[] OUTPUT_TABLE_NAME_BYTES =
Bytes.toBytes(IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME);
public final static byte[] OUTPUT_TABLE_COLUMN_FAMILY =
QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;

public static String getOutputTableName() {
return IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME;
}

public static byte[] getOutputTableNameBytes() {
return IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME_BYTES;
}

public final static String DATA_TABLE_NAME = "DTName";
public final static byte[] DATA_TABLE_NAME_BYTES = Bytes.toBytes(DATA_TABLE_NAME);
public final static String INDEX_TABLE_NAME = "ITName";
Expand Down Expand Up @@ -109,7 +114,7 @@ public IndexVerificationOutputRepository() {
@VisibleForTesting
public IndexVerificationOutputRepository(byte[] indexName, Connection conn) throws SQLException {
ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices();
outputTable = queryServices.getTable(OUTPUT_TABLE_NAME_BYTES);
outputTable = queryServices.getTable(getOutputTableNameBytes());
indexTable = queryServices.getTable(indexName);
}

Expand All @@ -124,7 +129,7 @@ public IndexVerificationOutputRepository(Table outputTable, Table indexTable,
public IndexVerificationOutputRepository(byte[] indexName, HTableFactory hTableFactory,
IndexTool.IndexDisableLoggingType disableLoggingVerifyType) throws IOException {
this.indexName = indexName;
outputTable = hTableFactory.getTable(new ImmutableBytesPtr(OUTPUT_TABLE_NAME_BYTES));
outputTable = hTableFactory.getTable(new ImmutableBytesPtr(getOutputTableNameBytes()));
indexTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName));
this.disableLoggingVerifyType = disableLoggingVerifyType;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,17 @@ public class IndexVerificationResultRepository implements AutoCloseable {
private Table indexTable;
public static final String ROW_KEY_SEPARATOR = "|";
public static final byte[] ROW_KEY_SEPARATOR_BYTE = Bytes.toBytes(ROW_KEY_SEPARATOR);
public static String RESULT_TABLE_NAME = IndexToolTableUtil.RESULT_TABLE_FULL_NAME;
public static byte[] RESULT_TABLE_NAME_BYTES =
Bytes.toBytes(IndexToolTableUtil.RESULT_TABLE_FULL_NAME);
public final static byte[] RESULT_TABLE_COLUMN_FAMILY =
QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;

public static String getResultTableName() {
return IndexToolTableUtil.RESULT_TABLE_FULL_NAME;
}

public static byte[] getResultTableNameBytes() {
return IndexToolTableUtil.RESULT_TABLE_FULL_NAME_BYTES;
}

public final static String SCANNED_DATA_ROW_COUNT = "ScannedDataRowCount";
public final static byte[] SCANNED_DATA_ROW_COUNT_BYTES = Bytes.toBytes(SCANNED_DATA_ROW_COUNT);
public final static String REBUILT_INDEX_ROW_COUNT = "RebuiltIndexRowCount";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package org.apache.phoenix.end2end;

import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_NAME;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REPAIR_EXTRA_UNVERIFIED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REPAIR_EXTRA_VERIFIED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
Expand Down Expand Up @@ -159,8 +158,9 @@ public void cleanup() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
deleteAllRows(conn,
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES));
deleteAllRows(conn, TableName.valueOf(IndexVerificationResultRepository.RESULT_TABLE_NAME));
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableNameBytes()));
deleteAllRows(conn,
TableName.valueOf(IndexVerificationResultRepository.getResultTableNameBytes()));
}
EnvironmentEdgeManager.reset();
resetIndexRegionObserverFailPoints();
Expand Down Expand Up @@ -227,11 +227,13 @@ private void initTablesAndAddExtraRowsToIndex(Connection conn, String schemaName

private void truncateIndexToolTables() throws IOException {
getUtility().getAdmin()
.disableTable(TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
.disableTable(TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
getUtility().getAdmin().truncateTable(
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()), true);
getUtility().getAdmin()
.truncateTable(TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME), true);
getUtility().getAdmin().disableTable(TableName.valueOf(RESULT_TABLE_NAME));
getUtility().getAdmin().truncateTable(TableName.valueOf(RESULT_TABLE_NAME), true);
.disableTable(TableName.valueOf(IndexVerificationResultRepository.getResultTableName()));
getUtility().getAdmin().truncateTable(
TableName.valueOf(IndexVerificationResultRepository.getResultTableName()), true);
}

private void assertExtraCounters(IndexTool indexTool, long extraVerified, long extraUnverified,
Expand Down Expand Up @@ -281,7 +283,7 @@ private void assertDisableLogging(Connection conn, int expectedExtraRows, int ex
}
} catch (AssertionError e) {
TestUtil.dumpTable(conn,
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
throw e;
}
if (expectedPITRows > 0) {
Expand Down Expand Up @@ -678,7 +680,7 @@ public void testViewIndexExtraRows() throws Exception {
assertEquals(2, rows.size());
} catch (AssertionError e) {
TestUtil.dumpTable(conn,
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
throw e;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
import static org.apache.phoenix.mapreduce.PhoenixJobCounters.INPUT_RECORDS;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.INDEX_TOOL_RUN_STATUS_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_COLUMN_FAMILY;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_NAME;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.ROW_KEY_SEPARATOR;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RUN_STATUS_EXECUTED;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RUN_STATUS_SKIPPED;
Expand Down Expand Up @@ -201,8 +199,9 @@ public void cleanup() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
deleteAllRows(conn,
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES));
deleteAllRows(conn, TableName.valueOf(IndexVerificationResultRepository.RESULT_TABLE_NAME));
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableNameBytes()));
deleteAllRows(conn,
TableName.valueOf(IndexVerificationResultRepository.getResultTableName()));
}
EnvironmentEdgeManager.reset();
}
Expand Down Expand Up @@ -325,7 +324,7 @@ public void testIndexToolVerifyWithExpiredIndexRows() throws Exception {
modifyColumnFams.get(41000, TimeUnit.MILLISECONDS);

TableName indexToolOutputTable =
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES);
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableNameBytes());
admin.disableTable(indexToolOutputTable);
admin.deleteTable(indexToolOutputTable);
// Run the index tool using the only-verify option, verify it gives no mismatch
Expand Down Expand Up @@ -760,7 +759,8 @@ public void testIndexToolOnlyVerifyOption() throws Exception {
Assert.assertEquals(PIndexState.BUILDING, TestUtil.getIndexState(conn, indexTableFullName));

// Delete the output table for the next test
deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
deleteAllRows(conn,
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
// Run the index tool to populate the index while verifying rows
IndexToolIT.runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
IndexTool.IndexVerifyType.AFTER);
Expand Down Expand Up @@ -801,7 +801,8 @@ public void testIndexToolForIncrementalRebuild() throws Exception {
try {
verifyRunStatusFromResultTable(conn, scn, indexTableFullName, 3, expectedStatus);
} catch (AssertionError ae) {
TestUtil.dumpTable(conn, TableName.valueOf(RESULT_TABLE_NAME));
TestUtil.dumpTable(conn,
TableName.valueOf(IndexVerificationResultRepository.getResultTableName()));
throw ae;
}

Expand Down Expand Up @@ -1093,7 +1094,8 @@ public void testDisableOutputLogging() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);

try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
deleteAllRows(conn,
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
String stmString1 = "CREATE TABLE " + dataTableFullName
+ " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) " + tableDDLOptions;
conn.createStatement().execute(stmString1);
Expand Down Expand Up @@ -1181,7 +1183,8 @@ public void testEnableOutputLoggingForMaxLookback() throws Exception {

try (Connection conn = DriverManager.getConnection(getUrl())) {

deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
deleteAllRows(conn,
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
String stmString1 = "CREATE TABLE " + dataTableFullName
+ " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) ";
conn.createStatement().execute(stmString1);
Expand All @@ -1206,13 +1209,14 @@ public void testEnableOutputLoggingForMaxLookback() throws Exception {
EnvironmentEdgeManager.injectEdge(injectEdge);
injectEdge.incrementValue(1L);
injectEdge.incrementValue(MAX_LOOKBACK_AGE * 1000);
deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
deleteAllRows(conn,
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
getUtility().getConfiguration()
.set(IndexRebuildRegionScanner.PHOENIX_INDEX_MR_LOG_BEYOND_MAX_LOOKBACK_ERRORS, "true");
IndexTool it = IndexToolIT.runIndexTool(useSnapshot, schemaName, dataTableName,
indexTableName, null, 0, IndexTool.IndexVerifyType.ONLY);
TestUtil.dumpTable(conn,
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
Counters counters = it.getJob().getCounters();
System.out.println(counters.toString());
assertEquals(2L,
Expand Down Expand Up @@ -1574,11 +1578,13 @@ private void truncateIndexAndIndexToolTables(String indexTableFullName) throws I

private void truncateIndexToolTables() throws IOException {
getUtility().getAdmin()
.disableTable(TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
.disableTable(TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
getUtility().getAdmin().truncateTable(
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()), true);
getUtility().getAdmin()
.truncateTable(TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME), true);
getUtility().getAdmin().disableTable(TableName.valueOf(RESULT_TABLE_NAME));
getUtility().getAdmin().truncateTable(TableName.valueOf(RESULT_TABLE_NAME), true);
.disableTable(TableName.valueOf(IndexVerificationResultRepository.getResultTableName()));
getUtility().getAdmin().truncateTable(
TableName.valueOf(IndexVerificationResultRepository.getResultTableName()), true);
}

private void assertDisableLogging(Connection conn, int expectedRows,
Expand All @@ -1599,7 +1605,7 @@ private void assertDisableLogging(Connection conn, int expectedRows,
assertEquals(expectedRows, rows.size());
} catch (AssertionError e) {
TestUtil.dumpTable(conn,
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
throw e;
}
if (expectedRows > 0) {
Expand All @@ -1609,8 +1615,8 @@ private void assertDisableLogging(Connection conn, int expectedRows,

private void deleteOneRowFromResultTable(Connection conn, Long scn, String indexTable)
throws SQLException, IOException {
Table hIndexToolTable =
conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(RESULT_TABLE_NAME_BYTES);
Table hIndexToolTable = conn.unwrap(PhoenixConnection.class).getQueryServices()
.getTable(IndexVerificationResultRepository.getResultTableNameBytes());
Scan s = new Scan();
s.setRowPrefixFilter(
Bytes.toBytes(String.format("%s%s%s", scn, ROW_KEY_SEPARATOR, indexTable)));
Expand All @@ -1620,8 +1626,8 @@ private void deleteOneRowFromResultTable(Connection conn, Long scn, String index

private List<String> verifyRunStatusFromResultTable(Connection conn, Long scn, String indexTable,
int totalRows, List<String> expectedStatus) throws SQLException, IOException {
Table hIndexToolTable =
conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(RESULT_TABLE_NAME_BYTES);
Table hIndexToolTable = conn.unwrap(PhoenixConnection.class).getQueryServices()
.getTable(IndexVerificationResultRepository.getResultTableNameBytes());
Assert.assertEquals(totalRows, TestUtil.getRowCount(hIndexToolTable, false));
List<String> output = new ArrayList<>();
Scan s = new Scan();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -394,11 +394,11 @@ public void testSecondaryIndex() throws Exception {
protected static void dropIndexToolTables(Connection conn) throws Exception {
Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
TableName indexToolOutputTable =
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES);
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableNameBytes());
admin.disableTable(indexToolOutputTable);
admin.deleteTable(indexToolOutputTable);
TableName indexToolResultTable =
TableName.valueOf(IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES);
TableName.valueOf(IndexVerificationResultRepository.getResultTableNameBytes());
admin.disableTable(indexToolResultTable);
admin.deleteTable(indexToolResultTable);
}
Expand Down Expand Up @@ -443,7 +443,7 @@ public static Cell getErrorMessageFromIndexToolOutputTable(Connection conn,
byte[] indexTableFullNameBytes = Bytes.toBytes(indexTableFullName);
byte[] dataTableFullNameBytes = Bytes.toBytes(dataTableFullName);
Table hIndexTable = conn.unwrap(PhoenixConnection.class).getQueryServices()
.getTable(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES);
.getTable(IndexVerificationOutputRepository.getOutputTableNameBytes());
Scan scan = new Scan();
ResultScanner scanner = hIndexTable.getScanner(scan);
boolean dataTableNameCheck = false;
Expand Down Expand Up @@ -488,7 +488,7 @@ public static Cell getErrorMessageFromIndexToolOutputTable(Connection conn,
assertTrue("Error message cell was null", errorMessageCell != null);
verifyIndexTableRowKey(CellUtil.cloneRow(errorMessageCell), indexTableFullName);
hIndexTable = conn.unwrap(PhoenixConnection.class).getQueryServices()
.getTable(IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES);
.getTable(IndexVerificationResultRepository.getResultTableNameBytes());
scan = new Scan();
scanner = hIndexTable.getScanner(scan);
Result result = scanner.next();
Expand Down Expand Up @@ -1081,7 +1081,7 @@ public static long verifyIndexTable(String fullTableName, String fullIndexName,
IndexTool indexTool = IndexToolIT.runIndexTool(false, schemaName, tableName, indexName, null,
0, IndexTool.IndexVerifyType.ONLY);
TestUtil.dumpTable(conn,
TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
TableName.valueOf(IndexVerificationOutputRepository.getOutputTableName()));
Counters counters = indexTool.getJob().getCounters();
LOGGER.info(counters.toString());
assertEquals(0, counters.findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
Expand Down
Loading