Skip to content

Phoenix 6219 GlobalIndexChecker doesn't work for SingleCell indexes #993

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gokceni wants to merge 1 commit into from
Closed

Phoenix 6219 GlobalIndexChecker doesn't work for SingleCell indexes #993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,13 @@ public void testSynchronousDeletesAndUpsertValues() throws Exception {
public void testConcurrentDeletesAndUpsertValues() throws Exception {
final String tableName = generateUniqueName();
final String indexName = generateUniqueName();
final String singleCellindexName = "SC_" + generateUniqueName();
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute("CREATE TABLE " + tableName
+ "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 INTEGER, CONSTRAINT pk PRIMARY KEY (k1,k2))");
TestUtil.addCoprocessor(conn, tableName, DelayingRegionObserver.class);
conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(v1)");
conn.createStatement().execute("CREATE INDEX " + singleCellindexName + " ON " + tableName + "(v1) IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=2");
final CountDownLatch doneSignal = new CountDownLatch(2);
Runnable r1 = new Runnable() {

Expand Down Expand Up @@ -263,6 +265,7 @@ public void testConcurrentDeletesAndUpsertValues() throws Exception {

doneSignal.await(60, TimeUnit.SECONDS);
verifyIndexTable(tableName, indexName, conn);
verifyIndexTable(tableName, singleCellindexName, conn);
}

@Test
Expand Down
34 changes: 23 additions & 11 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner;
import org.apache.phoenix.hbase.index.IndexRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.BaseTest;
Expand All @@ -57,6 +56,8 @@

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Tests for the {@link IndexTool}
Expand All @@ -67,19 +68,30 @@ public class IndexExtendedIT extends BaseTest {
private final boolean localIndex;
private final boolean useViewIndex;
private final String tableDDLOptions;
private final String indexDDLOptions;
private final boolean mutable;
private final boolean useSnapshot;

public IndexExtendedIT( boolean mutable, boolean localIndex, boolean useViewIndex, boolean useSnapshot) {
this.localIndex = localIndex;
this.useViewIndex = useViewIndex;
this.mutable = mutable;
this.useSnapshot = useSnapshot;
StringBuilder optionBuilder = new StringBuilder();
StringBuilder indexOptionBuilder = new StringBuilder();
if (!mutable) {
optionBuilder.append(" IMMUTABLE_ROWS=true ");
}

if (!localIndex) {
if (!(optionBuilder.length() == 0)) {
optionBuilder.append(",");
}
optionBuilder.append(" IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0 ");
indexOptionBuilder.append(" IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS,COLUMN_ENCODED_BYTES=2 ");
}
optionBuilder.append(" SPLIT ON(1,2)");
this.indexDDLOptions = indexOptionBuilder.toString();
this.tableDDLOptions = optionBuilder.toString();
}

Expand All @@ -102,7 +114,7 @@ public static synchronized Collection<Boolean[]> data() {
for (boolean localIndex : Booleans) {
for (boolean useViewIndex : Booleans) {
for (boolean useSnapshot : Booleans) {
list.add(new Boolean[] { mutable, localIndex, useViewIndex, useSnapshot });
list.add(new Boolean[] { mutable, localIndex, useViewIndex, useSnapshot});
}
}
}
Expand Down Expand Up @@ -139,7 +151,7 @@ public void testMutableIndexWithUpdates() throws Exception {
IndexToolIT.upsertRow(stmt1, id++);
conn.commit();

stmt.execute(String.format("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX %s ON %s (UPPER(NAME, 'en_US')) ASYNC ", indexTableName,dataTableFullName));
stmt.execute(String.format("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX %s ON %s (UPPER(NAME, 'en_US')) ASYNC %s" , indexTableName,dataTableFullName, this.indexDDLOptions));

//update a row
stmt1.setInt(1, 1);
Expand Down Expand Up @@ -178,7 +190,7 @@ public void testMutableIndexWithUpdates() throws Exception {
conn.close();
}
}

@Test
public void testDeleteFromImmutable() throws Exception {
if (mutable) {
Expand All @@ -190,7 +202,7 @@ public void testDeleteFromImmutable() throws Exception {
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
String indexTableName = generateUniqueName();
String indexTableName = "IDX_" + generateUniqueName();
String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
Expand All @@ -208,7 +220,7 @@ public void testDeleteFromImmutable() throws Exception {
conn.createStatement().execute("upsert into " + dataTableFullName + " (pk1, pk2, pk3) values ('a', '1', '1')");
conn.createStatement().execute("upsert into " + dataTableFullName + " (pk1, pk2, pk3) values ('b', '2', '2')");
conn.commit();
conn.createStatement().execute("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexTableName + " ON " + dataTableFullName + " (pk3, pk2) ASYNC");
conn.createStatement().execute("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexTableName + " ON " + dataTableFullName + " (pk3, pk2) ASYNC " + this.indexDDLOptions);

// this delete will be issued at a timestamp later than the above timestamp of the index table
conn.createStatement().execute("delete from " + dataTableFullName + " where pk1 = 'a'");
Expand Down Expand Up @@ -287,8 +299,8 @@ public void testBuildDisabledIndex() throws Exception {
Table hIndexTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(physicalTableNameOfIndex));

stmt.execute(
String.format("CREATE INDEX %s ON %s (UPPER(NAME, 'en_US')) ", indexName,
baseTableFullNameOfIndex));
String.format("CREATE INDEX %s ON %s (UPPER(NAME, 'en_US')) %s", indexName,
baseTableFullNameOfIndex, this.indexDDLOptions));
long dataCnt = getRowCount(conn, dataTableFullName);
long indexCnt = getUtility().countRows(hIndexTable);
assertEquals(dataCnt, indexCnt);
Expand Down Expand Up @@ -344,8 +356,8 @@ public void testIndexStateOnException() throws Exception {
// lead to any change on index and thus index verify during index rebuild should fail
IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(true);
stmt.execute(String.format(
"CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC",
indexTableName, dataTableFullName));
"CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC %s",
indexTableName, dataTableFullName, this.indexDDLOptions));

// Verify that the index table is not in the ACTIVE state
assertFalse(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));
Expand Down
20 changes: 16 additions & 4 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexRepairRegionScannerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,25 +72,37 @@
public class IndexRepairRegionScannerIT extends ParallelStatsDisabledIT {

private final String tableDDLOptions;
private final String indexDDLOptions;
private boolean mutable;
@Rule
public ExpectedException exceptionRule = ExpectedException.none();

public IndexRepairRegionScannerIT(boolean mutable) {
public IndexRepairRegionScannerIT(boolean mutable, boolean singleCellIndex) {
StringBuilder optionBuilder = new StringBuilder();
StringBuilder indexOptionBuilder = new StringBuilder();
this.mutable = mutable;
if (!mutable) {
optionBuilder.append(" IMMUTABLE_ROWS=true ");
}
if (singleCellIndex) {
if (!(optionBuilder.length() == 0)) {
optionBuilder.append(",");
}
optionBuilder.append(" IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0 ");
indexOptionBuilder.append(" IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS,COLUMN_ENCODED_BYTES=2");
}
optionBuilder.append(" SPLIT ON(1,2)");
this.indexDDLOptions = indexOptionBuilder.toString();
this.tableDDLOptions = optionBuilder.toString();
}

@Parameterized.Parameters(name = "mutable={0}")
@Parameterized.Parameters(name = "mutable={0}, singleCellIndex={1}")
public static synchronized Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{true},
{false} });
{true, true},
{true, false},
{false, true},
{false, false}});
}

@Before
Expand Down
8 changes: 8 additions & 0 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,9 @@ private boolean isOnlyIndexSingleCell() {
* a covered index value is incorrect. Scrutiny should report the invalid row
*/
@Test public void testCoveredValueIncorrect() throws Exception {
if (isOnlyIndexSingleCell()) {
return;
}
// insert one valid row
upsertRow(dataTableUpsertStmt, 1, "name-1", 94010);
conn.commit();
Expand All @@ -322,6 +325,7 @@ private boolean isOnlyIndexSingleCell() {
List<Job> completedJobs = runScrutiny(schemaName, dataTableName, indexTableName);
Job job = completedJobs.get(0);
assertTrue(job.isSuccessful());

Counters counters = job.getCounters();
assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT));
assertEquals(1, getCounterValue(counters, INVALID_ROW_COUNT));
Expand Down Expand Up @@ -405,6 +409,7 @@ private boolean isOnlyIndexSingleCell() {
runScrutiny(schemaName, dataTableName, indexTableName, 10L, SourceTable.INDEX_TABLE_SOURCE);
Job job = completedJobs.get(0);
assertTrue(job.isSuccessful());

Counters counters = job.getCounters();
assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT));
assertEquals(2, getCounterValue(counters, INVALID_ROW_COUNT));
Expand All @@ -415,6 +420,9 @@ private boolean isOnlyIndexSingleCell() {
* incorrectly indexed row, it should be reported in each direction
*/
@Test public void testBothDataAndIndexAsSource() throws Exception {
if (isOnlyIndexSingleCell()) {
return;
}
// insert one valid row
upsertRow(dataTableUpsertStmt, 1, "name-1", 94010);
conn.commit();
Expand Down
4 changes: 4 additions & 0 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,10 @@ public IndexToolIT(String transactionProvider, boolean mutable, boolean localInd
this.tableDDLOptions = optionBuilder.toString();
StringBuilder indexOptionBuilder = new StringBuilder();
if (!localIndex && transactionProvider == null) {
if (!(optionBuilder.length() == 0)) {
optionBuilder.append(",");
}
optionBuilder.append(" COLUMN_ENCODED_BYTES=0");
indexOptionBuilder.append(" IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS,COLUMN_ENCODED_BYTES=2");
}
this.indexDDLOptions = indexOptionBuilder.toString();
Expand Down
Loading