diff --git a/Jenkinsfile b/Jenkinsfile index 46a06283885..49a44d3766f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { stage('BuildAndTest') { options { - timeout(time: 5, unit: 'HOURS') + timeout(time: 9, unit: 'HOURS') } steps { dir("HBASE_${HBASE_PROFILE}") { diff --git a/Jenkinsfile.yetus b/Jenkinsfile.yetus index a0e9c43f654..7e99f45b056 100644 --- a/Jenkinsfile.yetus +++ b/Jenkinsfile.yetus @@ -37,7 +37,7 @@ pipeline { } options { - timeout(time: 5, unit: 'HOURS') + timeout(time: 9, unit: 'HOURS') } steps { diff --git a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g index 2324c78d673..9eccd4df007 100644 --- a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g @@ -85,6 +85,8 @@ tokens PRE='pre'; POST='post'; CHANGE='change'; + IDX_MUTATIONS='idx_mutations'; + DATA_ROW_STATE='data_row_state'; LATEST='latest'; ALL='all'; INDEX='index'; @@ -161,6 +163,9 @@ tokens REGIONS = 'regions'; NOVERIFY = 'noverify'; RETURNING = 'returning'; + CONSISTENCY = 'consistency'; + EVENTUAL = 'eventual'; + STRONG = 'strong'; } @@ -209,6 +214,7 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.IllegalDataException; import org.apache.phoenix.schema.PIndexState; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTable.CDCChangeScope; @@ -585,7 +591,7 @@ cdc_change_scopes returns [Set ret] ; cdc_change_scope returns [CDCChangeScope ret] - : v=(PRE | POST | CHANGE) + : v=(PRE | POST | CHANGE | IDX_MUTATIONS | DATA_ROW_STATE) { ret = CDCChangeScope.valueOf(v.getText().toUpperCase()); } @@ -699,8 +705,8 @@ drop_cdc_node returns [DropCDCStatement ret] // Parse a alter index statement alter_index_node returns [AlterIndexStatement ret] : ALTER INDEX (IF ex=EXISTS)? i=index_name ON t=from_table_name - ((s=(USABLE | UNUSABLE | REBUILD (isRebuildAll=ALL)? | DISABLE | ACTIVE)) (async=ASYNC)? ((SET?)p=fam_properties)?) - {ret = factory.alterIndex(factory.namedTable(null, TableName.create(t.getSchemaName(), i.getName())), t.getTableName(), ex!=null, PIndexState.valueOf(SchemaUtil.normalizeIdentifier(s.getText())), isRebuildAll!=null, async!=null, p); } + ((s=(USABLE | UNUSABLE | REBUILD (isRebuildAll=ALL)? | DISABLE | ACTIVE)) (async=ASYNC)? ((SET?)p=fam_properties)? | (CONSISTENCY EQ c=(STRONG | EVENTUAL))) + {ret = factory.alterIndex(factory.namedTable(null, TableName.create(t.getSchemaName(), i.getName())), t.getTableName(), ex!=null, s!=null ? PIndexState.valueOf(SchemaUtil.normalizeIdentifier(s.getText())) : null, isRebuildAll!=null, async!=null, p, c!=null ? IndexConsistency.valueOf(SchemaUtil.normalizeIdentifier(c.getText())) : null); } ; // Parse a trace statement. @@ -1318,6 +1324,9 @@ identifier returns [String ret] parseNoReserved returns [String ret] : n=NAME { $ret = n.getText(); } + | CONSISTENCY { $ret = "CONSISTENCY"; } + | EVENTUAL { $ret = "EVENTUAL"; } + | STRONG { $ret = "STRONG"; } ; case_statement returns [ParseNode ret] diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index ad31baeaf84..8a091fbdc06 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -1544,7 +1544,8 @@ public static PTable joinProjectedTables(PTable left, PTable right, JoinType typ .setDisableWAL(PTable.DEFAULT_DISABLE_WAL).setMultiTenant(left.isMultiTenant()) .setStoreNulls(left.getStoreNulls()).setViewType(left.getViewType()) .setViewIndexIdType(left.getviewIndexIdType()).setViewIndexId(left.getViewIndexId()) - .setIndexType(left.getIndexType()).setTransactionProvider(left.getTransactionProvider()) + .setIndexType(left.getIndexType()).setIndexConsistency(left.getIndexConsistency()) + .setTransactionProvider(left.getTransactionProvider()) .setUpdateCacheFrequency(left.getUpdateCacheFrequency()) .setNamespaceMapped(left.isNamespaceMapped()) .setAutoPartitionSeqName(left.getAutoPartitionSeqName()) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java index cc7120f71da..cc5f3144767 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java @@ -91,7 +91,7 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0 = MIN_TABLE_TIMESTAMP + 33; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_1_0 = MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_2_0 = MIN_TABLE_TIMESTAMP + 38; - public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 = MIN_TABLE_TIMESTAMP + 42; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 = MIN_TABLE_TIMESTAMP + 44; // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* // constants public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexCDCConsumerSource.java b/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexCDCConsumerSource.java new file mode 100644 index 00000000000..c278c1fa4ef --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexCDCConsumerSource.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hbase.index.metrics; + +import org.apache.hadoop.hbase.metrics.BaseSource; + +/** + * IndexCDCConsumer metrics for eventually consistent index updates. + */ +public interface MetricsIndexCDCConsumerSource extends BaseSource { + + String METRICS_NAME = "IndexCDCConsumer"; + String METRICS_CONTEXT = "phoenix"; + String METRICS_DESCRIPTION = "Metrics about the Phoenix Index CDC Consumer"; + String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME; + + String CDC_BATCH_PROCESS_TIME = "cdcBatchProcessTime"; + String CDC_BATCH_PROCESS_TIME_DESC = + "Histogram for the end-to-end time in milliseconds for processing one CDC batch"; + + String CDC_MUTATION_GENERATE_TIME = "cdcMutationGenerateTime"; + String CDC_MUTATION_GENERATE_TIME_DESC = + "Histogram for the time in milliseconds to generate index mutations from data row states"; + + String CDC_MUTATION_APPLY_TIME = "cdcMutationApplyTime"; + String CDC_MUTATION_APPLY_TIME_DESC = + "Histogram for the time in milliseconds to apply index mutations to index tables"; + + String CDC_BATCH_COUNT = "cdcBatchCount"; + String CDC_BATCH_COUNT_DESC = "The number of CDC batches processed"; + + String CDC_MUTATION_COUNT = "cdcMutationCount"; + String CDC_MUTATION_COUNT_DESC = "The number of individual index mutations applied through CDC"; + + String CDC_BATCH_FAILURE_COUNT = "cdcBatchFailureCount"; + String CDC_BATCH_FAILURE_COUNT_DESC = "The number of CDC batch processing failures"; + + String CDC_INDEX_UPDATE_LAG = "cdcIndexUpdateLag"; + String CDC_INDEX_UPDATE_LAG_DESC = + "Histogram for the lag in milliseconds between current time and the last processed CDC event"; + + /** + * Updates the CDC batch processing time histogram. + * @param dataTableName physical data table name + * @param t time taken in milliseconds + */ + void updateCdcBatchProcessTime(String dataTableName, long t); + + /** + * Updates the CDC mutation generation time histogram. + * @param dataTableName physical data table name + * @param t time taken in milliseconds + */ + void updateCdcMutationGenerateTime(String dataTableName, long t); + + /** + * Updates the CDC mutation apply time histogram. + * @param dataTableName physical data table name + * @param t time taken in milliseconds + */ + void updateCdcMutationApplyTime(String dataTableName, long t); + + /** + * Increments the CDC batch count. + * @param dataTableName physical data table name + */ + void incrementCdcBatchCount(String dataTableName); + + /** + * Increments the CDC mutation count by the given amount. + * @param dataTableName physical data table name + * @param count number of mutations applied + */ + void incrementCdcMutationCount(String dataTableName, long count); + + /** + * Increments the CDC batch failure count. + * @param dataTableName physical data table name + */ + void incrementCdcBatchFailureCount(String dataTableName); + + /** + * Updates the CDC lag histogram. + * @param dataTableName physical data table name + * @param lag lag in milliseconds between current time and last processed event + */ + void updateCdcLag(String dataTableName, long lag); +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexCDCConsumerSourceImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexCDCConsumerSourceImpl.java new file mode 100644 index 00000000000..71c63826339 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexCDCConsumerSourceImpl.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hbase.index.metrics; + +import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; + +/** + * Implementation for tracking IndexCDCConsumer metrics. + */ +public class MetricsIndexCDCConsumerSourceImpl extends BaseSourceImpl + implements MetricsIndexCDCConsumerSource { + + private final MetricHistogram cdcBatchProcessTimeHisto; + private final MetricHistogram cdcMutationGenerateTimeHisto; + private final MetricHistogram cdcMutationApplyTimeHisto; + private final MutableFastCounter cdcBatchCounter; + private final MutableFastCounter cdcMutationCounter; + private final MutableFastCounter cdcBatchFailureCounter; + private final MetricHistogram cdcIndexUpdateLagHisto; + + public MetricsIndexCDCConsumerSourceImpl() { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); + } + + public MetricsIndexCDCConsumerSourceImpl(String metricsName, String metricsDescription, + String metricsContext, String metricsJmxContext) { + super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + + cdcBatchProcessTimeHisto = + getMetricsRegistry().newHistogram(CDC_BATCH_PROCESS_TIME, CDC_BATCH_PROCESS_TIME_DESC); + cdcMutationGenerateTimeHisto = getMetricsRegistry().newHistogram(CDC_MUTATION_GENERATE_TIME, + CDC_MUTATION_GENERATE_TIME_DESC); + cdcMutationApplyTimeHisto = + getMetricsRegistry().newHistogram(CDC_MUTATION_APPLY_TIME, CDC_MUTATION_APPLY_TIME_DESC); + cdcBatchCounter = getMetricsRegistry().newCounter(CDC_BATCH_COUNT, CDC_BATCH_COUNT_DESC, 0L); + cdcMutationCounter = + getMetricsRegistry().newCounter(CDC_MUTATION_COUNT, CDC_MUTATION_COUNT_DESC, 0L); + cdcBatchFailureCounter = + getMetricsRegistry().newCounter(CDC_BATCH_FAILURE_COUNT, CDC_BATCH_FAILURE_COUNT_DESC, 0L); + cdcIndexUpdateLagHisto = + getMetricsRegistry().newHistogram(CDC_INDEX_UPDATE_LAG, CDC_INDEX_UPDATE_LAG_DESC); + } + + @Override + public void updateCdcBatchProcessTime(String dataTableName, long t) { + incrementTableSpecificHistogram(CDC_BATCH_PROCESS_TIME, dataTableName, t); + cdcBatchProcessTimeHisto.add(t); + } + + @Override + public void updateCdcMutationGenerateTime(String dataTableName, long t) { + incrementTableSpecificHistogram(CDC_MUTATION_GENERATE_TIME, dataTableName, t); + cdcMutationGenerateTimeHisto.add(t); + } + + @Override + public void updateCdcMutationApplyTime(String dataTableName, long t) { + incrementTableSpecificHistogram(CDC_MUTATION_APPLY_TIME, dataTableName, t); + cdcMutationApplyTimeHisto.add(t); + } + + @Override + public void incrementCdcBatchCount(String dataTableName) { + incrementTableSpecificCounter(CDC_BATCH_COUNT, dataTableName); + cdcBatchCounter.incr(); + } + + @Override + public void incrementCdcMutationCount(String dataTableName, long count) { + MutableFastCounter tableCounter = + getMetricsRegistry().getCounter(getMetricName(CDC_MUTATION_COUNT, dataTableName), 0); + tableCounter.incr(count); + cdcMutationCounter.incr(count); + } + + @Override + public void incrementCdcBatchFailureCount(String dataTableName) { + incrementTableSpecificCounter(CDC_BATCH_FAILURE_COUNT, dataTableName); + cdcBatchFailureCounter.incr(); + } + + @Override + public void updateCdcLag(String dataTableName, long lag) { + incrementTableSpecificHistogram(CDC_INDEX_UPDATE_LAG, dataTableName, lag); + cdcIndexUpdateLagHisto.add(lag); + } + + private void incrementTableSpecificCounter(String baseName, String tableName) { + MutableFastCounter tableCounter = + getMetricsRegistry().getCounter(getMetricName(baseName, tableName), 0); + tableCounter.incr(); + } + + private void incrementTableSpecificHistogram(String baseName, String tableName, long t) { + MetricHistogram tableHistogram = + getMetricsRegistry().getHistogram(getMetricName(baseName, tableName)); + tableHistogram.add(t); + } + + private String getMetricName(String baseName, String tableName) { + return baseName + "." + tableName; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceFactory.java b/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceFactory.java index e579b2f836e..55b648c53a1 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceFactory.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceFactory.java @@ -24,6 +24,7 @@ public class MetricsIndexerSourceFactory { private static final MetricsIndexerSourceFactory INSTANCE = new MetricsIndexerSourceFactory(); private volatile MetricsIndexerSource indexerSource; private GlobalIndexCheckerSource globalIndexCheckerSource; + private MetricsIndexCDCConsumerSource indexCDCConsumerSource; private MetricsIndexerSourceFactory() { } @@ -45,4 +46,11 @@ public synchronized GlobalIndexCheckerSource getGlobalIndexCheckerSource() { } return INSTANCE.globalIndexCheckerSource; } + + public synchronized MetricsIndexCDCConsumerSource getIndexCDCConsumerSource() { + if (INSTANCE.indexCDCConsumerSource == null) { + INSTANCE.indexCDCConsumerSource = new MetricsIndexCDCConsumerSourceImpl(); + } + return INSTANCE.indexCDCConsumerSource; + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java index e2b33a17d7e..70b83d17950 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java @@ -108,6 +108,7 @@ import org.apache.phoenix.schema.tuple.BaseTuple; import org.apache.phoenix.schema.tuple.MultiKeyValueTuple; import org.apache.phoenix.schema.tuple.ValueGetterTuple; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PBoolean; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarbinaryEncoded; @@ -477,10 +478,12 @@ public static IndexMaintainer getIndexMaintainer(List maintaine private Expression indexWhere; private Set indexWhereColumns; private boolean isCDCIndex; + private IndexConsistency indexConsistency; protected IndexMaintainer(RowKeySchema dataRowKeySchema, boolean isDataTableSalted) { this.dataRowKeySchema = dataRowKeySchema; this.isDataTableSalted = isDataTableSalted; + this.indexConsistency = null; } private IndexMaintainer(final PTable dataTable, final PTable index, PhoenixConnection connection) @@ -501,6 +504,7 @@ private IndexMaintainer(final PTable dataTable, final PTable cdcTable, final PTa this.isUncovered = index.getIndexType() == IndexType.UNCOVERED_GLOBAL; this.encodingScheme = index.getEncodingScheme(); this.isCDCIndex = CDCUtil.isCDCIndex(index); + this.indexConsistency = index.getIndexConsistency(); // null check for b/w compatibility this.encodingScheme = index.getEncodingScheme() == null @@ -1933,6 +1937,7 @@ public PDataType getDataType() { this.encodingScheme = QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; this.dataImmutableStorageScheme = ImmutableStorageScheme.ONE_CELL_PER_COLUMN; this.dataEncodingScheme = QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + this.indexConsistency = null; initCachedState(); } @@ -2068,6 +2073,11 @@ public static IndexMaintainer fromProto(ServerCachingProtos.IndexMaintainer prot } else { maintainer.isCDCIndex = false; } + if (proto.hasIndexConsistency()) { + maintainer.indexConsistency = IndexConsistency.valueOf(proto.getIndexConsistency()); + } else { + maintainer.indexConsistency = null; + } maintainer.nDataTableSaltBuckets = proto.hasDataTableSaltBuckets() ? proto.getDataTableSaltBuckets() : -1; maintainer.initCachedState(); @@ -2223,6 +2233,9 @@ public static ServerCachingProtos.IndexMaintainer toProto(IndexMaintainer mainta } } builder.setIsCDCIndex(maintainer.isCDCIndex); + if (maintainer.indexConsistency != null) { + builder.setIndexConsistency(maintainer.indexConsistency.name()); + } if (maintainer.isDataTableSalted) { builder.setDataTableSaltBuckets(maintainer.nDataTableSaltBuckets); } @@ -2609,6 +2622,10 @@ public Set getIndexedColumns() { return indexedColumns; } + public IndexConsistency getIndexConsistency() { + return indexConsistency; + } + public static class UDFParseNodeVisitor extends StatelessTraverseAllParseNodeVisitor { private Map udfParseNodes; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index 3bc68c5ddb7..ed63485440f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -213,6 +213,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { public static final byte[] VIEW_TYPE_BYTES = Bytes.toBytes(VIEW_TYPE); public static final String INDEX_TYPE = "INDEX_TYPE"; public static final byte[] INDEX_TYPE_BYTES = Bytes.toBytes(INDEX_TYPE); + public static final String INDEX_CONSISTENCY = "INDEX_CONSISTENCY"; + public static final byte[] INDEX_CONSISTENCY_BYTES = Bytes.toBytes(INDEX_CONSISTENCY); public static final String LINK_TYPE = "LINK_TYPE"; public static final byte[] LINK_TYPE_BYTES = Bytes.toBytes(LINK_TYPE); public static final String TASK_TYPE = "TASK_TYPE"; @@ -498,6 +500,16 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { "PARTITION_END_TIME IS NOT NULL " + "AND TO_NUMBER(CURRENT_TIME()) - TO_NUMBER(PHOENIX_ROW_TIMESTAMP()) >= %d"; + public static final String SYSTEM_IDX_CDC_TRACKER_TABLE = "IDX_CDC_TRACKER"; + public static final String SYSTEM_IDX_CDC_TRACKER_NAME = + SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_IDX_CDC_TRACKER_TABLE); + public static final String LAST_TIMESTAMP = "LAST_TIMESTAMP"; + public static final String OWNER_PARTITION_ID = "OWNER_PARTITION_ID"; + public static final String TRACKER_STATUS = "STATUS"; + public static final String TRACKER_STATUS_IN_PROGRESS = "i"; + public static final String TRACKER_STATUS_COMPLETE = "c"; + public static final int IDX_CDC_TRACKER_TTL = 10 * 24 * 60 * 60; + public static final String QUERY_ID = "QUERY_ID"; public static final String USER = "USER"; public static final String CLIENT_IP = "CLIENT_IP"; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 136d7a2d621..e8c89451b2c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -220,6 +220,7 @@ import org.apache.phoenix.schema.tuple.MultiKeyValueTuple; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.types.PVarchar; @@ -1623,9 +1624,10 @@ private static class ExecutableCreateIndexStatement extends CreateIndexStatement public ExecutableCreateIndexStatement(NamedNode indexName, NamedTableNode dataTable, IndexKeyConstraint ikConstraint, List includeColumns, List splits, ListMultimap> props, boolean ifNotExists, IndexType indexType, - boolean async, int bindCount, Map udfParseNodes, ParseNode where) { + boolean async, int bindCount, Map udfParseNodes, ParseNode where, + IndexConsistency indexConsistency) { super(indexName, dataTable, ikConstraint, includeColumns, splits, props, ifNotExists, - indexType, async, bindCount, udfParseNodes, where); + indexType, async, bindCount, udfParseNodes, where, indexConsistency); } @SuppressWarnings("unchecked") @@ -1857,12 +1859,19 @@ public MutationState execute() throws SQLException { private static class ExecutableAlterIndexStatement extends AlterIndexStatement implements CompilableStatement { - public ExecutableAlterIndexStatement(NamedTableNode indexTableNode, String dataTableName, + ExecutableAlterIndexStatement(NamedTableNode indexTableNode, String dataTableName, boolean ifExists, PIndexState state, boolean isRebuildAll, boolean async, ListMultimap> props) { super(indexTableNode, dataTableName, ifExists, state, isRebuildAll, async, props); } + ExecutableAlterIndexStatement(NamedTableNode indexTableNode, String dataTableName, + boolean ifExists, PIndexState state, boolean isRebuildAll, boolean async, + ListMultimap> props, IndexConsistency indexConsistency) { + super(indexTableNode, dataTableName, ifExists, state, isRebuildAll, async, props, + indexConsistency); + } + @SuppressWarnings("unchecked") @Override public MutationPlan compilePlan(final PhoenixStatement stmt, Sequence.ValueOp seqAction) @@ -2248,7 +2257,8 @@ public CreateIndexStatement createIndex(NamedNode indexName, NamedTableNode data ListMultimap> props, boolean ifNotExists, IndexType indexType, boolean async, int bindCount, Map udfParseNodes, ParseNode where) { return new ExecutableCreateIndexStatement(indexName, dataTable, ikConstraint, includeColumns, - splits, props, ifNotExists, indexType, async, bindCount, udfParseNodes, where); + splits, props, ifNotExists, indexType, async, bindCount, udfParseNodes, where, + CreateIndexStatement.getIndexConsistency(props)); } @Override @@ -2305,6 +2315,14 @@ public AlterIndexStatement alterIndex(NamedTableNode indexTableNode, String data isRebuildAll, async, props); } + @Override + public AlterIndexStatement alterIndex(NamedTableNode indexTableNode, String dataTableName, + boolean ifExists, PIndexState state, boolean isRebuildAll, boolean async, + ListMultimap> props, IndexConsistency indexConsistency) { + return new ExecutableAlterIndexStatement(indexTableNode, dataTableName, ifExists, state, + isRebuildAll, async, props, indexConsistency); + } + @Override public TraceStatement trace(boolean isTraceOn, double samplingRate) { return new ExecutableTraceStatement(isTraceOn, samplingRate); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/AlterIndexStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/AlterIndexStatement.java index 7ef5a618286..2b12e32ae6a 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/AlterIndexStatement.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/AlterIndexStatement.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableListMultimap; import org.apache.phoenix.thirdparty.com.google.common.collect.ListMultimap; @@ -31,16 +32,23 @@ public class AlterIndexStatement extends SingleTableStatement { private boolean async; private boolean isRebuildAll; private ListMultimap> props; + private final IndexConsistency indexConsistency; private static final PTableType tableType = PTableType.INDEX; public AlterIndexStatement(NamedTableNode indexTableNode, String dataTableName, boolean ifExists, PIndexState indexState, boolean isRebuildAll, boolean async) { - this(indexTableNode, dataTableName, ifExists, indexState, isRebuildAll, async, null); + this(indexTableNode, dataTableName, ifExists, indexState, isRebuildAll, async, null, null); } public AlterIndexStatement(NamedTableNode indexTableNode, String dataTableName, boolean ifExists, PIndexState indexState, boolean isRebuildAll, boolean async, ListMultimap> props) { + this(indexTableNode, dataTableName, ifExists, indexState, isRebuildAll, async, props, null); + } + + public AlterIndexStatement(NamedTableNode indexTableNode, String dataTableName, boolean ifExists, + PIndexState indexState, boolean isRebuildAll, boolean async, + ListMultimap> props, IndexConsistency indexConsistency) { super(indexTableNode, 0); this.dataTableName = dataTableName; this.ifExists = ifExists; @@ -48,6 +56,7 @@ public AlterIndexStatement(NamedTableNode indexTableNode, String dataTableName, this.async = async; this.isRebuildAll = isRebuildAll; this.props = props == null ? ImmutableListMultimap.> of() : props; + this.indexConsistency = indexConsistency; } public String getTableName() { @@ -82,4 +91,8 @@ public ListMultimap> getProps() { public PTableType getTableType() { return tableType; } + + public IndexConsistency getIndexConsistency() { + return indexConsistency; + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java index ec71fb10246..8cc7685a313 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java @@ -21,7 +21,9 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.thirdparty.com.google.common.collect.ArrayListMultimap; import org.apache.phoenix.thirdparty.com.google.common.collect.ListMultimap; @@ -37,11 +39,21 @@ public class CreateIndexStatement extends SingleTableStatement { private final boolean async; private final Map udfParseNodes; private final ParseNode where; + private final IndexConsistency indexConsistency; public CreateIndexStatement(NamedNode indexTableName, NamedTableNode dataTable, IndexKeyConstraint indexKeyConstraint, List includeColumns, List splits, ListMultimap> props, boolean ifNotExists, IndexType indexType, boolean async, int bindCount, Map udfParseNodes, ParseNode where) { + this(indexTableName, dataTable, indexKeyConstraint, includeColumns, splits, props, ifNotExists, + indexType, async, bindCount, udfParseNodes, where, getIndexConsistency(props)); + } + + public CreateIndexStatement(NamedNode indexTableName, NamedTableNode dataTable, + IndexKeyConstraint indexKeyConstraint, List includeColumns, List splits, + ListMultimap> props, boolean ifNotExists, IndexType indexType, + boolean async, int bindCount, Map udfParseNodes, ParseNode where, + IndexConsistency indexConsistency) { super(dataTable, bindCount); this.indexTableName = TableName.create(dataTable.getName().getSchemaName(), indexTableName.getName()); @@ -56,6 +68,7 @@ public CreateIndexStatement(NamedNode indexTableName, NamedTableNode dataTable, this.async = async; this.udfParseNodes = udfParseNodes; this.where = where; + this.indexConsistency = indexConsistency; } public CreateIndexStatement(CreateIndexStatement createStmt, @@ -71,6 +84,23 @@ public CreateIndexStatement(CreateIndexStatement createStmt, this.async = createStmt.isAsync(); this.udfParseNodes = createStmt.getUdfParseNodes(); this.where = createStmt.where; + this.indexConsistency = createStmt.getIndexConsistency(); + } + + public static IndexConsistency + getIndexConsistency(ListMultimap> props) { + IndexConsistency indexConsistency = null; + if (props != null) { + for (Pair prop : props.get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY)) { + if (prop != null && "CONSISTENCY".equalsIgnoreCase(prop.getFirst())) { + Object value = prop.getSecond(); + indexConsistency = + value == null ? null : IndexConsistency.valueOf(value.toString().toUpperCase()); + break; + } + } + } + return indexConsistency; } public IndexKeyConstraint getIndexConstraint() { @@ -112,4 +142,8 @@ public Map getUdfParseNodes() { public ParseNode getWhere() { return where; } + + public IndexConsistency getIndexConsistency() { + return indexConsistency; + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java index f3c373601c9..163b99f8735 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java @@ -49,6 +49,7 @@ import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TypeMismatchException; import org.apache.phoenix.schema.stats.StatisticsCollectionScope; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PBinary; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PLong; @@ -470,6 +471,13 @@ public AlterIndexStatement alterIndex(NamedTableNode indexTableNode, String data async, props); } + public AlterIndexStatement alterIndex(NamedTableNode indexTableNode, String dataTableName, + boolean ifExists, PIndexState state, boolean isRebuildAll, boolean async, + ListMultimap> props, IndexConsistency indexConsistency) { + return new AlterIndexStatement(indexTableNode, dataTableName, ifExists, state, isRebuildAll, + async, props, indexConsistency); + } + public AlterIndexStatement alterIndex(NamedTableNode indexTableNode, String dataTableName, boolean ifExists, PIndexState state) { return new AlterIndexStatement(indexTableNode, dataTableName, ifExists, state, false, false); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 8e867842bc4..0f19965030a 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -60,6 +60,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_STATUS_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_HBASE_TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_COLUMN_NAME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME; @@ -4101,6 +4102,10 @@ protected String getCDCStreamDDL() { return ddl + ",TTL='" + ttlExpression + "'"; } + protected String getIdxCdcTrackerDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_IDX_CDC_TRACKER_METADATA); + } + private String setSystemDDLProperties(String ddl) { return String.format(ddl, props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, @@ -4434,6 +4439,10 @@ private void createOtherSystemTables(PhoenixConnection metaConnection) metaConnection.createStatement().executeUpdate(getCDCStreamDDL()); } catch (TableAlreadyExistsException ignore) { } + try { + metaConnection.createStatement().executeUpdate(getIdxCdcTrackerDDL()); + } catch (TableAlreadyExistsException ignore) { + } } /** @@ -4755,22 +4764,22 @@ protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection met } if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0) { metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 8, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 9, PhoenixDatabaseMetaData.PHYSICAL_TABLE_NAME + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 7, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 8, PhoenixDatabaseMetaData.SCHEMA_VERSION + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 6, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 7, PhoenixDatabaseMetaData.EXTERNAL_SCHEMA_ID + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 5, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 6, PhoenixDatabaseMetaData.STREAMING_TOPIC_NAME + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 4, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 5, PhoenixDatabaseMetaData.INDEX_WHERE + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 3, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 4, PhoenixDatabaseMetaData.CDC_INCLUDE_TABLE + " " + PVarchar.INSTANCE.getSqlTypeName()); /** @@ -4778,19 +4787,16 @@ protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection met * PHOENIX_TTL Column. See PHOENIX-7023 */ metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 2, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 3, PhoenixDatabaseMetaData.TTL + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 1, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 2, PhoenixDatabaseMetaData.ROW_KEY_MATCHER + " " + PVarbinary.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 1, PhoenixDatabaseMetaData.IS_STRICT_TTL + " " + PBoolean.INSTANCE.getSqlTypeName()); - // Values in PHOENIX_TTL column will not be used for further release as PHOENIX_TTL column is - // being deprecated - // and will be removed in later release. To copy copyDataFromPhoenixTTLtoTTL(metaConnection) - // can be used but - // as that feature was not fully built we are not moving old value to new column + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0, PhoenixDatabaseMetaData.INDEX_CONSISTENCY + " CHAR(1)"); // move TTL values stored in descriptor to SYSCAT TTL column. moveTTLFromHBaseLevelTTLToPhoenixLevelTTL(metaConnection); @@ -5009,6 +5015,7 @@ private PhoenixConnection upgradeOtherSystemTablesIfRequired(PhoenixConnection m metaConnection = upgradeSystemMutex(metaConnection); metaConnection = upgradeSystemCDCStreamStatus(metaConnection); metaConnection = upgradeSystemCDCStream(metaConnection); + metaConnection = upgradeSystemIdxCdcTracker(metaConnection); // As this is where the most time will be spent during an upgrade, // especially when there are large number of views. @@ -5331,6 +5338,15 @@ private PhoenixConnection upgradeSystemCDCStream(PhoenixConnection metaConnectio return metaConnection; } + private PhoenixConnection upgradeSystemIdxCdcTracker(PhoenixConnection metaConnection) + throws SQLException { + try { + metaConnection.createStatement().executeUpdate(getIdxCdcTrackerDDL()); + } catch (TableAlreadyExistsException ignored) { + } + return metaConnection; + } + // Special method for adding the column qualifier column for 4.10. private PhoenixConnection addColumnQualifierColumn(PhoenixConnection oldMetaConnection, Long timestamp) throws SQLException { @@ -6931,6 +6947,8 @@ public void deleteAllStreamMetadataForTable(java.sql.Connection conn, String tab "DELETE FROM " + SYSTEM_CDC_STREAM_STATUS_NAME + " WHERE TABLE_NAME = ?"; String deleteStreamPartitionsQuery = "DELETE FROM " + SYSTEM_CDC_STREAM_NAME + " WHERE TABLE_NAME = ?"; + String deleteIdxCdcTrackerQuery = + "DELETE FROM " + SYSTEM_IDX_CDC_TRACKER_NAME + " WHERE TABLE_NAME = ?"; LOGGER.info("Deleting Stream Metadata for table {}", tableName); try (PreparedStatement ps = conn.prepareStatement(deleteStreamStatusQuery)) { ps.setString(1, tableName); @@ -6942,6 +6960,11 @@ public void deleteAllStreamMetadataForTable(java.sql.Connection conn, String tab ps.executeUpdate(); conn.commit(); } + try (PreparedStatement ps = conn.prepareStatement(deleteIdxCdcTrackerQuery)) { + ps.setString(1, tableName); + ps.executeUpdate(); + conn.commit(); + } } /** diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index ad5df1e39e2..2e62e0117ff 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -219,6 +219,10 @@ protected String getCDCStreamDDL() { return setSystemDDLProperties(QueryConstants.CREATE_CDC_STREAM_METADATA); } + protected String getIdxCdcTrackerDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_IDX_CDC_TRACKER_METADATA); + } + private String setSystemDDLProperties(String ddl) { return String.format(ddl, props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, @@ -496,6 +500,10 @@ public void init(String url, Properties props) throws SQLException { metaConnection.createStatement().executeUpdate(getCDCStreamDDL()); } catch (TableAlreadyExistsException ignore) { } + try { + metaConnection.createStatement().executeUpdate(getIdxCdcTrackerDDL()); + } catch (TableAlreadyExistsException ignore) { + } } catch (SQLException e) { sqlE = e; } finally { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java index fe9ea6ef8bf..aa73c833145 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -54,9 +54,11 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IDX_CDC_TRACKER_TTL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_CONSISTENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE; @@ -73,6 +75,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_DDL_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LOGICAL_PARENT_NAME; @@ -88,6 +91,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_PREC_RADIX; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.OLD_METADATA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.OWNER_PARTITION_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PARENT_PARTITION_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PARENT_PARTITION_START_TIME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PARTITION_END_KEY; @@ -133,6 +137,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE; @@ -150,6 +155,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TASK_TS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TASK_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRACKER_STATUS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSFORM_FUNCTION; @@ -376,12 +382,17 @@ enum JoinType { String CDC_PRE_IMAGE = "pre_image"; String CDC_POST_IMAGE = "post_image"; String CDC_CHANGE_IMAGE = "change_image"; + String CDC_IDX_MUTATIONS = "idx_mutations"; String CDC_UPSERT_EVENT_TYPE = "upsert"; String CDC_DELETE_EVENT_TYPE = "delete"; String SPLITS_FILE = "SPLITS_FILE"; String CDC_TTL_DELETE_EVENT_TYPE = "ttl_delete"; String CDC_IMAGE_CQ = "_CDC_IMG_"; byte[] CDC_IMAGE_CQ_BYTES = Bytes.toBytes(CDC_IMAGE_CQ); + String CDC_INDEX_PRE_MUTATIONS_CQ = "_IDX_PRE_"; + byte[] CDC_INDEX_PRE_MUTATIONS_CQ_BYTES = Bytes.toBytes(CDC_INDEX_PRE_MUTATIONS_CQ); + String CDC_INDEX_POST_MUTATIONS_CQ = "_IDX_POST_"; + byte[] CDC_INDEX_POST_MUTATIONS_CQ_BYTES = Bytes.toBytes(CDC_INDEX_POST_MUTATIONS_CQ); /** * We mark counter values 0 to 10 as reserved. Value 0 is used by @@ -408,7 +419,7 @@ enum JoinType { + " BOOLEAN, \n" + SCHEMA_VERSION + " VARCHAR, \n" + EXTERNAL_SCHEMA_ID + " VARCHAR, \n" + STREAMING_TOPIC_NAME + " VARCHAR, \n" + INDEX_WHERE + " VARCHAR, \n" + CDC_INCLUDE_TABLE + " VARCHAR, \n" + TTL + " VARCHAR, \n" + ROW_KEY_MATCHER + " VARBINARY_ENCODED, \n" - + IS_STRICT_TTL + " BOOLEAN, \n" + + + IS_STRICT_TTL + " BOOLEAN, \n" + INDEX_CONSISTENCY + " CHAR(1), \n" + // Column metadata (will be null for table row) DATA_TYPE + " INTEGER," + COLUMN_SIZE + " INTEGER," + DECIMAL_DIGITS + " INTEGER," + NULLABLE + " INTEGER," + ORDINAL_POSITION + " INTEGER," + SORT_ORDER + " INTEGER," + ARRAY_SIZE @@ -586,4 +597,18 @@ enum JoinType { + PARENT_PARTITION_ID + "))\n" + HConstants.VERSIONS + "=%s,\n" + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" + TRANSACTIONAL + "=" + Boolean.FALSE + ",\n" + UPDATE_CACHE_FREQUENCY + "=" + "7200000"; + + String CREATE_IDX_CDC_TRACKER_METADATA = + "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_IDX_CDC_TRACKER_TABLE + "\"(\n" + + // PK columns + TABLE_NAME + " VARCHAR NOT NULL," + PARTITION_ID + " VARCHAR NOT NULL," + OWNER_PARTITION_ID + + " VARCHAR NOT NULL," + + // Non-PK columns + LAST_TIMESTAMP + " BIGINT," + TRACKER_STATUS + " CHAR(1),\n" + "CONSTRAINT " + + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TABLE_NAME + "," + PARTITION_ID + "," + + OWNER_PARTITION_ID + "))\n" + HConstants.VERSIONS + "=%s,\n" + + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" + TRANSACTIONAL + "=" + + Boolean.FALSE + ",\n" + UPDATE_CACHE_FREQUENCY + "=" + "7200000" + ",\n" + + ColumnFamilyDescriptorBuilder.TTL + "=" + IDX_CDC_TRACKER_TTL + ",\n" + + "\"phoenix.max.lookback.age.seconds\"=0"; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java index 44df43c929a..b41da687826 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -28,6 +28,7 @@ import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.schema.transform.TransformMaintainer; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.transaction.TransactionFactory; @@ -283,6 +284,11 @@ public IndexType getIndexType() { return delegate.getIndexType(); } + @Override + public IndexConsistency getIndexConsistency() { + return delegate.getIndexConsistency(); + } + private final PTable delegate; public DelegateTable(PTable delegate) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 93c9cdd39bb..530c5612ac2 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -57,6 +57,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_CONSISTENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE; @@ -276,6 +277,7 @@ import org.apache.phoenix.schema.task.SystemTaskParams; import org.apache.phoenix.schema.task.Task; import org.apache.phoenix.schema.transform.TransformClient; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PBson; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PDate; @@ -344,9 +346,9 @@ public class MetaDataClient { + IMMUTABLE_STORAGE_SCHEME + "," + ENCODING_SCHEME + "," + USE_STATS_FOR_PARALLELIZATION + "," + VIEW_INDEX_ID_DATA_TYPE + "," + CHANGE_DETECTION_ENABLED + "," + PHYSICAL_TABLE_NAME + "," + SCHEMA_VERSION + "," + STREAMING_TOPIC_NAME + "," + INDEX_WHERE + "," + CDC_INCLUDE_TABLE - + "," + TTL + "," + ROW_KEY_MATCHER + "," + IS_STRICT_TTL + + "," + TTL + "," + ROW_KEY_MATCHER + "," + IS_STRICT_TTL + "," + INDEX_CONSISTENCY + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " - + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String CREATE_SCHEMA = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TABLE_SCHEM + "," + TABLE_NAME + ") VALUES (?,?)"; @@ -398,6 +400,10 @@ public class MetaDataClient { + INDEX_STATE + "," + INDEX_DISABLE_TIMESTAMP + "," + ASYNC_REBUILD_TIMESTAMP + " " + PLong.INSTANCE.getSqlTypeName() + ") VALUES (?, ?, ?, ?, ?, ?)"; + public static final String UPDATE_INDEX_CONSISTENCY = + "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TENANT_ID + "," + + TABLE_SCHEM + "," + TABLE_NAME + "," + INDEX_CONSISTENCY + ") VALUES (?, ?, ?, ?)"; + /* * Custom sql to add a column to SYSTEM.CATALOG table during upgrade. We can't use the regular * ColumnMetaDataOps.UPSERT_COLUMN sql because the COLUMN_QUALIFIER column was added in 4.10. And @@ -1062,7 +1068,7 @@ public MutationState createTable(CreateTableStatement statement, byte[][] splits } } table = createTableInternal(statement, splits, parent, viewStatement, viewType, viewIndexIdType, - rowKeyMatcher, viewColumnConstants, isViewColumnReferenced, false, null, null, null, + rowKeyMatcher, viewColumnConstants, isViewColumnReferenced, false, null, null, null, null, tableProps, commonFamilyProps); if (table == null || table.getType() == PTableType.VIEW || statement.isNoVerify() /* @@ -1191,6 +1197,10 @@ private void populatePropertyMaps(ListMultimap> sta continue; } + if (tableType == PTableType.INDEX && "CONSISTENCY".equalsIgnoreCase(prop.getFirst())) { + continue; + } + // HTableDescriptor property or Phoenix Table Property if (defaultDescriptor.getValue(Bytes.toBytes(prop.getFirst())) == null) { // See PHOENIX-4891 @@ -1895,7 +1905,7 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits statement.ifNotExists(), null, statement.getWhere(), statement.getBindCount(), null); table = createTableInternal(tableStatement, splits, dataTable, null, null, getViewIndexDataType(), null, null, null, allocateIndexId, statement.getIndexType(), - asyncCreatedDate, null, tableProps, commonFamilyProps); + statement.getIndexConsistency(), asyncCreatedDate, null, tableProps, commonFamilyProps); } finally { deleteMutexCells(physicalSchemaName, physicalTableName, acquiredColumnMutexSet); } @@ -1905,6 +1915,13 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits if (LOGGER.isInfoEnabled()) LOGGER.info("Created index " + table.getName().getString() + " at " + table.getTimeStamp()); + + if ( + statement.getIndexConsistency() != null && statement.getIndexConsistency().isAsynchronous() + ) { + createCDCForEventuallyConsistentIndex(dataTable); + } + boolean asyncIndexBuildEnabled = connection.getQueryServices().getProps().getBoolean(QueryServices.INDEX_ASYNC_BUILD_ENABLED, QueryServicesOptions.DEFAULT_INDEX_ASYNC_BUILD_ENABLED); @@ -2018,7 +2035,7 @@ public MutationState createCDC(CreateCDCStatement statement) throws SQLException columnDefs, FACTORY.primaryKey(null, pkColumnDefs), Collections.emptyList(), PTableType.CDC, statement.isIfNotExists(), null, null, statement.getBindCount(), null); createTableInternal(tableStatement, null, dataTable, null, null, null, null, null, null, false, - null, null, statement.getIncludeScopes(), tableProps, commonFamilyProps); + null, null, null, statement.getIncludeScopes(), tableProps, commonFamilyProps); // for now, only track stream partition metadata for tables, TODO: updatable views if (PTableType.TABLE == dataTable.getType()) { updateStreamPartitionMetadata(dataTableFullName, cdcObjName); @@ -2441,12 +2458,37 @@ private boolean checkIfParentIsTable(PTable view) { return parentName.getString().equals(view.getPhysicalName().getString()); } + /** + * Creates a CDC stream on the data table for eventually consistent indexes if not already + * present. + * @param dataTable The data table PTable object for which to create the CDC stream. + * @throws SQLException If there is an error executing the CREATE CDC statement or accessing the + * database connection. + */ + private void createCDCForEventuallyConsistentIndex(PTable dataTable) throws SQLException { + String dataTableName = dataTable.getTableName().getString(); + String schemaName = dataTable.getSchemaName().getString(); + String cdcName = "CDC_" + dataTableName; + String fullTableName = SchemaUtil.getCaseSensitiveColumnDisplayName(schemaName, dataTableName); + String cdcSql = "CREATE CDC IF NOT EXISTS \"" + cdcName + "\" ON " + fullTableName; + try { + connection.createStatement().execute(cdcSql); + } catch (SQLException e) { + if (SQLExceptionCode.CDC_ALREADY_ENABLED.getErrorCode() != e.getErrorCode()) { + LOGGER.error("Error while creating CDC for {}", fullTableName, e); + throw e; + } + LOGGER.debug("CDC already exists for {}", fullTableName, e); + } + } + private PTable createTableInternal(CreateTableStatement statement, byte[][] splits, final PTable parent, String viewStatement, ViewType viewType, PDataType viewIndexIdType, final byte[] rowKeyMatcher, final byte[][] viewColumnConstants, final BitSet isViewColumnReferenced, boolean allocateIndexId, IndexType indexType, - Date asyncCreatedDate, Set cdcIncludeScopes, - Map tableProps, Map commonFamilyProps) throws SQLException { + IndexConsistency indexConsistency, Date asyncCreatedDate, + Set cdcIncludeScopes, Map tableProps, + Map commonFamilyProps) throws SQLException { final PTableType tableType = statement.getTableType(); boolean wasAutoCommit = connection.getAutoCommit(); TableName tableNameNode = null; @@ -3396,9 +3438,10 @@ else if ( .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP).setIndexDisableTimestamp(0L) .setSequenceNumber(PTable.INITIAL_SEQ_NUM).setImmutableRows(isImmutableRows) .setDisableWAL(Boolean.TRUE.equals(disableWAL)).setMultiTenant(false).setStoreNulls(false) - .setViewIndexIdType(viewIndexIdType).setIndexType(indexType).setUpdateCacheFrequency(0) - .setNamespaceMapped(isNamespaceMapped).setAutoPartitionSeqName(autoPartitionSeq) - .setAppendOnlySchema(isAppendOnlySchema).setImmutableStorageScheme(ONE_CELL_PER_COLUMN) + .setViewIndexIdType(viewIndexIdType).setIndexType(indexType).setIndexConsistency(null) + .setUpdateCacheFrequency(0).setNamespaceMapped(isNamespaceMapped) + .setAutoPartitionSeqName(autoPartitionSeq).setAppendOnlySchema(isAppendOnlySchema) + .setImmutableStorageScheme(ONE_CELL_PER_COLUMN) .setQualifierEncodingScheme(NON_ENCODED_QUALIFIERS) .setBaseColumnCount(QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT) .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER) @@ -3703,6 +3746,16 @@ public boolean isViewReferenced() { tableUpsert.setBoolean(38, isStrictTTL); + if (tableType == PTableType.INDEX) { + if (indexConsistency == null) { + tableUpsert.setNull(39, Types.CHAR); + } else { + tableUpsert.setString(39, indexConsistency.getCodeAsString()); + } + } else { + tableUpsert.setNull(39, Types.CHAR); + } + tableUpsert.execute(); if (asyncCreatedDate != null) { @@ -3806,6 +3859,7 @@ public boolean isViewReferenced() { .setDisableWAL(Boolean.TRUE.equals(disableWAL)).setMultiTenant(multiTenant) .setStoreNulls(storeNulls).setViewType(viewType).setViewIndexIdType(viewIndexIdType) .setViewIndexId(result.getViewIndexId()).setIndexType(indexType) + .setIndexConsistency(tableType == PTableType.INDEX ? indexConsistency : null) .setTransactionProvider(transactionProvider).setUpdateCacheFrequency(updateCacheFrequency) .setNamespaceMapped(isNamespaceMapped).setAutoPartitionSeqName(autoPartitionSeq) .setAppendOnlySchema(isAppendOnlySchema).setImmutableStorageScheme(immutableStorageScheme) @@ -5791,6 +5845,37 @@ public MutationState alterIndex(AlterIndexStatement statement) throws SQLExcepti metaPropertiesEvaluated, table, schemaName, tableName, new MutableBoolean(false)); PIndexState newIndexState = statement.getIndexState(); + IndexConsistency newIndexConsistency = statement.getIndexConsistency(); + + if (newIndexConsistency != null) { + try (PreparedStatement consistencyUpsert = + connection.prepareStatement(UPDATE_INDEX_CONSISTENCY)) { + consistencyUpsert.setString(1, tenantId); + consistencyUpsert.setString(2, schemaName); + consistencyUpsert.setString(3, indexName); + consistencyUpsert.setString(4, newIndexConsistency.getCodeAsString()); + consistencyUpsert.execute(); + connection.commit(); + + if (newIndexConsistency == IndexConsistency.EVENTUAL) { + PTable dataTable = + connection.getTable(tenantId, SchemaUtil.getTableName(schemaName, dataTableName)); + createCDCForEventuallyConsistentIndex(dataTable); + } + + connection.getQueryServices().clearTableFromCache( + connection.getTenantId() == null + ? ByteUtil.EMPTY_BYTE_ARRAY + : connection.getTenantId().getBytes(), + schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName), + Bytes.toBytes(indexName), HConstants.LATEST_TIMESTAMP); + connection.removeTable(connection.getTenantId(), + SchemaUtil.getTableName(schemaName, indexName), + SchemaUtil.getTableName(schemaName, dataTableName), HConstants.LATEST_TIMESTAMP); + + return new MutationState(1, 1000, connection); + } + } if (isAsync && newIndexState != PIndexState.REBUILD) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.ASYNC_NOT_ALLOWED) @@ -5804,13 +5889,8 @@ public MutationState alterIndex(AlterIndexStatement statement) throws SQLExcepti connection.setAutoCommit(false); // Confirm index table is valid and up-to-date TableRef indexRef = FromCompiler.getResolver(statement, connection).getTables().get(0); - PreparedStatement tableUpsert = null; - try { - if (newIndexState == PIndexState.ACTIVE) { - tableUpsert = connection.prepareStatement(UPDATE_INDEX_STATE_TO_ACTIVE); - } else { - tableUpsert = connection.prepareStatement(UPDATE_INDEX_STATE); - } + try (PreparedStatement tableUpsert = connection.prepareStatement( + newIndexState == PIndexState.ACTIVE ? UPDATE_INDEX_STATE_TO_ACTIVE : UPDATE_INDEX_STATE)) { tableUpsert.setString(1, connection.getTenantId() == null ? null : connection.getTenantId().getString()); tableUpsert.setString(2, schemaName); @@ -5821,10 +5901,6 @@ public MutationState alterIndex(AlterIndexStatement statement) throws SQLExcepti tableUpsert.setLong(6, 0); } tableUpsert.execute(); - } finally { - if (tableUpsert != null) { - tableUpsert.close(); - } } Long timeStamp = indexRef.getTable().isTransactional() ? indexRef.getTimeStamp() : null; List tableMetadata = @@ -5913,8 +5989,8 @@ public MutationState alterIndex(AlterIndexStatement statement) throws SQLExcepti } } } else { - try { - tableUpsert = connection.prepareStatement(UPDATE_INDEX_REBUILD_ASYNC_STATE); + try (PreparedStatement tableUpsert = + connection.prepareStatement(UPDATE_INDEX_REBUILD_ASYNC_STATE)) { tableUpsert.setString(1, connection.getTenantId() == null ? null : connection.getTenantId().getString()); tableUpsert.setString(2, schemaName); @@ -5923,10 +5999,6 @@ public MutationState alterIndex(AlterIndexStatement statement) throws SQLExcepti tableUpsert.setLong(4, beginTimestamp); tableUpsert.execute(); connection.commit(); - } finally { - if (tableUpsert != null) { - tableUpsert.close(); - } } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java index 36a9ab739f8..effd7773ec2 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java @@ -39,6 +39,7 @@ import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.schema.transform.TransformMaintainer; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PArrayDataType; import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PArrayDataTypeEncoder; @@ -931,6 +932,8 @@ IndexMaintainer getIndexMaintainer(PTable dataTable, PTable cdcTable, IndexType getIndexType(); + IndexConsistency getIndexConsistency(); + int getBaseColumnCount(); /** @@ -1154,5 +1157,15 @@ enum CDCChangeScope { * Include only the post image (state past the change) of the row. */ POST, + + /** + * Include index mutations for eventually consistent indexes. + */ + IDX_MUTATIONS, + + /** + * Include raw before/after data row states as serialized Puts for index mutation generation. + */ + DATA_ROW_STATE, } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 027d0317054..dee96ceafc8 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -103,6 +103,7 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.transform.TransformMaintainer; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PBinary; import org.apache.phoenix.schema.types.PChar; import org.apache.phoenix.schema.types.PDataType; @@ -193,6 +194,7 @@ public class PTableImpl implements PTable { private final Long viewIndexId; private final int estimatedSize; private final IndexType indexType; + private final IndexConsistency indexConsistency; private final int baseColumnCount; private final boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been // upgrade for PHOENIX-2067 @@ -267,6 +269,7 @@ public static class Builder { private Long viewIndexId; private int estimatedSize; private IndexType indexType; + private IndexConsistency indexConsistency; private int baseColumnCount; private boolean rowKeyOrderOptimizable; private boolean hasColumnsRequiringUpgrade; @@ -534,6 +537,11 @@ public Builder setIndexType(IndexType indexType) { return this; } + public Builder setIndexConsistency(IndexConsistency indexConsistency) { + this.indexConsistency = indexConsistency; + return this; + } + public Builder setBaseColumnCount(int baseColumnCount) { this.baseColumnCount = baseColumnCount; return this; @@ -994,6 +1002,7 @@ private PTableImpl(Builder builder) { this.viewIndexId = builder.viewIndexId; this.estimatedSize = builder.estimatedSize; this.indexType = builder.indexType; + this.indexConsistency = builder.indexConsistency; this.baseColumnCount = builder.baseColumnCount; this.rowKeyOrderOptimizable = builder.rowKeyOrderOptimizable; this.hasColumnsRequiringUpgrade = builder.hasColumnsRequiringUpgrade; @@ -1052,6 +1061,7 @@ public static PTableImpl.Builder builderFromExisting(PTable table) { .setMultiTenant(table.isMultiTenant()).setStoreNulls(table.getStoreNulls()) .setViewType(table.getViewType()).setViewIndexIdType(table.getviewIndexIdType()) .setViewIndexId(table.getViewIndexId()).setIndexType(table.getIndexType()) + .setIndexConsistency(table.getIndexConsistency()) .setTransactionProvider(table.getTransactionProvider()) .setUpdateCacheFrequency(table.getUpdateCacheFrequency()) .setNamespaceMapped(table.isNamespaceMapped()) @@ -1890,6 +1900,11 @@ public IndexType getIndexType() { return indexType; } + @Override + public IndexConsistency getIndexConsistency() { + return type == PTableType.INDEX ? indexConsistency : null; + } + /** * Construct a PTable instance from ProtoBuffered PTable instance */ @@ -2096,6 +2111,14 @@ public static PTable createFromProto(PTableProtos.PTable table) { if (table.hasRowKeyMatcher()) { rowKeyMatcher = table.getRowKeyMatcher().toByteArray(); } + IndexConsistency indexConsistency = null; + if (tableType == PTableType.INDEX) { + if (table.hasIndexConsistency()) { + indexConsistency = IndexConsistency.valueOf(table.getIndexConsistency()); + } else { + indexConsistency = null; + } + } try { return new PTableImpl.Builder().setType(tableType).setState(indexState) @@ -2103,7 +2126,7 @@ public static PTable createFromProto(PTableProtos.PTable table) { .setSequenceNumber(sequenceNumber).setImmutableRows(isImmutableRows) .setViewStatement(viewStatement).setDisableWAL(disableWAL).setMultiTenant(multiTenant) .setStoreNulls(storeNulls).setViewType(viewType).setViewIndexIdType(viewIndexIdType) - .setViewIndexId(viewIndexId).setIndexType(indexType) + .setViewIndexId(viewIndexId).setIndexType(indexType).setIndexConsistency(indexConsistency) .setTransactionProvider(transactionProvider).setUpdateCacheFrequency(updateCacheFrequency) .setNamespaceMapped(isNamespaceMapped).setAutoPartitionSeqName(autoPartitionSeqName) .setAppendOnlySchema(isAppendOnlySchema) @@ -2166,6 +2189,9 @@ public static PTableProtos.PTable toProto(PTable table, long clientVersion) { builder.setIndexType( ByteStringer.wrap(new byte[] { table.getIndexType().getSerializedValue() })); } + if (table.getIndexConsistency() != null) { + builder.setIndexConsistency(table.getIndexConsistency().name()); + } } builder.setSequenceNumber(table.getSequenceNumber()); builder.setTimeStamp(table.getTimeStamp()); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaSQLUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaSQLUtil.java index 6fc4c1c98a1..d6b513ad26e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaSQLUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaSQLUtil.java @@ -26,6 +26,7 @@ import org.apache.phoenix.parse.CreateTableStatement; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.thirdparty.com.google.common.collect.ListMultimap; @@ -74,6 +75,12 @@ protected static String getCreateIndexSQL(CreateIndexStatement createStmt) { if (createStmt.isAsync()) { sb.append(" ASYNC"); } + if ( + createStmt.getIndexConsistency() != null + && createStmt.getIndexConsistency() != IndexConsistency.getDefault() + ) { + sb.append(" CONSISTENCY=").append(createStmt.getIndexConsistency().name()); + } appendProperties(sb, createStmt.getProps()); return sb.toString(); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/transform/TransformClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/transform/TransformClient.java index ebcd9f034ca..f37c414f5b7 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/transform/TransformClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/transform/TransformClient.java @@ -258,9 +258,10 @@ protected static PTable addTransform(PTable table, MetaDataClient.MetaProperties .setEstimatedSize(table.getEstimatedSize()).setFamilies(table.getColumnFamilies()) .setImmutableRows(table.isImmutableRows()) .setIsChangeDetectionEnabled(table.isChangeDetectionEnabled()) - .setIndexType(table.getIndexType()).setIndexes(Collections. emptyList()) - .setName(newTableName).setMultiTenant(table.isMultiTenant()) - .setParentName(table.getParentName()).setParentSchemaName(table.getParentSchemaName()) + .setIndexType(table.getIndexType()).setIndexConsistency(table.getIndexConsistency()) + .setIndexes(Collections.emptyList()).setName(newTableName) + .setMultiTenant(table.isMultiTenant()).setParentName(table.getParentName()) + .setParentSchemaName(table.getParentSchemaName()) .setNamespaceMapped(table.isNamespaceMapped()).setSchemaName(table.getSchemaName()) .setPkColumns(table.getPKColumns()).setPkName(table.getPKName()) .setRowKeySchema(table.getRowKeySchema()).setStoreNulls(table.getStoreNulls()) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/types/IndexConsistency.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/types/IndexConsistency.java new file mode 100644 index 00000000000..13b61fab4ef --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/types/IndexConsistency.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema.types; + +/** + * Enum representing the consistency level for Phoenix secondary indexes. + *

+ * STRONG: Synchronous index maintenance where index mutations are applied synchronously with + * data table mutations. This provides immediate consistency but may impact write performance and + * availability. + *

+ *

+ * EVENTUAL: Asynchronous index maintenance where index mutations are applied in the + * background. This provides better write performance and availability but with eventual consistency + * guarantees. + *

+ */ +public enum IndexConsistency { + /** + * Strong consistency - synchronous index maintenance (default behavior). Index updates are + * applied synchronously with data table updates. + */ + STRONG('s'), + + /** + * Eventual consistency - asynchronous index maintenance using CDC. Index updates are applied in + * the background, providing eventual consistency. + */ + EVENTUAL('e'); + + private final char code; + + IndexConsistency(char code) { + this.code = code; + } + + /** + * Returns the single character code for the consistency. + * @return 's' for STRONG, 'e' for EVENTUAL + */ + public char getCode() { + return code; + } + + /** + * Returns the string representation of the code for storage. + * @return single character string for storage + */ + public String getCodeAsString() { + return String.valueOf(code); + } + + /** + * Parses an IndexConsistency from its stored code. + * @param code the stored code. + * @return the corresponding IndexConsistency, or null if code is null/empty/invalid. + */ + public static IndexConsistency fromCode(String code) { + if (code == null || code.isEmpty()) { + return null; + } + char c = code.charAt(0); + for (IndexConsistency ic : values()) { + if (ic.code == c) { + return ic; + } + } + return null; + } + + /** + * Returns the default consistency level for indexes. + * @return STRONG consistency as the default + */ + public static IndexConsistency getDefault() { + return STRONG; + } + + /** + * Checks if this consistency level requires synchronous index maintenance. + * @return true if synchronous maintenance is required, false otherwise + */ + public boolean isSynchronous() { + return this == STRONG; + } + + /** + * Checks if this consistency level uses asynchronous index maintenance. + * @return true if asynchronous maintenance is used, false otherwise + */ + public boolean isAsynchronous() { + return this == EVENTUAL; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCChangeBuilder.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCChangeBuilder.java index 45a7e957e2d..880a0a327b2 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCChangeBuilder.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCChangeBuilder.java @@ -26,9 +26,12 @@ import static org.apache.phoenix.query.QueryConstants.CDC_UPSERT_EVENT_TYPE; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import org.apache.hadoop.hbase.Cell; +import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.index.CDCTableInfo; import org.apache.phoenix.schema.PTable; @@ -36,6 +39,8 @@ public class CDCChangeBuilder { private final boolean isChangeImageInScope; private final boolean isPreImageInScope; private final boolean isPostImageInScope; + private final boolean isIdxMutationsInScope; + private final boolean isDataRowStateInScope; private final CDCTableInfo cdcDataTableInfo; private String changeType; private long lastDeletedTimestamp; @@ -43,12 +48,20 @@ public class CDCChangeBuilder { private Map preImage = null; private Map changeImage = null; + private boolean isFullRowDelete; + private Map rawLatestBeforeChange; + private Map rawAtChange; + private Set rawDeletedColumnsAtChange; + private Map rawDeletedColumnsBeforeChange; + public CDCChangeBuilder(CDCTableInfo cdcDataTableInfo) { this.cdcDataTableInfo = cdcDataTableInfo; Set changeScopes = cdcDataTableInfo.getIncludeScopes(); isChangeImageInScope = changeScopes.contains(PTable.CDCChangeScope.CHANGE); isPreImageInScope = changeScopes.contains(PTable.CDCChangeScope.PRE); isPostImageInScope = changeScopes.contains(PTable.CDCChangeScope.POST); + isIdxMutationsInScope = changeScopes.contains(PTable.CDCChangeScope.IDX_MUTATIONS); + isDataRowStateInScope = changeScopes.contains(PTable.CDCChangeScope.DATA_ROW_STATE); } public void initChange(long ts) { @@ -61,6 +74,13 @@ public void initChange(long ts) { if (isChangeImageInScope || isPostImageInScope) { changeImage = new HashMap<>(); } + if (isDataRowStateInScope) { + isFullRowDelete = false; + rawLatestBeforeChange = new LinkedHashMap<>(); + rawAtChange = new LinkedHashMap<>(); + rawDeletedColumnsAtChange = new HashSet<>(); + rawDeletedColumnsBeforeChange = new HashMap<>(); + } } public long getChangeTimestamp() { @@ -77,6 +97,9 @@ public boolean isNonEmptyEvent() { public void markAsDeletionEvent() { changeType = CDC_DELETE_EVENT_TYPE; + if (isDataRowStateInScope) { + isFullRowDelete = true; + } } public long getLastDeletedTimestamp() { @@ -141,12 +164,69 @@ public Map buildCDCEvent() { } public boolean isOlderThanChange(Cell cell) { - return (cell.getTimestamp() < changeTimestamp && cell.getTimestamp() > lastDeletedTimestamp) - ? true - : false; + return cell.getTimestamp() < changeTimestamp && cell.getTimestamp() > lastDeletedTimestamp; + } + + public void registerRawPut(Cell cell, ImmutableBytesPtr colKey) { + if (cell.getTimestamp() == changeTimestamp) { + rawAtChange.putIfAbsent(colKey, cell); + } else if (isOlderThanChange(cell)) { + Long colDeleteTs = rawDeletedColumnsBeforeChange.get(colKey); + if ( + (colDeleteTs == null || cell.getTimestamp() > colDeleteTs) + && !rawLatestBeforeChange.containsKey(colKey) + ) { + rawLatestBeforeChange.put(colKey, cell); + } + } + } + + public void registerRawDeleteColumn(Cell cell, ImmutableBytesPtr colKey) { + if (cell.getTimestamp() == changeTimestamp) { + rawDeletedColumnsAtChange.add(colKey); + } else if (isOlderThanChange(cell)) { + rawDeletedColumnsBeforeChange.putIfAbsent(colKey, cell.getTimestamp()); + } + } + + public boolean hasValidDataRowStateChanges() { + return isFullRowDelete || !rawAtChange.isEmpty() || !rawDeletedColumnsAtChange.isEmpty(); + } + + public boolean isFullRowDelete() { + return isFullRowDelete; + } + + public Map getRawLatestBeforeChange() { + return rawLatestBeforeChange; + } + + public Map getRawAtChange() { + return rawAtChange; + } + + public Set getRawDeletedColumnsAtChange() { + return rawDeletedColumnsAtChange; } public boolean isPreImageInScope() { return isPreImageInScope; } + + public boolean isPostImageInScope() { + return isPostImageInScope; + } + + public boolean isChangeImageInScope() { + return isChangeImageInScope; + } + + public boolean isIdxMutationsInScope() { + return isIdxMutationsInScope; + } + + public boolean isDataRowStateInScope() { + return isDataRowStateInScope; + } + } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCUtil.java index 68c9b48aa77..d36c957f7a6 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/CDCUtil.java @@ -36,6 +36,7 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.execute.DescVarLengthFastByteComparisons; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.types.PDataType; @@ -148,6 +149,34 @@ public static boolean hasCDCIndex(PTable table) { return table.getIndexes().stream().anyMatch(CDCUtil::isCDCIndex); } + /** + * Get the CDC object name for the given data table that can be used in SQL queries. + * @param dataTable The data table PTable object. + * @param withSchema If true, returns the CDC object name with schema prefix. If false, returns + * only the CDC object name. + * @return The CDC object name that can be used in SQL queries, optionally prefixed with schema + * name, or null if no active CDC index exists. + */ + public static String getCDCObjectName(PTable dataTable, boolean withSchema) { + if (dataTable == null) { + return null; + } + PTable cdcIndex = getActiveCDCIndex(dataTable); + if (cdcIndex == null) { + return null; + } + String cdcIndexName = cdcIndex.getTableName().getString(); + String schemaName = dataTable.getSchemaName().getString(); + if (isCDCIndex(cdcIndexName)) { + String cdcObj = cdcIndexName.substring(CDC_INDEX_PREFIX.length()); + if (!withSchema || schemaName == null || schemaName.isEmpty()) { + return cdcObj; + } + return schemaName + QueryConstants.NAME_SEPARATOR + cdcObj; + } + return null; + } + public static Scan setupScanForCDC(Scan scan) { scan.setRaw(true); scan.readAllVersions(); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index 5396fc70f8c..9249a8ad68f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -126,7 +126,8 @@ public class MetaDataUtil { public static final List SYSTEM_TABLES_WITH_TTL_SUPPORTED = Arrays.asList(SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_CDC_STREAM_TABLE), - SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_CDC_STREAM_STATUS_TABLE)); + SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_CDC_STREAM_STATUS_TABLE), + SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_IDX_CDC_TRACKER_TABLE)); public static Put getLastDDLTimestampUpdate(byte[] tableHeaderRowKey, long clientTimestamp, long lastDDLTimestamp) { diff --git a/phoenix-core-client/src/main/protobuf/IndexMutations.proto b/phoenix-core-client/src/main/protobuf/IndexMutations.proto new file mode 100644 index 00000000000..cba4360dd76 --- /dev/null +++ b/phoenix-core-client/src/main/protobuf/IndexMutations.proto @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.phoenix.coprocessor.generated"; +option java_outer_classname = "IndexMutationsProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +// The batch of index mutations for eventually consistent indexes. +// Each entry pairs a target index table name (tables[i]) with its +// corresponding serialized HBase MutationProto (mutations[i]). +message IndexMutations { + repeated bytes tables = 1; + repeated bytes mutations = 2; +} + +// Raw data row states for generating index mutations. +// Contains the data row key and serialized MutationProto for the before and after states. +message DataRowStates { + optional bytes dataRowKey = 1; + optional bytes currentDataRowState = 2; + optional bytes nextDataRowState = 3; +} diff --git a/phoenix-core-client/src/main/protobuf/PTable.proto b/phoenix-core-client/src/main/protobuf/PTable.proto index 43f53151147..a98239f9ed2 100644 --- a/phoenix-core-client/src/main/protobuf/PTable.proto +++ b/phoenix-core-client/src/main/protobuf/PTable.proto @@ -124,6 +124,7 @@ message PTable { optional bytes ttl = 55; optional bytes rowKeyMatcher = 56; optional bool isStrictTTL = 57; + optional string indexConsistency = 58; } message EncodedCQCounter { diff --git a/phoenix-core-client/src/main/protobuf/ServerCachingService.proto b/phoenix-core-client/src/main/protobuf/ServerCachingService.proto index d8e19455273..2c4f358cccd 100644 --- a/phoenix-core-client/src/main/protobuf/ServerCachingService.proto +++ b/phoenix-core-client/src/main/protobuf/ServerCachingService.proto @@ -73,6 +73,7 @@ message IndexMaintainer { repeated ColumnReference indexWhereColumns = 30; optional bool isCDCIndex = 31; optional int32 dataTableSaltBuckets = 32; + optional string indexConsistency = 33; } message TransformMaintainer { diff --git a/phoenix-core-server/pom.xml b/phoenix-core-server/pom.xml index 9f786ab111f..541557b9661 100644 --- a/phoenix-core-server/pom.xml +++ b/phoenix-core-server/pom.xml @@ -173,6 +173,10 @@ com.google.code.findbugs jsr305 + + org.xerial.snappy + snappy-java + diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CDCGlobalIndexRegionScanner.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CDCGlobalIndexRegionScanner.java index 5bb0c402172..97b388dc43e 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CDCGlobalIndexRegionScanner.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CDCGlobalIndexRegionScanner.java @@ -21,18 +21,22 @@ import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.protobuf.ByteString; import java.io.IOException; import java.util.Arrays; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilder; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; @@ -41,6 +45,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.generated.CDCInfoProtos; +import org.apache.phoenix.coprocessor.generated.IndexMutationsProtos; import org.apache.phoenix.execute.TupleProjector; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.SingleCellColumnExpression; @@ -50,6 +55,7 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.CDCChangeBuilder; import org.apache.phoenix.util.CDCUtil; import org.apache.phoenix.util.EncodedColumnsUtil; @@ -57,6 +63,10 @@ import org.apache.phoenix.util.JacksonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xerial.snappy.Snappy; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; /** * CDC (Change Data Capture) enabled region scanner for global indexes that processes uncovered CDC @@ -86,6 +96,10 @@ public class CDCGlobalIndexRegionScanner extends UncoveredGlobalIndexRegionScann private static final Logger LOGGER = LoggerFactory.getLogger(CDCGlobalIndexRegionScanner.class); private CDCTableInfo cdcDataTableInfo; private CDCChangeBuilder changeBuilder; + private static final byte[] SEPARATOR = { 0 }; + + private static final byte[] EMPTY_IDX_MUTATIONS = PVarchar.INSTANCE.toBytes(Base64.getEncoder() + .encodeToString(IndexMutationsProtos.IndexMutations.getDefaultInstance().toByteArray())); public CDCGlobalIndexRegionScanner(final RegionScanner innerScanner, final Region region, final Scan scan, final RegionCoprocessorEnvironment env, final Scan dataTableScan, @@ -102,6 +116,13 @@ public CDCGlobalIndexRegionScanner(final RegionScanner innerScanner, final Regio @Override protected Scan prepareDataTableScan(Collection dataRowKeys) throws IOException { + if ( + changeBuilder.isIdxMutationsInScope() && !changeBuilder.isDataRowStateInScope() + && !changeBuilder.isChangeImageInScope() && !changeBuilder.isPreImageInScope() + && !changeBuilder.isPostImageInScope() + ) { + return null; + } // TODO: Get Timerange from the start row and end row of the index scan object // and set it in the datatable scan object. // if (scan.getStartRow().length == 8) { @@ -112,7 +133,11 @@ protected Scan prepareDataTableScan(Collection dataRowKeys) throws IOExc // stopTimeRange = PLong.INSTANCE.getCodec().decodeLong( // scan.getStopRow(), 0, SortOrder.getDefault()); // } - return CDCUtil.setupScanForCDC(prepareDataTableScan(dataRowKeys, true)); + Scan dataScan = prepareDataTableScan(dataRowKeys, true); + if (dataScan == null) { + return null; + } + return CDCUtil.setupScanForCDC(dataScan); } protected boolean getNextCoveredIndexRow(List result) throws IOException { @@ -120,9 +145,11 @@ protected boolean getNextCoveredIndexRow(List result) throws IOException { List indexRow = indexRowIterator.next(); Cell indexCell = indexRow.get(0); byte[] indexRowKey = ImmutableBytesPtr.cloneCellRowIfNecessary(indexCell); + if (handleIdxMutationsCDCEvent(indexRow, indexRowKey, indexCell, result)) { + return true; + } if (indexRow.size() > 1) { - boolean success = handlePreImageCDCEvent(indexRow, indexRowKey, indexCell, result); - if (success) { + if (handlePreImageCDCEvent(indexRow, indexRowKey, indexCell, result)) { return true; } } @@ -158,59 +185,74 @@ protected boolean getNextCoveredIndexRow(List result) throws IOException { // marker after that. changeBuilder.setLastDeletedTimestamp(cell.getTimestamp()); } - } else if ( - (cell.getType() == Cell.Type.DeleteColumn || cell.getType() == Cell.Type.Put) - && !Arrays.equals(cellQual, emptyCQ) - ) { - if (!changeBuilder.isChangeRelevant(cell)) { - // We don't need to build the change image, just skip it. - continue; - } - // In this case, cell is the row, meaning we loop over rows.. - if (isSingleCell) { - while (curColumnNum < cdcColumnInfoList.size()) { - boolean hasValue = dataTableProjector.getSchema().extractValue(cell, - (SingleCellColumnExpression) expressions[curColumnNum], ptr); - if (hasValue) { - Object cellValue = getColumnValue(ptr.get(), ptr.getOffset(), ptr.getLength(), - cdcColumnInfoList.get(curColumnNum).getColumnType()); - changeBuilder.registerChange(cell, curColumnNum, cellValue); + } else + if (cell.getType() == Cell.Type.DeleteColumn || cell.getType() == Cell.Type.Put) { + boolean isEmptyCQ = Arrays.equals(cellQual, emptyCQ); + if (changeBuilder.isDataRowStateInScope()) { + ImmutableBytesPtr colKey = + new ImmutableBytesPtr(Bytes.add(cellFam, SEPARATOR, cellQual)); + if (cell.getType() == Cell.Type.DeleteColumn) { + changeBuilder.registerRawDeleteColumn(cell, colKey); + } else { + changeBuilder.registerRawPut(cell, colKey); } - ++curColumnNum; } - break cellLoop; - } - while (true) { - CDCTableInfo.CDCColumnInfo currentColumnInfo = cdcColumnInfoList.get(curColumnNum); - int columnComparisonResult = - CDCUtil.compareCellFamilyAndQualifier(cellFam, cellQual, - currentColumnInfo.getColumnFamily(), currentColumnInfo.getColumnQualifier()); - if (columnComparisonResult > 0) { - if (++curColumnNum >= cdcColumnInfoList.size()) { - // Have no more column definitions, so the rest of the cells - // must be for dropped columns and so can be ignored. - break cellLoop; - } - // Continue looking for the right column definition - // for this cell. + if ( + isEmptyCQ || changeBuilder.isDataRowStateInScope() + && !changeBuilder.isChangeImageInScope() && !changeBuilder.isPreImageInScope() + && !changeBuilder.isPostImageInScope() + ) { continue; - } else if (columnComparisonResult < 0) { - // We didn't find a column definition for this cell, ignore the - // current cell but continue working on the rest of the cells. - continue cellLoop; } + // In this case, cell is the row, meaning we loop over rows.. + if (isSingleCell) { + while (curColumnNum < cdcColumnInfoList.size()) { + boolean hasValue = dataTableProjector.getSchema().extractValue(cell, + (SingleCellColumnExpression) expressions[curColumnNum], ptr); + if (hasValue) { + Object cellValue = getColumnValue(ptr.get(), ptr.getOffset(), ptr.getLength(), + cdcColumnInfoList.get(curColumnNum).getColumnType()); + changeBuilder.registerChange(cell, curColumnNum, cellValue); + } + ++curColumnNum; + } + break cellLoop; + } + while (true) { + CDCTableInfo.CDCColumnInfo currentColumnInfo = + cdcColumnInfoList.get(curColumnNum); + int columnComparisonResult = + CDCUtil.compareCellFamilyAndQualifier(cellFam, cellQual, + currentColumnInfo.getColumnFamily(), currentColumnInfo.getColumnQualifier()); + if (columnComparisonResult > 0) { + if (++curColumnNum >= cdcColumnInfoList.size()) { + // Have no more column definitions, so the rest of the cells + // must be for dropped columns and so can be ignored. + break cellLoop; + } + // Continue looking for the right column definition + // for this cell. + continue; + } else if (columnComparisonResult < 0) { + // We didn't find a column definition for this cell, ignore the + // current cell but continue working on the rest of the cells. + continue cellLoop; + } - // else, found the column definition. - Object cellValue = cell.getType() == Cell.Type.DeleteColumn - ? null - : getColumnValue(cell, cdcColumnInfoList.get(curColumnNum).getColumnType()); - changeBuilder.registerChange(cell, curColumnNum, cellValue); - // Done processing the current cell, check the next cell. - break; + // else, found the column definition. + Object cellValue = cell.getType() == Cell.Type.DeleteColumn + ? null + : getColumnValue(cell, cdcColumnInfoList.get(curColumnNum).getColumnType()); + changeBuilder.registerChange(cell, curColumnNum, cellValue); + // Done processing the current cell, check the next cell. + break; + } } - } } - if (changeBuilder.isNonEmptyEvent()) { + if (changeBuilder.isDataRowStateInScope()) { + buildDataRowStateResult(dataRowKey.copyBytesIfNecessary(), indexRowKey, indexCell, + result); + } else if (changeBuilder.isNonEmptyEvent()) { Result cdcRow = getCDCImage(indexRowKey, indexCell); if (cdcRow != null && tupleProjector != null) { if (indexCell.getType() == Cell.Type.DeleteFamily) { @@ -235,7 +277,12 @@ protected boolean getNextCoveredIndexRow(List result) throws IOException { result.clear(); } } else { - result.clear(); + if (changeBuilder.isDataRowStateInScope()) { + buildDataRowStateResult(dataRowKey.copyBytesIfNecessary(), indexRowKey, indexCell, + result); + } else { + result.clear(); + } } return true; @@ -317,18 +364,151 @@ private boolean handlePreImageCDCEvent(List indexRow, byte[] indexRowKey, cdcJson.remove(QueryConstants.CDC_PRE_IMAGE); cdcEventBytes = JacksonUtil.getObjectWriter(HashMap.class).writeValueAsBytes(cdcJson); } + addResult(indexRowKey, indexCell, result, cdcDataCell, cdcEventBytes); + return true; + } + + /** + * Builds a DataRowStates protobuf result from the raw cell maps collected by CDCChangeBuilder + * during the raw cell iteration. Constructs before/after HBase Put objects representing the row + * state before and after the change, serializes them, and populates the result. If no valid + * changes were found at the change timestamp (data not yet visible), only the dataRowKey is + * included, the consumer needs to retry. + * @param dataRowKey The data table row key bytes. + * @param indexRowKey The CDC index row key. + * @param indexCell The index cell. + * @param result The result list to populate. + * @throws IOException if serialization fails. + */ + private void buildDataRowStateResult(byte[] dataRowKey, byte[] indexRowKey, Cell indexCell, + List result) throws IOException { + Put currentDataRowState = null; + Put nextDataRowState = null; + Map latestBeforeChange = changeBuilder.getRawLatestBeforeChange(); + Map atChange = changeBuilder.getRawAtChange(); + Set deletedColumnsAtChange = changeBuilder.getRawDeletedColumnsAtChange(); + if (changeBuilder.hasValidDataRowStateChanges()) { + if (!latestBeforeChange.isEmpty()) { + currentDataRowState = new Put(dataRowKey); + for (Cell cell : latestBeforeChange.values()) { + currentDataRowState.add(cell); + } + } + if (!changeBuilder.isFullRowDelete()) { + Put nextState = new Put(dataRowKey); + for (Map.Entry entry : latestBeforeChange.entrySet()) { + if ( + !atChange.containsKey(entry.getKey()) + && !deletedColumnsAtChange.contains(entry.getKey()) + ) { + nextState.add(entry.getValue()); + } + } + for (Cell cell : atChange.values()) { + nextState.add(cell); + } + if (!nextState.isEmpty()) { + nextDataRowState = nextState; + } + } + } + IndexMutationsProtos.DataRowStates.Builder builder = + IndexMutationsProtos.DataRowStates.newBuilder(); + builder.setDataRowKey(ByteString.copyFrom(dataRowKey)); + if (currentDataRowState != null) { + builder.setCurrentDataRowState(ByteString.copyFrom( + ProtobufUtil.toMutation(ClientProtos.MutationProto.MutationType.PUT, currentDataRowState) + .toByteArray())); + } + if (nextDataRowState != null) { + builder.setNextDataRowState(ByteString.copyFrom(ProtobufUtil + .toMutation(ClientProtos.MutationProto.MutationType.PUT, nextDataRowState).toByteArray())); + } + String base64String = Base64.getEncoder().encodeToString(builder.build().toByteArray()); + byte[] cdcEventBytes = PVarchar.INSTANCE.toBytes(base64String); + addResult(indexRowKey, indexCell, result, indexCell, cdcEventBytes); + } + + /** + * Handles CDC event when IDX_MUTATIONS scope is enabled. Returns the index mutations as a + * serialized IndexMutations, or an empty proto if no mutations are present. Skips the data table + * scan. + * @param indexRow The CDC index row. + * @param indexRowKey The CDC index row key. + * @param indexCell The primary index cell. + * @param result The result list to populate. + * @return true if IDX_MUTATIONS scope is enabled, false otherwise. + * @throws IOException if decompression or proto parsing fails. + */ + private boolean handleIdxMutationsCDCEvent(List indexRow, byte[] indexRowKey, + Cell indexCell, List result) throws IOException { + if (!changeBuilder.isIdxMutationsInScope()) { + return false; + } + byte[] idxMutationsBytes = EMPTY_IDX_MUTATIONS; + Cell idxMutationsCell = indexCell; + IndexMutationsProtos.IndexMutations preMutations = null; + IndexMutationsProtos.IndexMutations postMutations = null; + Cell preCell = null; + for (Cell cell : indexRow) { + if ( + Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), + QueryConstants.CDC_INDEX_PRE_MUTATIONS_CQ_BYTES, 0, + QueryConstants.CDC_INDEX_PRE_MUTATIONS_CQ_BYTES.length) + ) { + byte[] rawBytes = CellUtil.cloneValue(cell); + preMutations = IndexMutationsProtos.IndexMutations.parseFrom(maybeDecompress(rawBytes)); + preCell = cell; + } else if ( + Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), + QueryConstants.CDC_INDEX_POST_MUTATIONS_CQ_BYTES, 0, + QueryConstants.CDC_INDEX_POST_MUTATIONS_CQ_BYTES.length) + ) { + byte[] rawBytes = CellUtil.cloneValue(cell); + postMutations = IndexMutationsProtos.IndexMutations.parseFrom(maybeDecompress(rawBytes)); + } + } + if (preMutations != null) { + IndexMutationsProtos.IndexMutations merged = mergeIndexMutations(preMutations, postMutations); + String base64String = Base64.getEncoder().encodeToString(merged.toByteArray()); + idxMutationsBytes = PVarchar.INSTANCE.toBytes(base64String); + idxMutationsCell = preCell; + } + addResult(indexRowKey, indexCell, result, idxMutationsCell, idxMutationsBytes); + return true; + } + + private static byte[] maybeDecompress(byte[] input) { + try { + if (Snappy.isValidCompressedBuffer(input)) { + return Snappy.uncompress(input); + } + } catch (IOException e) { + LOGGER.error("Error uncompressing CDC Index mutations", e); + } + return input; + } + + private IndexMutationsProtos.IndexMutations mergeIndexMutations( + IndexMutationsProtos.IndexMutations pre, IndexMutationsProtos.IndexMutations post) { + if (post == null) { + return pre; + } + return IndexMutationsProtos.IndexMutations.newBuilder().addAllTables(pre.getTablesList()) + .addAllMutations(pre.getMutationsList()).addAllTables(post.getTablesList()) + .addAllMutations(post.getMutationsList()).build(); + } + + private void addResult(byte[] indexRowKey, Cell indexCell, List result, Cell cdcDataCell, + byte[] cdcEventBytes) { Result cdcRow = createCDCResult(indexRowKey, indexCell, cdcDataCell.getTimestamp(), cdcEventBytes); - if (tupleProjector != null) { result.add(indexCell); IndexUtil.addTupleAsOneCell(result, new ResultTuple(cdcRow), tupleProjector, ptr); } else { result.clear(); } - LOGGER.debug( - "Processed CDC event with embedded data, skipped data table scan for" + " row key: {}", - Bytes.toStringBinary(indexRowKey)); - return true; } + } diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 9d3d0b3ca47..053976b586d 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -39,6 +39,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXTERNAL_SCHEMA_ID_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_CONSISTENCY_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE_BYTES; @@ -263,8 +264,10 @@ import org.apache.phoenix.schema.metrics.MetricsMetadataSourceFactory; import org.apache.phoenix.schema.task.ServerTask; import org.apache.phoenix.schema.task.SystemTaskParams; +import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PBinary; import org.apache.phoenix.schema.types.PBoolean; +import org.apache.phoenix.schema.types.PChar; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PLong; @@ -418,6 +421,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, CDC_INCLUDE_BYTES); private static final Cell INDEX_WHERE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, INDEX_WHERE_BYTES); + private static final Cell INDEX_CONSISTENCY_KV = + createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, INDEX_CONSISTENCY_BYTES); private static final Cell TTL_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TTL_BYTES); @@ -436,7 +441,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr AUTO_PARTITION_SEQ_KV, APPEND_ONLY_SCHEMA_KV, STORAGE_SCHEME_KV, ENCODING_SCHEME_KV, USE_STATS_FOR_PARALLELIZATION_KV, LAST_DDL_TIMESTAMP_KV, CHANGE_DETECTION_ENABLED_KV, SCHEMA_VERSION_KV, EXTERNAL_SCHEMA_ID_KV, STREAMING_TOPIC_NAME_KV, INDEX_WHERE_KV, - CDC_INCLUDE_KV, TTL_KV, ROW_KEY_MATCHER_KV, IS_STRICT_TTL_KV); + CDC_INCLUDE_KV, TTL_KV, ROW_KEY_MATCHER_KV, IS_STRICT_TTL_KV, INDEX_CONSISTENCY_KV); static { Collections.sort(TABLE_KV_COLUMNS, CellComparatorImpl.COMPARATOR); @@ -498,6 +503,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr private static final int TTL_INDEX = TABLE_KV_COLUMNS.indexOf(TTL_KV); private static final int ROW_KEY_MATCHER_INDEX = TABLE_KV_COLUMNS.indexOf(ROW_KEY_MATCHER_KV); private static final int IS_STRICT_TTL_INDEX = TABLE_KV_COLUMNS.indexOf(IS_STRICT_TTL_KV); + private static final int INDEX_CONSISTENCY_INDEX = TABLE_KV_COLUMNS.indexOf(INDEX_CONSISTENCY_KV); // KeyValues for Column private static final KeyValue DECIMAL_DIGITS_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, DECIMAL_DIGITS_BYTES); @@ -1586,6 +1592,17 @@ private PTable getTableFromCells(List tableCellList, List> allC builder.setIsStrictTTL( isStrictTTLKv != null ? isStrictTTL : oldTable == null || oldTable.isStrictTTL()); + Cell indexConsistencyKv = tableKeyValues[INDEX_CONSISTENCY_INDEX]; + IndexConsistency indexConsistency = null; + if (indexConsistencyKv != null) { + String consistencyStr = (String) PChar.INSTANCE.toObject(indexConsistencyKv.getValueArray(), + indexConsistencyKv.getValueOffset(), indexConsistencyKv.getValueLength()); + indexConsistency = IndexConsistency.fromCode(consistencyStr); + } + builder.setIndexConsistency(indexConsistency != null ? indexConsistency + : oldTable != null ? oldTable.getIndexConsistency() + : null); + // Check the cell tag to see whether the view has modified this property final byte[] tagUseStatsForParallelization = (useStatsForParallelizationKv == null) ? HConstants.EMPTY_BYTE_ARRAY diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UncoveredIndexRegionScanner.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UncoveredIndexRegionScanner.java index 640b85bc5ac..5d657541091 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UncoveredIndexRegionScanner.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UncoveredIndexRegionScanner.java @@ -63,6 +63,7 @@ import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.IndexUtil; +import org.apache.phoenix.util.PhoenixKeyValueUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.ServerUtil; import org.slf4j.Logger; @@ -280,7 +281,21 @@ protected boolean scanIndexTableRows(List result, final byte[] actualStart indexToDataRowKeyMap.put(offset == 0 ? lastIndexRowKey : CellUtil.cloneRow(firstCell), indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(lastIndexRowKey), viewConstants)); - indexRows.add(row); + // When a row has at most one cell, we can safely reuse it as we don't look at cell value, + // and we proceed with data table scan. + // For rows with multiple cells, we must copy each cell to avoid issues where the + // underlying byte buffers for cell values may be reused or invalidated by the scanner + // on subsequent nextRaw() calls and hence the cell value could be garbage under + // high GC pressure. + if (row.size() <= 1) { + indexRows.add(row); + } else { + List dupRow = new ArrayList<>(row.size()); + for (Cell cell : row) { + dupRow.add(PhoenixKeyValueUtil.maybeCopyCell(cell)); + } + indexRows.add(dupRow); + } indexRowCount++; if ( hasMore && (PhoenixScannerContext.isTimedOut(scannerContext, pageSizeMs) diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexCDCConsumer.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexCDCConsumer.java new file mode 100644 index 00000000000..6c4ceab789f --- /dev/null +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexCDCConsumer.java @@ -0,0 +1,1347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hbase.index; + +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.ConnectionUtils; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.coprocessor.DelegateRegionCoprocessorEnvironment; +import org.apache.phoenix.coprocessor.generated.IndexMutationsProtos; +import org.apache.phoenix.coprocessorclient.MetaDataProtocol; +import org.apache.phoenix.hbase.index.metrics.MetricsIndexCDCConsumerSource; +import org.apache.phoenix.hbase.index.metrics.MetricsIndexerSourceFactory; +import org.apache.phoenix.hbase.index.table.HTableInterfaceReference; +import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.hbase.index.write.IndexWriter; +import org.apache.phoenix.index.IndexMaintainer; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.RowKeySchema; +import org.apache.phoenix.schema.TableNotFoundException; +import org.apache.phoenix.schema.types.IndexConsistency; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CDCUtil; +import org.apache.phoenix.util.EnvironmentEdgeManager; +import org.apache.phoenix.util.QueryUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.ServerUtil.ConnectionType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.phoenix.thirdparty.com.google.common.collect.ArrayListMultimap; +import org.apache.phoenix.thirdparty.com.google.common.collect.ListMultimap; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; + +/** + * A single-threaded background consumer that processes CDC mutations for eventually consistent + * indexes. This consumer reads mutations from the CDC index table and applies them to the + * appropriate secondary indexes. + *

+ * The consumer tracks its progress in the SYSTEM.IDX_CDC_TRACKER table, allowing for proper + * handling of region splits and merges. + *

+ *

+ * Lifecycle: + *

    + *
  1. On startup, the consumer first replays any remaining mutations from parent regions (if this + * region was created from a split or merge). A region will have multiple parent regions in case of + * region merge.
    2. + *
  2. After replaying all parent mutations, the consumer marks each parent region as COMPLETE in + * SYSTEM.IDX_CDC_TRACKER.
    3. + *
  3. The consumer then begins processing mutations for the current region. With every batch of + * rows consumed (e.g., 1000 rows), it updates LAST_TIMESTAMP in SYSTEM.IDX_CDC_TRACKER. The tracker + * entry with status IN_PROGRESS is created on the first batch update.
    4. + *
  4. When the region is closed (due to split, merge, or shutdown), the consumer stops but does NOT + * mark itself as COMPLETE. The COMPLETE status is only set by child regions after they have + * replayed all parent mutations.
    5. + *
+ *

+ */ +public class IndexCDCConsumer implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(IndexCDCConsumer.class); + + public static final String INDEX_CDC_CONSUMER_BATCH_SIZE = + "phoenix.index.cdc.consumer.batch.size"; + private static final int DEFAULT_CDC_BATCH_SIZE = 500; + + public static final String INDEX_CDC_CONSUMER_STARTUP_DELAY_MS = + "phoenix.index.cdc.consumer.startup.delay.ms"; + private static final long DEFAULT_STARTUP_DELAY_MS = 10000; + + /** + * The interval in milliseconds between processing batches when mutations are found. + */ + public static final String INDEX_CDC_CONSUMER_POLL_INTERVAL_MS = + "phoenix.index.cdc.consumer.poll.interval.ms"; + private static final long DEFAULT_POLL_INTERVAL_MS = 1000; + + /** + * The time buffer in milliseconds subtracted from current time when querying CDC mutations to + * help avoid reading mutations that are too recent. + */ + public static final String INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS = + "phoenix.index.cdc.consumer.timestamp.buffer.ms"; + private static final long DEFAULT_TIMESTAMP_BUFFER_MS = 5000; + + /** + * Maximum number of retries when CDC events exist but the corresponding data table mutations are + * not yet visible (or permanently failed). After exceeding this limit, the consumer advances past + * the unprocessable events to avoid blocking indefinitely. This is only used for index mutation + * generation approach (serializeCDCMutations = false). + */ + public static final String INDEX_CDC_CONSUMER_MAX_DATA_VISIBILITY_RETRIES = + "phoenix.index.cdc.consumer.max.data.visibility.retries"; + private static final int DEFAULT_MAX_DATA_VISIBILITY_RETRIES = 10; + + public static final String INDEX_CDC_CONSUMER_RETRY_PAUSE_MS = + "phoenix.index.cdc.consumer.retry.pause.ms"; + private static final long DEFAULT_RETRY_PAUSE_MS = 2000; + + public static final String INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS = + "phoenix.index.cdc.consumer.parent.progress.pause.ms"; + private static final long DEFAULT_PARENT_PROGRESS_PAUSE_MS = 15000; + + private final RegionCoprocessorEnvironment env; + private final String dataTableName; + private final String encodedRegionName; + private final IndexWriter indexWriter; + private final long pause; + private final long startupDelayMs; + private final int batchSize; + private final long pollIntervalMs; + private final long timestampBufferMs; + private final int maxDataVisibilityRetries; + private final long parentProgressPauseMs; + private final Configuration config; + private final boolean serializeCDCMutations; + private final MetricsIndexCDCConsumerSource metricSource; + private volatile boolean stopped = false; + private Thread consumerThread; + private boolean hasParentPartitions = false; + private PTable cachedDataTable; + + private boolean tenantInit = false; + private boolean isMultiTenant = false; + private String tenantIdColName; + private PDataType tenantIdDataType; + private TenantScanInfo ownRegionScanInfo; + + private final Map ancestorScanInfoCache = new HashMap<>(); + + private static class TenantScanInfo { + + private static final TenantScanInfo EMPTY = new TenantScanInfo("", "", null, null, null); + + private final String filter; + private final String orderBy; + private final Object startValue; + private final Object endValue; + private final PDataType dataType; + + TenantScanInfo(String filter, String orderBy, Object startValue, Object endValue, + PDataType dataType) { + this.filter = filter; + this.orderBy = orderBy; + this.startValue = startValue; + this.endValue = endValue; + this.dataType = dataType; + } + + int bindParams(PreparedStatement ps, int startIndex) throws SQLException { + int idx = startIndex; + if (startValue != null) { + ps.setObject(idx++, startValue, dataType.getSqlType()); + } + if (endValue != null) { + ps.setObject(idx++, endValue, dataType.getSqlType()); + } + return idx; + } + } + + /** + * Creates a new IndexCDCConsumer for the given region with configurable serialization mode. + * @param env region coprocessor environment. + * @param dataTableName name of the data table. + * @param serverName server name. + * @param serializeCDCMutations when true, consumes pre-serialized index mutations; when false, + * generates index mutations from data row states. + * @throws IOException if the IndexWriter cannot be created. + */ + public IndexCDCConsumer(RegionCoprocessorEnvironment env, String dataTableName, String serverName, + boolean serializeCDCMutations) throws IOException { + this.env = env; + this.dataTableName = dataTableName; + this.encodedRegionName = env.getRegion().getRegionInfo().getEncodedName(); + this.config = env.getConfiguration(); + this.serializeCDCMutations = serializeCDCMutations; + this.pause = config.getLong(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, DEFAULT_RETRY_PAUSE_MS); + this.startupDelayMs = + config.getLong(INDEX_CDC_CONSUMER_STARTUP_DELAY_MS, DEFAULT_STARTUP_DELAY_MS); + int baseBatchSize = config.getInt(INDEX_CDC_CONSUMER_BATCH_SIZE, DEFAULT_CDC_BATCH_SIZE); + int jitter = ThreadLocalRandom.current().nextInt(baseBatchSize / 5 + 1); + this.batchSize = baseBatchSize + jitter; + this.pollIntervalMs = + config.getLong(INDEX_CDC_CONSUMER_POLL_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS); + this.timestampBufferMs = + config.getLong(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, DEFAULT_TIMESTAMP_BUFFER_MS); + this.maxDataVisibilityRetries = config.getInt(INDEX_CDC_CONSUMER_MAX_DATA_VISIBILITY_RETRIES, + DEFAULT_MAX_DATA_VISIBILITY_RETRIES); + this.parentProgressPauseMs = + config.getLong(INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS, DEFAULT_PARENT_PROGRESS_PAUSE_MS); + this.metricSource = MetricsIndexerSourceFactory.getInstance().getIndexCDCConsumerSource(); + DelegateRegionCoprocessorEnvironment indexWriterEnv = + new DelegateRegionCoprocessorEnvironment(env, ConnectionType.INDEX_WRITER_CONNECTION); + this.indexWriter = + new IndexWriter(indexWriterEnv, serverName + "-index-eventual-writer", false); + } + + /** + * Starts the consumer thread in the background. + */ + public void start() { + consumerThread = + new Thread(this, "IndexCDCConsumer-" + dataTableName + "-" + encodedRegionName); + consumerThread.setDaemon(true); + consumerThread.start(); + } + + /** + * Stops the consumer thread gracefully. + */ + public void stop() { + stopped = true; + if (consumerThread != null) { + consumerThread.interrupt(); + } + if (indexWriter != null) { + indexWriter.stop("IndexCDCConsumer stopped for " + dataTableName); + } + } + + /** + * Sleeps for the specified duration if the consumer has not been stopped. + * @param millis the duration to sleep in milliseconds. + * @throws InterruptedException if the thread is interrupted while sleeping. + */ + private void sleepIfNotStopped(long millis) throws InterruptedException { + if (!stopped) { + Thread.sleep(millis); + } + } + + private PTable getDataTable(PhoenixConnection conn) throws SQLException { + PTable dataTable = cachedDataTable; + if (dataTable == null) { + dataTable = conn.getTable(dataTableName); + cachedDataTable = dataTable; + } + return dataTable; + } + + private void refreshDataTableCache(PhoenixConnection conn) throws SQLException { + cachedDataTable = conn.getTable(dataTableName); + } + + private void initTenantInfo(PhoenixConnection conn) throws SQLException { + if (tenantInit) { + return; + } + PTable dataTable = getDataTable(conn); + isMultiTenant = dataTable.isMultiTenant(); + if (!isMultiTenant) { + ownRegionScanInfo = TenantScanInfo.EMPTY; + tenantInit = true; + return; + } + int tenantColIndex = dataTable.getBucketNum() != null ? 1 : 0; + PColumn tenantCol = dataTable.getPKColumns().get(tenantColIndex); + tenantIdColName = tenantCol.getName().getString(); + tenantIdDataType = tenantCol.getDataType(); + + byte[] regionStartKey = env.getRegion().getRegionInfo().getStartKey(); + byte[] regionEndKey = env.getRegion().getRegionInfo().getEndKey(); + ownRegionScanInfo = buildTenantScanInfo(regionStartKey, regionEndKey, dataTable); + LOG.debug( + "Initialized multi-tenant scan for table {} region {}:" + + " tenantCol {}, startTenant {}, endTenant {}", + dataTableName, encodedRegionName, tenantIdColName, ownRegionScanInfo.startValue, + ownRegionScanInfo.endValue); + tenantInit = true; + } + + private TenantScanInfo buildTenantScanInfo(byte[] startKey, byte[] endKey, PTable dataTable) { + Object startVal = extractTenantIdFromRegionKey(startKey, dataTable); + Object endVal = extractTenantIdFromRegionKey(endKey, dataTable); + StringBuilder sb = new StringBuilder(); + if (startVal != null) { + sb.append("\"").append(tenantIdColName).append("\" >= ? AND "); + } + if (endVal != null) { + sb.append("\"").append(tenantIdColName).append("\" <= ? AND "); + } + String filter = sb.toString(); + String orderBy = filter.isEmpty() ? "" : "\"" + tenantIdColName + "\" ASC,"; + return new TenantScanInfo(filter, orderBy, startVal, endVal, tenantIdDataType); + } + + private Object extractTenantIdFromRegionKey(byte[] regionKey, PTable dataTable) { + if (regionKey == null || regionKey.length == 0) { + return null; + } + final RowKeySchema schema = dataTable.getRowKeySchema(); + int pkPos = dataTable.getBucketNum() != null ? 1 : 0; + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + int maxOffset = schema.iterator(regionKey, 0, regionKey.length, ptr); + for (int i = 0; i <= pkPos; i++) { + Boolean hasValue = schema.next(ptr, i, maxOffset); + if (!Boolean.TRUE.equals(hasValue)) { + return null; + } + } + byte[] tenantBytes = ByteUtil.copyKeyBytesIfNecessary(ptr); + PColumn tenantCol = dataTable.getPKColumns().get(pkPos); + return tenantCol.getDataType().toObject(tenantBytes, 0, tenantBytes.length, + tenantCol.getDataType(), tenantCol.getSortOrder(), tenantCol.getMaxLength(), + tenantCol.getScale()); + } + + private byte[][] lookupPartitionKeys(String partitionId) throws InterruptedException { + int retryCount = 0; + final String query = "SELECT PARTITION_START_KEY, PARTITION_END_KEY FROM " + + PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_NAME + + " WHERE TABLE_NAME = ? AND PARTITION_ID = ? LIMIT 1"; + while (!stopped) { + try ( + PhoenixConnection conn = + QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class); + PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, dataTableName); + ps.setString(2, partitionId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + byte[] startKey = rs.getBytes(1); + byte[] endKey = rs.getBytes(2); + return new byte[][] { startKey == null ? new byte[0] : startKey, + endKey == null ? new byte[0] : endKey }; + } + } + LOG.error("No CDC_STREAM entry found for partition {} table {}. This should not happen.", + partitionId, dataTableName); + return new byte[][] { new byte[0], new byte[0] }; + } catch (SQLException e) { + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error while retrieving partition keys from CDC_STREAM for partition {} table {}. " + + "Retry #{}, sleeping {} ms before retrying...", + partitionId, dataTableName, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + return null; + } + + private TenantScanInfo getPartitionTenantScanInfo(String partitionId) + throws InterruptedException { + if (!isMultiTenant) { + return TenantScanInfo.EMPTY; + } + if (partitionId.equals(encodedRegionName)) { + return ownRegionScanInfo; + } + TenantScanInfo cached = ancestorScanInfoCache.get(partitionId); + if (cached != null) { + return cached; + } + byte[][] keys = lookupPartitionKeys(partitionId); + if (keys == null) { + return TenantScanInfo.EMPTY; + } + TenantScanInfo info = buildTenantScanInfo(keys[0], keys[1], cachedDataTable); + ancestorScanInfoCache.put(partitionId, info); + return info; + } + + @Override + public void run() { + try { + if (startupDelayMs > 0 && getCDCStreamNumPartitions() <= 1) { + sleepIfNotStopped(startupDelayMs); + } + if (stopped) { + return; + } + if (!hasEventuallyConsistentIndexes()) { + LOG.trace("No eventually consistent indexes found for table {}. Exiting consumer.", + dataTableName); + return; + } + LOG.info( + "IndexCDCConsumer started for table {} region {}" + + " [batchSize: {}, pollIntervalMs: {}, timestampBufferMs: {}, startupDelayMs: {}," + + " pause: {}, maxDataVisibilityRetries: {}, parentProgressPauseMs: {}," + + " serializeCDCMutations: {}]", + dataTableName, encodedRegionName, batchSize, pollIntervalMs, timestampBufferMs, + startupDelayMs, pause, maxDataVisibilityRetries, parentProgressPauseMs, + serializeCDCMutations); + if (!waitForCDCStreamEntry()) { + LOG.error( + "IndexCDCConsumer stopped while waiting for CDC_STREAM entry for table {} region {}", + dataTableName, encodedRegionName); + return; + } + long lastProcessedTimestamp = checkTrackerStatus(encodedRegionName, encodedRegionName); + if (lastProcessedTimestamp == -1) { + // should never happen as COMPLETE is only set by child regions for their parent regions + LOG.error( + "Unexpected COMPLETE status in IDX_CDC_TRACKER for current region. " + + "Table: {}, Partition: {}. Exiting consumer thread.", + dataTableName, encodedRegionName); + return; + } else if (lastProcessedTimestamp > 0) { + LOG.info( + "Found existing tracker entry for table {} region {} with lastTimestamp {}. " + + "Resuming from last position (region movement scenario).", + dataTableName, encodedRegionName, lastProcessedTimestamp); + } else { + if (hasParentPartitions) { + sleepIfNotStopped(timestampBufferMs + 1); + replayAndCompleteParentRegions(encodedRegionName); + } else { + LOG.info("No parent partitions for table {} region {}, skipping parent replay", + dataTableName, encodedRegionName); + } + } + int retryCount = 0; + while (!stopped) { + try { + long previousTimestamp = lastProcessedTimestamp; + if (serializeCDCMutations) { + lastProcessedTimestamp = + processCDCBatch(encodedRegionName, encodedRegionName, lastProcessedTimestamp, false); + } else { + lastProcessedTimestamp = processCDCBatchGenerated(encodedRegionName, encodedRegionName, + lastProcessedTimestamp, false); + } + if (lastProcessedTimestamp == previousTimestamp) { + sleepIfNotStopped(ConnectionUtils.getPauseTime(pause, ++retryCount)); + } else { + retryCount = 0; + sleepIfNotStopped(pollIntervalMs); + } + } catch (Exception e) { + if (e instanceof InterruptedException) { + throw (InterruptedException) e; + } + metricSource.incrementCdcBatchFailureCount(dataTableName); + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.error( + "Error processing CDC mutations for table {} region {}. " + + "Retry #{}, sleeping {} ms before retrying...", + dataTableName, encodedRegionName, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + } catch (InterruptedException e) { + if (!stopped) { + LOG.warn("IndexCDCConsumer interrupted unexpectedly for table {} region {}", dataTableName, + encodedRegionName, e); + } + Thread.currentThread().interrupt(); + } + LOG.info("IndexCDCConsumer exiting for table {} region {}", dataTableName, encodedRegionName); + } + + private boolean hasEventuallyConsistentIndexes() throws InterruptedException { + int retryCount = 0; + while (!stopped) { + try (PhoenixConnection conn = + QueryUtil.getConnectionOnServer(config).unwrap(PhoenixConnection.class)) { + refreshDataTableCache(conn); + PTable dataTable = getDataTable(conn); + String cdcObjectName = CDCUtil.getCDCObjectName(dataTable, false); + if (cdcObjectName == null) { + LOG.debug("No CDC index found for table {}. Exiting consumer.", dataTableName); + return false; + } + for (PTable index : dataTable.getIndexes()) { + IndexConsistency consistency = index.getIndexConsistency(); + if (consistency != null && consistency.isAsynchronous()) { + LOG.debug("Found eventually consistent index {} for table {}", + index.getName().getString(), dataTableName); + return true; + } + } + return false; + } catch (SQLException e) { + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error checking for eventually consistent indexes for table {}. " + + "Retry #{}, sleeping {} ms before retrying...", + dataTableName, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + return false; + } + + /** + * Retrieves the count of partitions for the given table. + * @return the count of CDC_STREAM rows for this table. + * @throws InterruptedException if the thread is interrupted while waiting. + */ + private long getCDCStreamNumPartitions() throws InterruptedException { + int retryCount = 0; + String query = "SELECT COUNT(*) FROM " + PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_NAME + + " WHERE TABLE_NAME = ?"; + while (!stopped) { + try ( + PhoenixConnection conn = + QueryUtil.getConnectionOnServer(config).unwrap(PhoenixConnection.class); + PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, dataTableName); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + } + return 0; + } catch (SQLException e) { + if (e instanceof TableNotFoundException) { + TableNotFoundException tnfe = (TableNotFoundException) e; + // 5.3.0+ server with old metadata tables (i.e. EXECUTE_UPGRADE is not yet run) + if (PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_TABLE.equals(tnfe.getTableName())) { + stopped = true; + return -1; + } + } + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error getting CDC_STREAM row count for table {}. " + + "Retry #{}, sleeping {} ms before retrying...", + dataTableName, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + return -1; + } + + /** + * Waits for the CDC_STREAM entry for the given table and partition to be available. + * @return true if the entry was found, false if the consumer was stopped before finding the + * entry. + * @throws InterruptedException if the thread is interrupted while waiting. + */ + private boolean waitForCDCStreamEntry() throws InterruptedException { + int retryCount = 0; + String query = + "SELECT PARENT_PARTITION_ID FROM " + PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_NAME + + " WHERE TABLE_NAME = ? AND PARTITION_ID = ? LIMIT 1"; + while (!stopped) { + try ( + PhoenixConnection conn = + QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class); + PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, dataTableName); + ps.setString(2, encodedRegionName); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + String parentPartitionId = rs.getString(1); + hasParentPartitions = parentPartitionId != null; + LOG.debug("Found CDC_STREAM entry for table {} partition {}, hasParentPartitions={}", + dataTableName, encodedRegionName, hasParentPartitions); + return true; + } + } + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.info( + "CDC_STREAM entry not found for table {} partition {}. " + + "Attempt #{}, sleeping {} ms before retrying...", + dataTableName, encodedRegionName, retryCount, sleepTime); + sleepIfNotStopped(sleepTime); + } catch (SQLException e) { + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error checking CDC_STREAM for table {} partition {}. " + + "Retry #{}, sleeping {} ms before retrying...", + dataTableName, encodedRegionName, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + return false; + } + + /** + * Checks for an existing entry for the given partition and owner. + * @param partitionId the partition ID to check. + * @param ownerPartitionId the owner partition ID. + * @return the last processed timestamp if an entry exists with status IN_PROGRESS, -1 if the + * entry exists with status COMPLETE, or 0 if no entry exists. + * @throws InterruptedException if the thread is interrupted while waiting + */ + private long checkTrackerStatus(String partitionId, String ownerPartitionId) + throws InterruptedException { + int retryCount = 0; + String query = + "SELECT LAST_TIMESTAMP, STATUS FROM " + PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_NAME + + " WHERE TABLE_NAME = ? AND PARTITION_ID = ? AND OWNER_PARTITION_ID = ?"; + while (!stopped) { + try ( + PhoenixConnection conn = + QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class); + PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, dataTableName); + ps.setString(2, partitionId); + ps.setString(3, ownerPartitionId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + long lastTimestamp = rs.getLong(1); + String status = rs.getString(2); + LOG.debug( + "Found IDX_CDC_TRACKER entry for table {} partition {} owner {} " + + "with status={}, lastTimestamp={}", + dataTableName, partitionId, ownerPartitionId, status, lastTimestamp); + if (PhoenixDatabaseMetaData.TRACKER_STATUS_IN_PROGRESS.equals(status)) { + return lastTimestamp; + } else if (PhoenixDatabaseMetaData.TRACKER_STATUS_COMPLETE.equals(status)) { + return -1; + } else { + // Unknown status - should not happen + throw new IllegalStateException( + String.format("Unknown tracker status '%s' for table %s partition %s owner %s.", + status, dataTableName, partitionId, ownerPartitionId)); + } + } else { + LOG.debug("No IDX_CDC_TRACKER entry found for table {} partition {} owner {}", + dataTableName, partitionId, ownerPartitionId); + return 0; + } + } + } catch (SQLException e) { + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error checking IDX_CDC_TRACKER for table {} partition {} owner {}. " + + "Retry #{}, sleeping {} ms before retrying...", + dataTableName, partitionId, ownerPartitionId, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + return 0; + } + + /** + * Checks if any consumer has already completed processing the given partition. + * @param partitionId the partition ID to check + * @return true if the partition is done processing. + * @throws InterruptedException if the thread is interrupted while waiting. + */ + private boolean isPartitionCompleted(String partitionId) throws InterruptedException { + int retryCount = 0; + String query = "SELECT * FROM " + PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_NAME + + " WHERE TABLE_NAME = ? AND PARTITION_ID = ? AND STATUS = ?"; + while (!stopped) { + try ( + PhoenixConnection conn = + QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class); + PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, dataTableName); + ps.setString(2, partitionId); + ps.setString(3, PhoenixDatabaseMetaData.TRACKER_STATUS_COMPLETE); + try (ResultSet rs = ps.executeQuery()) { + return rs.next(); + } + } catch (SQLException e) { + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error checking if partition {} is completed for table {}. " + + "Retry #{}, sleeping {} ms before retrying...", + partitionId, dataTableName, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + return false; + } + + /** + * Gets the maximum last processed timestamp from any consumer's tracker record for the given + * partition. This is used when a child region needs to continue processing from where any + * previous consumer left off. + * @param partitionId the partition ID to get progress for. + * @return the max last processed timestamp across all owners, or 0 if not found. + * @throws InterruptedException if the thread is interrupted while waiting. + */ + private long getParentProgress(String partitionId) throws InterruptedException { + int retryCount = 0; + String query = + "SELECT MAX(LAST_TIMESTAMP) FROM " + PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_NAME + + " WHERE TABLE_NAME = ? AND PARTITION_ID = ? AND STATUS = ?"; + while (!stopped) { + try ( + PhoenixConnection conn = + QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class); + PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, dataTableName); + ps.setString(2, partitionId); + ps.setString(3, PhoenixDatabaseMetaData.TRACKER_STATUS_IN_PROGRESS); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + long maxTimestamp = rs.getLong(1); + if (maxTimestamp > 0) { + LOG.debug("Found max progress {} for partition {} from previous consumers", + maxTimestamp, partitionId); + return maxTimestamp; + } + } + return 0; + } + } catch (SQLException e) { + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error getting parent progress for partition {} table {}. " + + "Retry #{}, sleeping {} ms before retrying...", + partitionId, dataTableName, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + throw new InterruptedException("IndexCDCConsumer stopped while getting parent progress."); + } + + /** + * Retrieves all parent partition IDs for the given partition. + * @param partitionId the partition ID to find parents for. + * @return list of parent partition IDs. + * @throws InterruptedException if the thread is interrupted. + */ + private List getParentPartitionIds(String partitionId) throws InterruptedException { + int retryCount = 0; + String query = + "SELECT PARENT_PARTITION_ID FROM " + PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_NAME + + " WHERE TABLE_NAME = ? AND PARTITION_ID = ? AND PARENT_PARTITION_ID IS NOT NULL"; + while (!stopped) { + try ( + PhoenixConnection conn = + QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class); + PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, dataTableName); + ps.setString(2, partitionId); + List parentIds = new ArrayList<>(); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + parentIds.add(rs.getString(1)); + } + } + LOG.debug("Found {} parent partition(s) for table {} partition {}: {}", parentIds.size(), + dataTableName, partitionId, parentIds); + return parentIds; + } catch (SQLException e) { + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error querying parent partitions from CDC_STREAM for table {} partition {}. " + + "Retry #{}, sleeping {} ms before retrying...", + dataTableName, partitionId, retryCount, sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + return Collections.emptyList(); + } + + /** + * Processes all remaining CDC mutations for the given partition until completion, then marks the + * partition as COMPLETE. Used to replay ancestor partitions (regions) after they are split or + * merged into new regions. + * @param partitionId the partition ID to process. + * @param ownerPartitionId the owner partition ID. + * @param lastProcessedTimestamp the timestamp to start processing from. + * @throws InterruptedException if the thread is interrupted. + */ + private void processPartitionToCompletion(String partitionId, String ownerPartitionId, + long lastProcessedTimestamp) throws InterruptedException { + LOG.debug( + "Processing partition {} owner {} to completion for table {}, starting from timestamp {}", + partitionId, ownerPartitionId, dataTableName, lastProcessedTimestamp); + long currentLastProcessedTimestamp = lastProcessedTimestamp; + int retryCount = 0; + int batchCount = 0; + while (!stopped) { + try { + if (batchCount > 0) { + if (isPartitionCompleted(partitionId)) { + return; + } + long otherProgress = getParentProgress(partitionId); + if (otherProgress > currentLastProcessedTimestamp) { + long previousOtherProgress; + do { + previousOtherProgress = otherProgress; + sleepIfNotStopped(parentProgressPauseMs); + if (isPartitionCompleted(partitionId)) { + return; + } + otherProgress = getParentProgress(partitionId); + } while (!stopped && otherProgress > previousOtherProgress); + currentLastProcessedTimestamp = otherProgress; + } + } + long newTimestamp; + if (serializeCDCMutations) { + newTimestamp = + processCDCBatch(partitionId, ownerPartitionId, currentLastProcessedTimestamp, true); + } else { + newTimestamp = processCDCBatchGenerated(partitionId, ownerPartitionId, + currentLastProcessedTimestamp, true); + } + batchCount++; + retryCount = 0; + if (newTimestamp == currentLastProcessedTimestamp) { + if (isPartitionCompleted(partitionId)) { + LOG.info( + "Partition {} for table {} was completed by another consumer before {} could mark it", + partitionId, dataTableName, ownerPartitionId); + return; + } + LOG.info("Partition {} owner {} for table {} fully processed, marking as COMPLETE", + partitionId, ownerPartitionId, dataTableName); + try (PhoenixConnection conn = QueryUtil.getConnectionOnServer(env.getConfiguration()) + .unwrap(PhoenixConnection.class)) { + updateTrackerProgress(conn, partitionId, ownerPartitionId, + currentLastProcessedTimestamp, PhoenixDatabaseMetaData.TRACKER_STATUS_COMPLETE); + } + return; + } + currentLastProcessedTimestamp = newTimestamp; + } catch (SQLException | IOException e) { + metricSource.incrementCdcBatchFailureCount(dataTableName); + long sleepTime = ConnectionUtils.getPauseTime(pause, ++retryCount); + LOG.warn( + "Error processing CDC batch for partition {} owner {} table {} " + + "lastProcessedTimestamp {}. Retry #{}, sleeping {} ms", + partitionId, ownerPartitionId, dataTableName, currentLastProcessedTimestamp, retryCount, + sleepTime, e); + sleepIfNotStopped(sleepTime); + } + } + LOG.info("Processing partition {} (owner {}) stopped before completion for table {}", + partitionId, ownerPartitionId, dataTableName); + } + + /** + * Recursive helper method to replay and complete parent regions. Replays all remaining mutations + * from parent regions and marks them as COMPLETE. This is called during initialization when a + * region has parent regions (split/merge scenarios). + * @param partitionId the partition (region) ID to find and process parents for. + */ + private void replayAndCompleteParentRegions(String partitionId) throws InterruptedException { + List parentIds = getParentPartitionIds(partitionId); + if (parentIds.isEmpty()) { + LOG.debug("No parent partitions found for table {} partition {}", dataTableName, partitionId); + return; + } + for (String parentId : parentIds) { + if (stopped) { + return; + } + if (isPartitionCompleted(parentId)) { + LOG.debug("Parent partition {} for table {} already has a COMPLETE record, skipping", + parentId, dataTableName); + continue; + } + replayAndCompleteParentRegions(parentId); + if (isPartitionCompleted(parentId)) { + LOG.debug( + "Parent partition {} for table {} was completed by sibling during ancestor processing, " + + "skipping", + parentId, dataTableName); + continue; + } + long parentProgress = getParentProgress(parentId); + LOG.debug("Processing/Resuming parent partition {} for table {} from timestamp {} owner: {}", + parentId, dataTableName, parentProgress, encodedRegionName); + processPartitionToCompletion(parentId, encodedRegionName, parentProgress); + } + } + + /** + * Processes a batch of CDC mutations for the given partition starting from the specified + * timestamp. This method reads mutations from the CDC index, applies them to the respective + * eventually consistent index tables, and updates the progress in SYSTEM.IDX_CDC_TRACKER. + * @param partitionId the partition (region) ID to process CDC mutations for. + * @param ownerPartitionId the owner partition ID. + * @param lastProcessedTimestamp the timestamp to start reading CDC mutations from. + * @param isParentReplay true if replaying a closed parent partition. + * @return the new last processed timestamp after this batch, or the same timestamp if no new + * records were found. + * @throws SQLException if SQL error occurs. + * @throws IOException if an I/O error occurs. + * @throws InterruptedException if the thread is interrupted while waiting. + */ + private long processCDCBatch(String partitionId, String ownerPartitionId, + long lastProcessedTimestamp, boolean isParentReplay) + throws SQLException, IOException, InterruptedException { + long batchStartTime = EnvironmentEdgeManager.currentTimeMillis(); + LOG.debug("Processing CDC batch for table {} partition {} owner {} from timestamp {}", + dataTableName, partitionId, ownerPartitionId, lastProcessedTimestamp); + try (PhoenixConnection conn = + QueryUtil.getConnectionOnServer(config).unwrap(PhoenixConnection.class)) { + initTenantInfo(conn); + String cdcObjectName = getCdcObjectName(conn); + TenantScanInfo scanInfo = getPartitionTenantScanInfo(partitionId); + String cdcQuery; + if (isParentReplay) { + cdcQuery = String.format( + "SELECT /*+ CDC_INCLUDE(IDX_MUTATIONS) */ PHOENIX_ROW_TIMESTAMP(), \"CDC JSON\" " + + "FROM %s WHERE %s PARTITION_ID() = ? AND PHOENIX_ROW_TIMESTAMP() > ? " + + "ORDER BY %s PARTITION_ID() ASC, PHOENIX_ROW_TIMESTAMP() ASC LIMIT ?", + cdcObjectName, scanInfo.filter, scanInfo.orderBy); + } else { + cdcQuery = String.format( + "SELECT /*+ CDC_INCLUDE(IDX_MUTATIONS) */ PHOENIX_ROW_TIMESTAMP(), \"CDC JSON\" " + + "FROM %s WHERE %s PARTITION_ID() = ? AND PHOENIX_ROW_TIMESTAMP() > ? " + + "AND PHOENIX_ROW_TIMESTAMP() < ? " + + "ORDER BY %s PARTITION_ID() ASC, PHOENIX_ROW_TIMESTAMP() ASC LIMIT ?", + cdcObjectName, scanInfo.filter, scanInfo.orderBy); + } + List> batchMutations = new ArrayList<>(); + long newLastTimestamp = lastProcessedTimestamp; + boolean hasMoreRows = true; + int retryCount = 0; + while (hasMoreRows && batchMutations.isEmpty()) { + try (PreparedStatement ps = conn.prepareStatement(cdcQuery)) { + setStatementParams(scanInfo, partitionId, isParentReplay, newLastTimestamp, ps); + Pair result = + getMutationsAndTimestamp(ps, newLastTimestamp, batchMutations); + hasMoreRows = result.getSecond(); + if (hasMoreRows) { + newLastTimestamp = result.getFirst(); + if (batchMutations.isEmpty()) { + sleepIfNotStopped(ConnectionUtils.getPauseTime(pause, ++retryCount)); + } + } + } + } + // With predefined LIMIT, there might be more rows with the same timestamp that were not + // included in this batch. + if (newLastTimestamp > lastProcessedTimestamp) { + String sameTimestampQuery = String.format( + "SELECT /*+ CDC_INCLUDE(IDX_MUTATIONS) */ PHOENIX_ROW_TIMESTAMP(), \"CDC JSON\" " + + "FROM %s WHERE %s PARTITION_ID() = ? AND PHOENIX_ROW_TIMESTAMP() = ? " + + "ORDER BY %s PARTITION_ID() ASC, PHOENIX_ROW_TIMESTAMP() ASC", + cdcObjectName, scanInfo.filter, scanInfo.orderBy); + final long timestampToRefetch = newLastTimestamp; + batchMutations.removeIf(pair -> pair.getFirst() == timestampToRefetch); + try (PreparedStatement ps = conn.prepareStatement(sameTimestampQuery)) { + int idx = scanInfo.bindParams(ps, 1); + ps.setString(idx++, partitionId); + ps.setDate(idx, new Date(newLastTimestamp)); + Pair result = + getMutationsAndTimestamp(ps, newLastTimestamp, batchMutations); + newLastTimestamp = result.getFirst(); + if (newLastTimestamp != timestampToRefetch) { + throw new IOException("Unexpected timestamp mismatch: expected " + timestampToRefetch + + " but got " + newLastTimestamp); + } + } + } + executeIndexMutations(partitionId, batchMutations, ownerPartitionId, newLastTimestamp); + if (!batchMutations.isEmpty()) { + metricSource.updateCdcBatchProcessTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - batchStartTime); + metricSource.incrementCdcBatchCount(dataTableName); + metricSource.updateCdcLag(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - newLastTimestamp); + updateTrackerProgress(conn, partitionId, ownerPartitionId, newLastTimestamp, + PhoenixDatabaseMetaData.TRACKER_STATUS_IN_PROGRESS); + } + return newLastTimestamp; + } + } + + private String getCdcObjectName(PhoenixConnection conn) throws SQLException { + PTable dataTable = getDataTable(conn); + String cdcObjectName = CDCUtil.getCDCObjectName(dataTable, false); + if (cdcObjectName == null) { + throw new SQLException("No CDC object found for table " + dataTableName); + } + return SchemaUtil.getEscapedTableName(dataTable.getSchemaName().getString(), cdcObjectName); + } + + /** + * Processes a batch of CDC events for the given partition starting from the specified timestamp + * by generating index mutations from data row states. This method queries the CDC index with the + * DATA_ROW_STATE scope, which triggers a server-side data table scan to reconstruct the + * before-image ({@code currentDataRowState}) and after-image ({@code nextDataRowState}) for each + * change. + * @param partitionId the partition (region) ID to process CDC events for. + * @param ownerPartitionId the owner partition ID. + * @param lastProcessedTimestamp the timestamp to start processing CDC events from. + * @param isParentReplay true if replaying a closed parent partition. + * @return the new last processed timestamp after this batch, or the same timestamp if no new + * records were found. + * @throws SQLException if a SQL error occurs. + * @throws IOException if an I/O error occurs. + * @throws InterruptedException if the thread is interrupted while waiting. + */ + private long processCDCBatchGenerated(String partitionId, String ownerPartitionId, + long lastProcessedTimestamp, boolean isParentReplay) + throws SQLException, IOException, InterruptedException { + long batchStartTime = EnvironmentEdgeManager.currentTimeMillis(); + LOG.debug( + "Processing CDC batch (generated mode) for table {} partition {} owner {} from timestamp {}", + dataTableName, partitionId, ownerPartitionId, lastProcessedTimestamp); + try (PhoenixConnection conn = + QueryUtil.getConnectionOnServer(config).unwrap(PhoenixConnection.class)) { + initTenantInfo(conn); + String cdcObjectName = getCdcObjectName(conn); + TenantScanInfo scanInfo = getPartitionTenantScanInfo(partitionId); + String cdcQuery; + if (isParentReplay) { + cdcQuery = String.format( + "SELECT /*+ CDC_INCLUDE(DATA_ROW_STATE) */ PHOENIX_ROW_TIMESTAMP(), \"CDC JSON\" " + + "FROM %s WHERE %s PARTITION_ID() = ? AND PHOENIX_ROW_TIMESTAMP() > ? " + + "ORDER BY %s PARTITION_ID() ASC, PHOENIX_ROW_TIMESTAMP() ASC LIMIT ?", + cdcObjectName, scanInfo.filter, scanInfo.orderBy); + } else { + cdcQuery = String.format( + "SELECT /*+ CDC_INCLUDE(DATA_ROW_STATE) */ PHOENIX_ROW_TIMESTAMP(), \"CDC JSON\" " + + "FROM %s WHERE %s PARTITION_ID() = ? AND PHOENIX_ROW_TIMESTAMP() > ? " + + "AND PHOENIX_ROW_TIMESTAMP() < ? " + + "ORDER BY %s PARTITION_ID() ASC, PHOENIX_ROW_TIMESTAMP() ASC LIMIT ?", + cdcObjectName, scanInfo.filter, scanInfo.orderBy); + } + + List> batchStates = new ArrayList<>(); + long newLastTimestamp = lastProcessedTimestamp; + long[] lastScannedTimestamp = { lastProcessedTimestamp }; + boolean hasMoreRows = true; + int retryCount = 0; + while (hasMoreRows && batchStates.isEmpty()) { + try (PreparedStatement ps = conn.prepareStatement(cdcQuery)) { + setStatementParams(scanInfo, partitionId, isParentReplay, newLastTimestamp, ps); + Pair result = + getDataRowStatesAndTimestamp(ps, newLastTimestamp, batchStates, lastScannedTimestamp); + hasMoreRows = result.getSecond(); + if (hasMoreRows) { + if (!batchStates.isEmpty()) { + newLastTimestamp = result.getFirst(); + } else if (retryCount >= maxDataVisibilityRetries) { + LOG.warn( + "Skipping CDC events for table {} partition {} from timestamp {}" + + " to {} after {} retries — data table mutations may have failed", + dataTableName, partitionId, newLastTimestamp, lastScannedTimestamp[0], retryCount); + newLastTimestamp = lastScannedTimestamp[0]; + break; + } else { + // CDC index entries are written but the data is not yet visible. + // Don't advance newLastTimestamp so the same events are re-fetched + // once the data becomes visible. + sleepIfNotStopped(ConnectionUtils.getPauseTime(pause, ++retryCount)); + } + } + } + } + if (newLastTimestamp > lastProcessedTimestamp) { + String sameTimestampQuery = String.format( + "SELECT /*+ CDC_INCLUDE(DATA_ROW_STATE) */ PHOENIX_ROW_TIMESTAMP(), \"CDC JSON\" " + + "FROM %s WHERE %s PARTITION_ID() = ? AND PHOENIX_ROW_TIMESTAMP() = ? " + + "ORDER BY %s PARTITION_ID() ASC, PHOENIX_ROW_TIMESTAMP() ASC", + cdcObjectName, scanInfo.filter, scanInfo.orderBy); + final long timestampToRefetch = newLastTimestamp; + batchStates.removeIf(pair -> pair.getFirst() == timestampToRefetch); + try (PreparedStatement ps = conn.prepareStatement(sameTimestampQuery)) { + int idx = scanInfo.bindParams(ps, 1); + ps.setString(idx++, partitionId); + ps.setDate(idx, new Date(newLastTimestamp)); + Pair result = + getDataRowStatesAndTimestamp(ps, newLastTimestamp, batchStates, lastScannedTimestamp); + newLastTimestamp = result.getFirst(); + if (batchStates.isEmpty()) { + newLastTimestamp = timestampToRefetch; + } else if (newLastTimestamp != timestampToRefetch) { + throw new IOException("Unexpected timestamp mismatch: expected " + timestampToRefetch + + " but got " + newLastTimestamp); + } + } + } + generateAndApplyIndexMutations(conn, batchStates, partitionId, ownerPartitionId, + newLastTimestamp); + if (!batchStates.isEmpty()) { + metricSource.updateCdcBatchProcessTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - batchStartTime); + metricSource.incrementCdcBatchCount(dataTableName); + metricSource.updateCdcLag(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - newLastTimestamp); + } + if (newLastTimestamp > lastProcessedTimestamp) { + updateTrackerProgress(conn, partitionId, ownerPartitionId, newLastTimestamp, + PhoenixDatabaseMetaData.TRACKER_STATUS_IN_PROGRESS); + } + return newLastTimestamp; + } + } + + private void setStatementParams(TenantScanInfo scanInfo, String partitionId, + boolean isParentReplay, long newLastTimestamp, PreparedStatement ps) throws SQLException { + int idx = scanInfo.bindParams(ps, 1); + ps.setString(idx++, partitionId); + ps.setDate(idx++, new Date(newLastTimestamp)); + if (isParentReplay) { + ps.setInt(idx, batchSize); + } else { + long currentTime = EnvironmentEdgeManager.currentTimeMillis() - timestampBufferMs; + ps.setDate(idx++, new Date(currentTime)); + ps.setInt(idx, batchSize); + } + } + + private static Pair getDataRowStatesAndTimestamp(PreparedStatement ps, + long initialLastTimestamp, List> batchStates, + long[] lastScannedTimestamp) throws SQLException, IOException { + boolean hasRows = false; + long lastTimestamp = initialLastTimestamp; + lastScannedTimestamp[0] = initialLastTimestamp; + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + hasRows = true; + long rowTimestamp = rs.getDate(1).getTime(); + lastScannedTimestamp[0] = rowTimestamp; + String cdcValue = rs.getString(2); + if (cdcValue != null && !cdcValue.isEmpty()) { + byte[] protoBytes = Base64.getDecoder().decode(cdcValue); + IndexMutationsProtos.DataRowStates dataRowStates = + IndexMutationsProtos.DataRowStates.parseFrom(protoBytes); + if ( + dataRowStates.hasDataRowKey() + && (dataRowStates.hasCurrentDataRowState() || dataRowStates.hasNextDataRowState()) + ) { + batchStates.add(Pair.newPair(rowTimestamp, dataRowStates)); + lastTimestamp = rowTimestamp; + } + } + } + } + return Pair.newPair(lastTimestamp, hasRows); + } + + private void generateAndApplyIndexMutations(PhoenixConnection conn, + List> batchStates, String partitionId, + String ownerPartitionId, long lastProcessedTimestamp) throws SQLException, IOException { + if (batchStates.isEmpty()) { + return; + } + refreshDataTableCache(conn); + PTable dataTable = getDataTable(conn); + byte[] encodedRegionNameBytes = env.getRegion().getRegionInfo().getEncodedNameAsBytes(); + List> indexTables = new ArrayList<>(); + for (PTable index : dataTable.getIndexes()) { + IndexConsistency consistency = index.getIndexConsistency(); + if (consistency != null && consistency.isAsynchronous()) { + IndexMaintainer maintainer = index.getIndexMaintainer(dataTable, conn); + HTableInterfaceReference tableRef = + new HTableInterfaceReference(new ImmutableBytesPtr(maintainer.getIndexTableName())); + indexTables.add(new Pair<>(maintainer, tableRef)); + } + } + if (indexTables.isEmpty()) { + return; + } + ListMultimap indexUpdates = ArrayListMultimap.create(); + int totalMutations = 0; + long generateStartTime = EnvironmentEdgeManager.currentTimeMillis(); + for (Pair entry : batchStates) { + long ts = entry.getFirst(); + IndexMutationsProtos.DataRowStates dataRowStates = entry.getSecond(); + byte[] dataRowKey = dataRowStates.getDataRowKey().toByteArray(); + ImmutableBytesPtr rowKeyPtr = new ImmutableBytesPtr(dataRowKey); + + Put currentDataRowState = null; + if (dataRowStates.hasCurrentDataRowState()) { + ClientProtos.MutationProto currentProto = ClientProtos.MutationProto + .parseFrom(dataRowStates.getCurrentDataRowState().toByteArray()); + Mutation currentMutation = ProtobufUtil.toMutation(currentProto); + if (currentMutation instanceof Put) { + currentDataRowState = (Put) currentMutation; + } + } + Put nextDataRowState = null; + if (dataRowStates.hasNextDataRowState()) { + ClientProtos.MutationProto nextProto = + ClientProtos.MutationProto.parseFrom(dataRowStates.getNextDataRowState().toByteArray()); + Mutation nextMutation = ProtobufUtil.toMutation(nextProto); + if (nextMutation instanceof Put) { + nextDataRowState = (Put) nextMutation; + } + } + if (currentDataRowState == null && nextDataRowState == null) { + continue; + } + IndexRegionObserver.generateIndexMutationsForRow(rowKeyPtr, currentDataRowState, + nextDataRowState, ts, encodedRegionNameBytes, QueryConstants.VERIFIED_BYTES, indexTables, + indexUpdates); + if (indexUpdates.size() >= batchSize) { + metricSource.updateCdcMutationGenerateTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - generateStartTime); + long applyStartTime = EnvironmentEdgeManager.currentTimeMillis(); + indexWriter.write(indexUpdates, false, MetaDataProtocol.PHOENIX_VERSION); + metricSource.updateCdcMutationApplyTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - applyStartTime); + totalMutations += indexUpdates.size(); + indexUpdates.clear(); + generateStartTime = EnvironmentEdgeManager.currentTimeMillis(); + } + } + if (!indexUpdates.isEmpty()) { + metricSource.updateCdcMutationGenerateTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - generateStartTime); + long applyStartTime = EnvironmentEdgeManager.currentTimeMillis(); + indexWriter.write(indexUpdates, false, MetaDataProtocol.PHOENIX_VERSION); + metricSource.updateCdcMutationApplyTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - applyStartTime); + totalMutations += indexUpdates.size(); + } + if (totalMutations > 0) { + metricSource.incrementCdcMutationCount(dataTableName, totalMutations); + LOG.debug( + "Applied total {} index mutations for table {} partition {} owner {} " + + ", last processed timestamp {}", + totalMutations, dataTableName, partitionId, ownerPartitionId, lastProcessedTimestamp); + } + } + + private void executeIndexMutations(String partitionId, + List> batchMutations, String ownerPartitionId, + long lastProcessedTimestamp) throws SQLException, IOException { + if (!batchMutations.isEmpty()) { + ListMultimap indexUpdates = ArrayListMultimap.create(); + Map tableRefCache = new HashMap<>(); + int totalMutations = 0; + for (Pair batchMutation : batchMutations) { + IndexMutationsProtos.IndexMutations mutationsProto = batchMutation.getSecond(); + List tables = mutationsProto.getTablesList(); + List mutations = mutationsProto.getMutationsList(); + if (tables.size() != mutations.size()) { + throw new SQLException("Tables and mutations sizes do not match. Tables size: " + + tables.size() + ", mutations size: " + mutations.size()); + } + for (int i = 0; i < tables.size(); i++) { + byte[] indexTableName = tables.get(i).toByteArray(); + byte[] mutationBytes = mutations.get(i).toByteArray(); + ImmutableBytesPtr tableNamePtr = new ImmutableBytesPtr(indexTableName); + HTableInterfaceReference tableRef = + tableRefCache.computeIfAbsent(tableNamePtr, HTableInterfaceReference::new); + ClientProtos.MutationProto mProto = ClientProtos.MutationProto.parseFrom(mutationBytes); + Mutation mutation = ProtobufUtil.toMutation(mProto); + indexUpdates.put(tableRef, mutation); + } + if (indexUpdates.size() >= batchSize) { + long applyStartTime = EnvironmentEdgeManager.currentTimeMillis(); + indexWriter.write(indexUpdates, false, MetaDataProtocol.PHOENIX_VERSION); + metricSource.updateCdcMutationApplyTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - applyStartTime); + totalMutations += indexUpdates.size(); + indexUpdates.clear(); + } + } + if (!indexUpdates.isEmpty()) { + long applyStartTime = EnvironmentEdgeManager.currentTimeMillis(); + indexWriter.write(indexUpdates, false, MetaDataProtocol.PHOENIX_VERSION); + metricSource.updateCdcMutationApplyTime(dataTableName, + EnvironmentEdgeManager.currentTimeMillis() - applyStartTime); + totalMutations += indexUpdates.size(); + } + if (totalMutations > 0) { + metricSource.incrementCdcMutationCount(dataTableName, totalMutations); + LOG.debug( + "Applied total {} index mutations for table {} partition {} owner {} " + + ", last processed timestamp {}", + totalMutations, dataTableName, partitionId, ownerPartitionId, lastProcessedTimestamp); + } + } + } + + private void updateTrackerProgress(PhoenixConnection conn, String partitionId, + String ownerPartitionId, long lastTimestamp, String status) throws SQLException { + String upsertSql = "UPSERT INTO " + PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_NAME + + " (TABLE_NAME, PARTITION_ID, OWNER_PARTITION_ID, LAST_TIMESTAMP, STATUS) " + + "VALUES (?, ?, ?, ?, ?)"; + try (PreparedStatement ps = conn.prepareStatement(upsertSql)) { + ps.setString(1, dataTableName); + ps.setString(2, partitionId); + ps.setString(3, ownerPartitionId); + ps.setLong(4, lastTimestamp); + ps.setString(5, status); + ps.executeUpdate(); + conn.commit(); + LOG.debug("Updated tracker for table {} partition {} owner {} to timestamp {} with status {}", + dataTableName, partitionId, ownerPartitionId, lastTimestamp, status); + } + } + + /** + * Executes the prepared statement and extracts mutations and timestamps from the result set. + * @param ps the prepared statement to execute. + * @param initialLastTimestamp the initial last timestamp to use if no rows are found. + * @param batchMutations list to add mutations to. + * @return Pair of last processed timestamp and boolean to indicate if any rows were returned. + */ + private static Pair getMutationsAndTimestamp(PreparedStatement ps, + long initialLastTimestamp, List> batchMutations) + throws SQLException, IOException { + boolean hasRows = false; + long lastTimestamp = initialLastTimestamp; + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + hasRows = true; + lastTimestamp = rs.getDate(1).getTime(); + String cdcValue = rs.getString(2); + if (cdcValue != null && !cdcValue.isEmpty()) { + byte[] protoBytes = Base64.getDecoder().decode(cdcValue); + IndexMutationsProtos.IndexMutations mutationsProto = + IndexMutationsProtos.IndexMutations.parseFrom(protoBytes); + if (mutationsProto.getTablesCount() > 0 && mutationsProto.getMutationsCount() > 0) { + batchMutations.add(Pair.newPair(lastTimestamp, mutationsProto)); + } + } + } + } + return Pair.newPair(lastTimestamp, hasRows); + } + +} diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java index fc55b2435dc..9763388effb 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java @@ -27,6 +27,7 @@ import static org.apache.phoenix.index.PhoenixIndexBuilderHelper.RETURN_RESULT; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; +import com.google.protobuf.ByteString; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.EOFException; @@ -84,6 +85,7 @@ import org.apache.htrace.TraceScope; import org.apache.phoenix.compile.ScanRanges; import org.apache.phoenix.coprocessor.DelegateRegionCoprocessorEnvironment; +import org.apache.phoenix.coprocessor.generated.IndexMutationsProtos; import org.apache.phoenix.coprocessor.generated.PTableProtos; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.exception.DataExceedsCapacityException; @@ -143,6 +145,7 @@ import org.apache.phoenix.util.ServerUtil.ConnectionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xerial.snappy.Snappy; import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; import org.apache.phoenix.thirdparty.com.google.common.collect.ArrayListMultimap; @@ -151,6 +154,9 @@ import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; import org.apache.phoenix.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; + /** * Do all the work of managing index updates from a single coprocessor. All Puts/Delets are passed * to an {@link IndexBuilder} to determine the actual updates to make. We don't need to implement @@ -166,6 +172,64 @@ public class IndexRegionObserver implements RegionCoprocessor, RegionObserver { private static final OperationStatus NOWRITE = new OperationStatus(SUCCESS); public static final String PHOENIX_APPEND_METADATA_TO_WAL = "phoenix.append.metadata.to.wal"; public static final boolean DEFAULT_PHOENIX_APPEND_METADATA_TO_WAL = false; + public static final String PHOENIX_INDEX_CDC_CONSUMER_ENABLED = + "phoenix.index.cdc.consumer.enabled"; + public static final boolean DEFAULT_PHOENIX_INDEX_CDC_CONSUMER_ENABLED = true; + public static final String PHOENIX_INDEX_CDC_MUTATIONS_COMPRESS_ENABLED = + "phoenix.index.cdc.mutations.compress.enabled"; + public static final boolean DEFAULT_PHOENIX_INDEX_CDC_MUTATIONS_COMPRESS_ENABLED = false; + /** + * Controls which approach is used for implementing eventually consistent global secondary indexes + * via the {@link IndexCDCConsumer}. + *

+ * Approach 1: Serialized mutations (value = true) + *

+ *

+ * During {@code preBatchMutate}, {@link IndexRegionObserver} generates index mutations for each + * data table mutation and serializes them into a Protobuf {@code IndexMutations} message. This + * serialized payload is written as a column value in the CDC index table row alongside the CDC + * event. The {@link IndexCDCConsumer} later reads these pre-computed mutations from the CDC + * index, deserializes them, and applies them directly to the index table(s). In this approach, + * the consumer does not need to understand index structure or re-derive mutations — it simply + * replays what was already computed on the write path. The trade-off is increased CDC index row + * size due to the serialized mutation payload, and additional write IO on the CDC index table. + *

+ *

+ * Approach 2: Generated mutations from data row states (default, value = false) + *

+ *

+ * During {@code preBatchMutate}, {@link IndexRegionObserver} writes only a lightweight CDC index + * entry without serialized index mutations. Instead, the CDC event is created with the + * {@code DATA_ROW_STATE} scope. When the {@link IndexCDCConsumer} processes these events, it + * reads the CDC index rows which trigger a server-side scan of the data table (via + * {@code CDCGlobalIndexRegionScanner}) to reconstruct the before-image + * ({@code currentDataRowState}) and after-image ({@code nextDataRowState}) of the data row at the + * change timestamp. These raw row states are returned as a Protobuf {@code DataRowStates} + * message. The consumer then feeds these states into {@code generateIndexMutationsForRow()} — the + * same core utility used by {@link IndexRegionObserver#prepareIndexMutations} on the write path — + * to derive index mutations at consume time. This approach keeps CDC index rows small, avoids + * additional write IO, and generates mutations based on the current index definition, but + * requires an additional data table read per CDC event and is sensitive to data visibility + * timing. Make sure max lookback age is long enough to retain before and after images of the row. + *

+ *

+ * When to use which approach: + *

+ *
    + *
  • Use Approach 2 (serialize = false, default) to minimize write IO: no serialized + * mutations are written to the CDC index, keeping CDC index rows small and write latency uniform. + * The trade-off is higher read IO at consume time — the consumer performs an additional data + * table point-lookup with a raw scan per CDC event to reconstruct row states.
    • + *
  • Use Approach 1 (serialize = true) to minimize read IO: the consumer reads + * pre-computed mutations from the CDC index and applies them directly, with no data table scan + * required at consume time. The trade-off is higher write IO — serialized index mutations are + * written alongside each CDC index entry, increasing CDC index row size and write-path latency. + * Although CDC index is expected to have TTL same as the data table max lookback age.
    • + *
+ */ + public static final String PHOENIX_INDEX_CDC_MUTATION_SERIALIZE = + "phoenix.index.cdc.mutation.serialize"; + public static final boolean DEFAULT_PHOENIX_INDEX_CDC_MUTATION_SERIALIZE = false; /** * Class to represent pending data table rows @@ -183,15 +247,16 @@ private class PendingRow { this.rowKey = rowKey; } - public boolean add(BatchMutateContext context) { + public BatchMutateContext addAndGetPrevCtx(BatchMutateContext context) { synchronized (this) { if (usable) { + BatchMutateContext previousContext = lastContext; count++; lastContext = context; - return true; + return previousContext; } } - return false; + return null; } public void remove() { @@ -208,9 +273,6 @@ public int getCount() { return count; } - public BatchMutateContext getLastContext() { - return lastContext; - } } private static boolean ignoreIndexRebuildForTesting = false; @@ -274,6 +336,15 @@ public static class BatchMutateContext { // The collection of candidate index mutations that will be applied after the data table // mutations. private ListMultimap> indexUpdates; + // Map of data table row key to IndexMutations bytes + // containing pre-index mutations for eventually consistent indexes + // (index mutations with UNVERIFIED Puts only, no Deletes) + private Map cdcPreMutationsBytes; + // Map of data table row key to IndexMutations bytes + // containing post-index mutations for eventually consistent indexes + // (index mutations with VERIFIED Puts for covered, + // no Put mutations for uncovered, and Deletes if needed) + private Map cdcPostMutationsBytes; private List rowLocks = Lists.newArrayListWithExpectedSize(QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE); // TreeSet to improve locking efficiency and avoid deadlock (PHOENIX-6871 and HBASE-17924) @@ -410,10 +481,14 @@ public int getMaxPendingRowCount() { private int concurrentMutationWaitDuration; private String dataTableName; private boolean shouldWALAppend = DEFAULT_PHOENIX_APPEND_METADATA_TO_WAL; + private boolean indexCDCConsumerEnabled = DEFAULT_PHOENIX_INDEX_CDC_CONSUMER_ENABLED; + private boolean compressCDCMutations = DEFAULT_PHOENIX_INDEX_CDC_MUTATIONS_COMPRESS_ENABLED; + private boolean serializeCDCMutations = DEFAULT_PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; private boolean isNamespaceEnabled = false; private boolean useBloomFilter = false; private long lastTimestamp = 0; private List> batchesWithLastTimestamp = new ArrayList<>(); + private IndexCDCConsumer indexCDCConsumer; private static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000; private static final int DEFAULT_CONCURRENT_MUTATION_WAIT_DURATION_IN_MS = 100; private byte[] encodedRegionName; @@ -465,12 +540,27 @@ public void start(CoprocessorEnvironment e) throws IOException { this.dataTableName = env.getRegionInfo().getTable().getNameAsString(); this.shouldWALAppend = env.getConfiguration().getBoolean(PHOENIX_APPEND_METADATA_TO_WAL, DEFAULT_PHOENIX_APPEND_METADATA_TO_WAL); + this.indexCDCConsumerEnabled = env.getConfiguration() + .getBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, DEFAULT_PHOENIX_INDEX_CDC_CONSUMER_ENABLED); + this.compressCDCMutations = + env.getConfiguration().getBoolean(PHOENIX_INDEX_CDC_MUTATIONS_COMPRESS_ENABLED, + DEFAULT_PHOENIX_INDEX_CDC_MUTATIONS_COMPRESS_ENABLED); + this.serializeCDCMutations = env.getConfiguration().getBoolean( + PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, DEFAULT_PHOENIX_INDEX_CDC_MUTATION_SERIALIZE); this.isNamespaceEnabled = SchemaUtil.isNamespaceMappingEnabled(PTableType.INDEX, env.getConfiguration()); TableDescriptor tableDescriptor = env.getRegion().getTableDescriptor(); BloomType bloomFilterType = tableDescriptor.getColumnFamilies()[0].getBloomFilterType(); // when the table descriptor changes, the coproc is reloaded this.useBloomFilter = bloomFilterType == BloomType.ROW; + if ( + this.indexCDCConsumerEnabled && !this.dataTableName.startsWith("SYSTEM.") + && !this.dataTableName.startsWith("SYSTEM:") + ) { + this.indexCDCConsumer = + new IndexCDCConsumer(env, this.dataTableName, serverName, this.serializeCDCMutations); + this.indexCDCConsumer.start(); + } } catch (NoSuchMethodError ex) { disabled = true; LOG.error("Must be too early a version of HBase. Disabled coprocessor ", ex); @@ -507,6 +597,9 @@ public void stop(CoprocessorEnvironment e) throws IOException { this.builder.stop(msg); this.preWriter.stop(msg); this.postWriter.stop(msg); + if (this.indexCDCConsumer != null) { + this.indexCDCConsumer.stop(); + } } /** @@ -1082,8 +1175,8 @@ private void getCurrentRowStates(ObserverContext c keys.add(PVarbinary.INSTANCE.getKeyRange(rowKeyPtr.get(), SortOrder.ASC)); } else { // There is a pending row for this row key. We need to retrieve the row from memory - BatchMutateContext lastContext = existingPendingRow.getLastContext(); - if (existingPendingRow.add(context)) { + BatchMutateContext lastContext = existingPendingRow.addAndGetPrevCtx(context); + if (lastContext != null) { BatchMutatePhase phase = lastContext.getCurrentPhase(); Preconditions.checkArgument(phase != BatchMutatePhase.POST, "the phase of the last batch cannot be POST"); @@ -1169,6 +1262,76 @@ public static Mutation getDeleteIndexMutation(Put dataRowState, IndexMaintainer IndexMaintainer.DeleteType.ALL_VERSIONS, ts); } + public static void generateIndexMutationsForRow(ImmutableBytesPtr rowKeyPtr, + Put currentDataRowState, Put nextDataRowState, long ts, byte[] encodedRegionName, + byte[] emptyColumnValue, List> indexTables, + ListMultimap indexUpdates) throws IOException { + for (Pair pair : indexTables) { + IndexMaintainer indexMaintainer = pair.getFirst(); + HTableInterfaceReference hTableInterfaceReference = pair.getSecond(); + if ( + nextDataRowState != null && indexMaintainer.shouldPrepareIndexMutations(nextDataRowState) + ) { + ValueGetter nextDataRowVG = new IndexUtil.SimpleValueGetter(nextDataRowState); + Put indexPut = indexMaintainer.buildUpdateMutation(GenericKeyValueBuilder.INSTANCE, + nextDataRowVG, rowKeyPtr, ts, null, null, false, encodedRegionName); + if (indexPut == null) { + // No covered column. Just prepare an index row with the empty column + byte[] indexRowKey = indexMaintainer.buildRowKey(nextDataRowVG, rowKeyPtr, null, null, ts, + encodedRegionName); + indexPut = new Put(indexRowKey); + } else { + IndexUtil.removeEmptyColumn(indexPut, + indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(), + indexMaintainer.getEmptyKeyValueQualifier()); + } + byte[] finalEmptyColumnValue = + indexMaintainer.isUncovered() ? QueryConstants.UNVERIFIED_BYTES : emptyColumnValue; + indexPut.addColumn(indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(), + indexMaintainer.getEmptyKeyValueQualifier(), ts, finalEmptyColumnValue); + indexUpdates.put(hTableInterfaceReference, indexPut); + if (!ignoreWritingDeleteColumnsToIndex) { + Delete deleteColumn = indexMaintainer.buildDeleteColumnMutation(indexPut, ts); + if (deleteColumn != null) { + indexUpdates.put(hTableInterfaceReference, deleteColumn); + } + } + // Delete the current index row if the new index key is different from the + // current one and the index is not a CDC index + if (currentDataRowState != null) { + ValueGetter currentDataRowVG = new IndexUtil.SimpleValueGetter(currentDataRowState); + byte[] indexRowKeyForCurrentDataRow = indexMaintainer.buildRowKey(currentDataRowVG, + rowKeyPtr, null, null, ts, encodedRegionName); + if ( + !indexMaintainer.isCDCIndex() + && Bytes.compareTo(indexPut.getRow(), indexRowKeyForCurrentDataRow) != 0 + ) { + Mutation del = indexMaintainer.buildRowDeleteMutation(indexRowKeyForCurrentDataRow, + IndexMaintainer.DeleteType.ALL_VERSIONS, ts); + indexUpdates.put(hTableInterfaceReference, del); + } + } + } else if ( + currentDataRowState != null + && indexMaintainer.shouldPrepareIndexMutations(currentDataRowState) + ) { + if (indexMaintainer.isCDCIndex()) { + // CDC Index needs two a delete marker for referencing the data table + // delete mutation with the right index row key, that is, the index row key + // starting with ts + Put cdcDataRowState = new Put(currentDataRowState.getRow()); + cdcDataRowState.addColumn(indexMaintainer.getDataEmptyKeyValueCF(), + indexMaintainer.getEmptyKeyValueQualifierForDataTable(), ts, ByteUtil.EMPTY_BYTE_ARRAY); + indexUpdates.put(hTableInterfaceReference, getDeleteIndexMutation(cdcDataRowState, + indexMaintainer, ts, rowKeyPtr, encodedRegionName)); + } else { + indexUpdates.put(hTableInterfaceReference, getDeleteIndexMutation(currentDataRowState, + indexMaintainer, ts, rowKeyPtr, encodedRegionName)); + } + } + } + } + /** * Generate the index update for a data row from the mutation that are obtained by merging the * previous data row state with the pending row mutation. @@ -1181,6 +1344,12 @@ private void prepareIndexMutations(BatchMutateContext context, List(indexMaintainer, hTableInterfaceReference)); @@ -1193,73 +1362,12 @@ private void prepareIndexMutations(BatchMutateContext context, List pair : indexTables) { - IndexMaintainer indexMaintainer = pair.getFirst(); - HTableInterfaceReference hTableInterfaceReference = pair.getSecond(); - if ( - nextDataRowState != null && indexMaintainer.shouldPrepareIndexMutations(nextDataRowState) - ) { - ValueGetter nextDataRowVG = new IndexUtil.SimpleValueGetter(nextDataRowState); - Put indexPut = indexMaintainer.buildUpdateMutation(GenericKeyValueBuilder.INSTANCE, - nextDataRowVG, rowKeyPtr, ts, null, null, false, encodedRegionName); - if (indexPut == null) { - // No covered column. Just prepare an index row with the empty column - byte[] indexRowKey = indexMaintainer.buildRowKey(nextDataRowVG, rowKeyPtr, null, null, - ts, encodedRegionName); - indexPut = new Put(indexRowKey); - } else { - IndexUtil.removeEmptyColumn(indexPut, - indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(), - indexMaintainer.getEmptyKeyValueQualifier()); - } - indexPut.addColumn(indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(), - indexMaintainer.getEmptyKeyValueQualifier(), ts, QueryConstants.UNVERIFIED_BYTES); - context.indexUpdates.put(hTableInterfaceReference, - new Pair(indexPut, rowKeyPtr.get())); - if (!ignoreWritingDeleteColumnsToIndex) { - Delete deleteColumn = indexMaintainer.buildDeleteColumnMutation(indexPut, ts); - if (deleteColumn != null) { - context.indexUpdates.put(hTableInterfaceReference, - new Pair(deleteColumn, rowKeyPtr.get())); - } - } - // Delete the current index row if the new index key is different from the - // current one and the index is not a CDC index - if (currentDataRowState != null) { - ValueGetter currentDataRowVG = new IndexUtil.SimpleValueGetter(currentDataRowState); - byte[] indexRowKeyForCurrentDataRow = indexMaintainer.buildRowKey(currentDataRowVG, - rowKeyPtr, null, null, ts, encodedRegionName); - if ( - !indexMaintainer.isCDCIndex() - && Bytes.compareTo(indexPut.getRow(), indexRowKeyForCurrentDataRow) != 0 - ) { - Mutation del = indexMaintainer.buildRowDeleteMutation(indexRowKeyForCurrentDataRow, - IndexMaintainer.DeleteType.ALL_VERSIONS, ts); - context.indexUpdates.put(hTableInterfaceReference, - new Pair(del, rowKeyPtr.get())); - } - } - } else if ( - currentDataRowState != null - && indexMaintainer.shouldPrepareIndexMutations(currentDataRowState) - ) { - if (indexMaintainer.isCDCIndex()) { - // CDC Index needs two a delete marker for referencing the data table - // delete mutation with the right index row key, that is, the index row key - // starting with ts - Put cdcDataRowState = new Put(currentDataRowState.getRow()); - cdcDataRowState.addColumn(indexMaintainer.getDataEmptyKeyValueCF(), - indexMaintainer.getEmptyKeyValueQualifierForDataTable(), ts, - ByteUtil.EMPTY_BYTE_ARRAY); - context.indexUpdates.put(hTableInterfaceReference, - new Pair(getDeleteIndexMutation(cdcDataRowState, indexMaintainer, - ts, rowKeyPtr, encodedRegionName), rowKeyPtr.get())); - } else { - context.indexUpdates.put(hTableInterfaceReference, - new Pair(getDeleteIndexMutation(currentDataRowState, - indexMaintainer, ts, rowKeyPtr, encodedRegionName), rowKeyPtr.get())); - } - } + ListMultimap idxUpdates = ArrayListMultimap.create(); + generateIndexMutationsForRow(rowKeyPtr, currentDataRowState, nextDataRowState, ts, + encodedRegionName, QueryConstants.UNVERIFIED_BYTES, indexTables, idxUpdates); + for (Map.Entry idxUpdate : idxUpdates.entries()) { + context.indexUpdates.put(idxUpdate.getKey(), + new Pair<>(idxUpdate.getValue(), rowKeyPtr.get())); } } } @@ -1287,9 +1395,20 @@ private void preparePreIndexMutations(BatchMutateContext context, long batchTime ArrayListMultimap.> create(); prepareIndexMutations(context, maintainers, batchTimestamp); + if (serializeCDCMutations) { + prepareEventuallyConsistentIndexMutations(context, batchTimestamp, maintainers, + compressCDCMutations); + } + context.preIndexUpdates = ArrayListMultimap. create(); int updateCount = 0; for (IndexMaintainer indexMaintainer : maintainers) { + if ( + indexMaintainer.getIndexConsistency() != null + && indexMaintainer.getIndexConsistency().isAsynchronous() + ) { + continue; + } updateCount++; byte[] emptyCF = indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(); byte[] emptyCQ = indexMaintainer.getEmptyKeyValueQualifier(); @@ -1299,6 +1418,15 @@ private void preparePreIndexMutations(BatchMutateContext context, long batchTime for (Pair update : updates) { Mutation m = update.getFirst(); if (m instanceof Put) { + if (indexMaintainer.isCDCIndex() && context.cdcPreMutationsBytes != null) { + ImmutableBytesPtr dataRowKeyPtr = new ImmutableBytesPtr(update.getSecond()); + byte[] cdcMutationsBytes = context.cdcPreMutationsBytes.get(dataRowKeyPtr); + if (cdcMutationsBytes != null) { + ((Put) m).addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, + QueryConstants.CDC_INDEX_PRE_MUTATIONS_CQ_BYTES, batchTimestamp, + cdcMutationsBytes); + } + } // This will be done before the data table row is updated (i.e., in the first write // phase) context.preIndexUpdates.put(hTableInterfaceReference, m); @@ -1309,6 +1437,15 @@ private void preparePreIndexMutations(BatchMutateContext context, long batchTime Put unverifiedPut = new Put(m.getRow()); unverifiedPut.addColumn(emptyCF, emptyCQ, batchTimestamp, QueryConstants.UNVERIFIED_BYTES); + if (indexMaintainer.isCDCIndex() && context.cdcPreMutationsBytes != null) { + ImmutableBytesPtr dataRowKeyPtr = new ImmutableBytesPtr(update.getSecond()); + byte[] cdcMutationsBytes = context.cdcPreMutationsBytes.get(dataRowKeyPtr); + if (cdcMutationsBytes != null) { + unverifiedPut.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, + QueryConstants.CDC_INDEX_PRE_MUTATIONS_CQ_BYTES, batchTimestamp, + cdcMutationsBytes); + } + } // This will be done before the data table row is updated (i.e., in the first write // phase) context.preIndexUpdates.put(hTableInterfaceReference, unverifiedPut); @@ -1319,6 +1456,113 @@ private void preparePreIndexMutations(BatchMutateContext context, long batchTime } } + /** + * Prepares pre-phase and post-phase cdc mutations for eventually consistent indexes. + * @param context batch mutate context. + * @param batchTimestamp the timestamp to use for mutations. + * @param maintainers the list of index maintainers. + * @param compressMutations whether to Snappy-compress the serialized proto bytes. + * @throws IOException if there is an error. + */ + private static void prepareEventuallyConsistentIndexMutations(BatchMutateContext context, + long batchTimestamp, List maintainers, boolean compressMutations) + throws IOException { + // Store pre-index and post-index mutations for each data table rowkey + Map preBuilderMap = + new HashMap<>(); + Map postBuilderMap = + new HashMap<>(); + + for (IndexMaintainer indexMaintainer : maintainers) { + if ( + indexMaintainer.getIndexConsistency() == null + || !indexMaintainer.getIndexConsistency().isAsynchronous() + ) { + continue; + } + byte[] emptyCF = indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(); + byte[] emptyCQ = indexMaintainer.getEmptyKeyValueQualifier(); + HTableInterfaceReference hTableInterfaceReference = + new HTableInterfaceReference(new ImmutableBytesPtr(indexMaintainer.getIndexTableName())); + List> updates = context.indexUpdates.get(hTableInterfaceReference); + for (Pair update : updates) { + Mutation m = update.getFirst(); + byte[] dataRowKey = update.getSecond(); + ImmutableBytesPtr rowKeyPtr = new ImmutableBytesPtr(dataRowKey); + IndexMutationsProtos.IndexMutations.Builder preBuilder = preBuilderMap + .computeIfAbsent(rowKeyPtr, k -> IndexMutationsProtos.IndexMutations.newBuilder()); + IndexMutationsProtos.IndexMutations.Builder postBuilder = postBuilderMap + .computeIfAbsent(rowKeyPtr, k -> IndexMutationsProtos.IndexMutations.newBuilder()); + if (m instanceof Put) { + preBuilder.addTables(ByteString.copyFrom(indexMaintainer.getIndexTableName())); + byte[] preMutation = + ProtobufUtil.toMutation(ClientProtos.MutationProto.MutationType.PUT, m).toByteArray(); + preBuilder.addMutations(ByteString.copyFrom(preMutation)); + if (!indexMaintainer.isUncovered()) { + Put verifiedPut = new Put(m.getRow()); + verifiedPut.addColumn(emptyCF, emptyCQ, batchTimestamp, QueryConstants.VERIFIED_BYTES); + postBuilder.addTables(ByteString.copyFrom(indexMaintainer.getIndexTableName())); + byte[] postMutation = ProtobufUtil + .toMutation(ClientProtos.MutationProto.MutationType.PUT, verifiedPut).toByteArray(); + postBuilder.addMutations(ByteString.copyFrom(postMutation)); + } + } else { + if (IndexUtil.isDeleteFamily(m)) { + Put unverifiedPut = new Put(m.getRow()); + unverifiedPut.addColumn(emptyCF, emptyCQ, batchTimestamp, + QueryConstants.UNVERIFIED_BYTES); + preBuilder.addTables(ByteString.copyFrom(indexMaintainer.getIndexTableName())); + byte[] preMutation = ProtobufUtil + .toMutation(ClientProtos.MutationProto.MutationType.PUT, unverifiedPut).toByteArray(); + preBuilder.addMutations(ByteString.copyFrom(preMutation)); + } + postBuilder.addTables(ByteString.copyFrom(indexMaintainer.getIndexTableName())); + byte[] deleteMutation = ProtobufUtil + .toMutation(ClientProtos.MutationProto.MutationType.DELETE, m).toByteArray(); + postBuilder.addMutations(ByteString.copyFrom(deleteMutation)); + } + } + } + + if (!preBuilderMap.isEmpty()) { + context.cdcPreMutationsBytes = new HashMap<>(); + for (Map.Entry entry : preBuilderMap.entrySet()) { + ImmutableBytesPtr rowKey = entry.getKey(); + IndexMutationsProtos.IndexMutations.Builder builder = entry.getValue(); + if (builder.getTablesCount() != builder.getMutationsCount()) { + throw new DoNotRetryIOException( + "Pre-phase tables and mutations sizes do not match for row key. Tables size: " + + builder.getTablesCount() + " , mutations size: " + builder.getMutationsCount()); + } + if (builder.getTablesCount() > 0) { + byte[] protoBytes = builder.build().toByteArray(); + context.cdcPreMutationsBytes.put(rowKey, + compressMutations ? Snappy.compress(protoBytes) : protoBytes); + } + } + } + + if (!postBuilderMap.isEmpty()) { + context.cdcPostMutationsBytes = new HashMap<>(); + for (Map.Entry entry : postBuilderMap.entrySet()) { + ImmutableBytesPtr rowKey = entry.getKey(); + IndexMutationsProtos.IndexMutations.Builder builder = entry.getValue(); + if (builder.getTablesCount() != builder.getMutationsCount()) { + throw new DoNotRetryIOException( + "Post-phase tables and mutations sizes do not match for row key. Tables size: " + + builder.getTablesCount() + " , mutations size: " + builder.getMutationsCount()); + } + if (builder.getTablesCount() > 0) { + byte[] protoBytes = builder.build().toByteArray(); + context.cdcPostMutationsBytes.put(rowKey, + compressMutations ? Snappy.compress(protoBytes) : protoBytes); + } + } + } + } + protected PhoenixIndexMetaData getPhoenixIndexMetaData( ObserverContext observerContext, MiniBatchOperationInProgress miniBatchOp) throws IOException { @@ -1337,6 +1581,12 @@ private void preparePostIndexMutations(BatchMutateContext context, long batchTim context.postIndexUpdates = ArrayListMultimap. create(); List maintainers = indexMetaData.getIndexMaintainers(); for (IndexMaintainer indexMaintainer : maintainers) { + if ( + indexMaintainer.getIndexConsistency() != null + && indexMaintainer.getIndexConsistency().isAsynchronous() + ) { + continue; + } byte[] emptyCF = indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(); byte[] emptyCQ = indexMaintainer.getEmptyKeyValueQualifier(); HTableInterfaceReference hTableInterfaceReference = @@ -1356,6 +1606,31 @@ private void preparePostIndexMutations(BatchMutateContext context, long batchTim } } } + + if (context.cdcPostMutationsBytes != null && !context.cdcPostMutationsBytes.isEmpty()) { + for (IndexMaintainer indexMaintainer : maintainers) { + if (!indexMaintainer.isCDCIndex()) { + continue; + } + HTableInterfaceReference hTableInterfaceReference = + new HTableInterfaceReference(new ImmutableBytesPtr(indexMaintainer.getIndexTableName())); + List> updates = context.indexUpdates.get(hTableInterfaceReference); + for (Pair update : updates) { + Mutation m = update.getFirst(); + if (m instanceof Put) { + ImmutableBytesPtr dataRowKeyPtr = new ImmutableBytesPtr(update.getSecond()); + byte[] cdcMutationsBytes = context.cdcPostMutationsBytes.get(dataRowKeyPtr); + if (cdcMutationsBytes != null) { + Put postPut = new Put(m.getRow()); + postPut.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, + QueryConstants.CDC_INDEX_POST_MUTATIONS_CQ_BYTES, batchTimestamp, + cdcMutationsBytes); + context.postIndexUpdates.put(hTableInterfaceReference, postPut); + } + } + } + } + } // all cleanup will be done in postBatchMutateIndispensably() } @@ -1445,7 +1720,7 @@ private void waitForPreviousConcurrentBatch(TableName table, BatchMutateContext CountDownLatch countDownLatch = lastContext.getCountDownLatch(); if (countDownLatch == null) { // phase changed from PRE to either FAILED or POST - if (phase == BatchMutatePhase.FAILED) { + if (lastContext.getCurrentPhase() == BatchMutatePhase.FAILED) { context.currentPhase = BatchMutatePhase.FAILED; break; } diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java index 69d782e078d..fcf839b7d33 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java @@ -151,11 +151,13 @@ public void write(Multimap toWrite, Set>> entries = toWrite.asMap().entrySet(); TaskBatch tasks = new TaskBatch(entries.size()); List tables = new ArrayList(entries.size()); + int totalMutations = 0; for (Entry> entry : entries) { // get the mutations for each table. We leak the implementation here a little bit to save // doing a complete copy over of all the index update for each table. final List mutations = kvBuilder.cloneIfNecessary((List) entry.getValue()); + totalMutations += mutations.size(); // track each reference so we can get at it easily later, when determing failures final HTableInterfaceReference tableReference = entry.getKey(); final RegionCoprocessorEnvironment env = this.env; @@ -249,7 +251,10 @@ private void throwFailureIfDone() throws SingleIndexWriteFailureException { Pair, List>> resultsAndFutures = null; try { - LOGGER.debug("Waiting on index update tasks to complete..."); + LOGGER.debug( + "Waiting on index update tasks to complete... " + + "tasks size {}, total mutations {}, num tables {}", + tasks.size(), totalMutations, tables.size()); resultsAndFutures = this.pool.submitUninterruptible(tasks); } catch (ExecutionException e) { throw new RuntimeException( diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java index e0d3bc1b35c..ec5992b50aa 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java @@ -105,7 +105,7 @@ public abstract class BasePermissionsIT extends BaseTest { private static final Set PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.CHILD_LINK", "SYSTEM.TRANSFORM", - "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM")); + "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER")); private static final Set PHOENIX_SYSTEM_TABLES_IDENTIFIERS = new HashSet<>(Arrays.asList("SYSTEM.\"CATALOG\"", "SYSTEM.\"SEQUENCE\"", "SYSTEM.\"STATS\"", @@ -123,7 +123,7 @@ public abstract class BasePermissionsIT extends BaseTest { static final Set PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", "SYSTEM:MUTEX", "SYSTEM:CHILD_LINK", "SYSTEM:TRANSFORM", - "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM")); + "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:IDX_CDC_TRACKER")); // Create Multiple users so that we can use Hadoop UGI to run tasks as various users // Permissions can be granted or revoke by superusers and admins only diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java index 103f0e6df5e..f35cc50f5c2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java @@ -37,6 +37,7 @@ import java.sql.Statement; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Properties; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -47,9 +48,13 @@ import org.apache.phoenix.hbase.index.metrics.GlobalIndexCheckerSource; import org.apache.phoenix.hbase.index.metrics.GlobalIndexCheckerSourceImpl; import org.apache.phoenix.hbase.index.metrics.MetricsIndexerSourceFactory; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.types.PDouble; +import org.apache.phoenix.util.CDCUtil; import org.apache.phoenix.util.PropertiesUtil; import org.bson.BsonArray; import org.bson.BsonBinary; @@ -145,6 +150,12 @@ public void testBsonValueFunction() throws Exception { conn.commit(); + PhoenixConnection pcon = conn.unwrap(PhoenixConnection.class); + PTable dataTable = pcon.getTable(new PTableKey(pcon.getTenantId(), tableName)); + List indexes = dataTable.getIndexes(); + assertEquals("Data table should have exactly 2 indexes", 2, indexes.size()); + assertFalse("Data table should not have CDC index", CDCUtil.hasCDCIndex(dataTable)); + ResultSet rs = conn.createStatement().executeQuery("SELECT count(*) FROM " + tableName); assertTrue(rs.next()); assertEquals(3, rs.getInt(1)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java index 99f4642baf6..3e6a4f19891 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -40,10 +43,15 @@ import java.util.Properties; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.ExplainPlan; +import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.types.PDouble; -import org.apache.phoenix.util.CDCUtil; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; import org.bson.BsonArray; import org.bson.BsonBinary; import org.bson.BsonDocument; @@ -52,19 +60,33 @@ import org.bson.BsonString; import org.bson.RawBsonDocument; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; /** * Tests for BSON with expression field key alias. */ -@Category(ParallelStatsDisabledTest.class) +@Category(NeedsOwnMiniClusterTest.class) public class Bson5IT extends ParallelStatsDisabledIT { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(1); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(60 * 60)); + props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2000)); + props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(5)); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.TRUE.toString()); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + private static String getJsonString(String jsonFilePath) throws IOException { URL fileUrl = Bson5IT.class.getClassLoader().getResource(jsonFilePath); Preconditions.checkArgument(fileUrl != null, "File path " + jsonFilePath + " seems invalid"); @@ -79,14 +101,24 @@ public void testBsonOpsWithSqlConditionsUpdateSuccess() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); String tableName = generateUniqueName(); String cdcName = generateUniqueName(); + String indexName1 = "IDX1_" + tableName; + String indexName2 = "IDX2_" + tableName; try (Connection conn = DriverManager.getConnection(getUrl(), props)) { String ddl = "CREATE TABLE " + tableName + " (PK1 VARCHAR NOT NULL, C1 VARCHAR, COL BSON" + " CONSTRAINT pk PRIMARY KEY(PK1))"; String cdcDdl = "CREATE CDC " + cdcName + " ON " + tableName; + String indexDdl1 = "CREATE UNCOVERED INDEX " + indexName1 + " ON " + tableName + + "(BSON_VALUE(COL, 'rather[3].outline.clock', 'VARCHAR')) " + + "WHERE BSON_VALUE(COL, 'rather[3].outline.clock', 'VARCHAR') IS NOT NULL " + + "CONSISTENCY = EVENTUAL"; + String indexDdl2 = "CREATE UNCOVERED INDEX " + indexName2 + " ON " + tableName + + "(BSON_VALUE(COL, 'result[1].location.coordinates.longitude', 'DOUBLE')) " + + "WHERE BSON_VALUE(COL, 'result[1].location.coordinates.longitude', 'DOUBLE') IS NOT NULL " + + "CONSISTENCY = EVENTUAL"; conn.createStatement().execute(ddl); conn.createStatement().execute(cdcDdl); - IndexToolIT.runIndexTool(false, "", tableName, - "\"" + CDCUtil.getCDCIndexName(cdcName) + "\""); + conn.createStatement().execute(indexDdl1); + conn.createStatement().execute(indexDdl2); Timestamp ts1 = new Timestamp(System.currentTimeMillis()); Thread.sleep(100); @@ -118,6 +150,30 @@ public void testBsonOpsWithSqlConditionsUpdateSuccess() throws Exception { Thread.sleep(100); Timestamp ts2 = new Timestamp(System.currentTimeMillis()); + PreparedStatement indexPs = conn.prepareStatement("SELECT PK1, COL FROM " + tableName + + " WHERE BSON_VALUE(COL, 'rather[3].outline.clock', 'VARCHAR') = ?"); + indexPs.setString(1, "personal"); + ResultSet indexRs = indexPs.executeQuery(); + assertFalse(indexRs.next()); + + Thread.sleep(11000); + indexRs = indexPs.executeQuery(); + assertTrue(indexRs.next()); + assertEquals("pk1010", indexRs.getString(1)); + BsonDocument actualDoc = (BsonDocument) indexRs.getObject(2); + assertEquals(bsonDocument2, actualDoc); + assertFalse(indexRs.next()); + validateExplainPlan(indexPs, indexName1, "RANGE SCAN "); + + indexPs = conn.prepareStatement("SELECT PK1 FROM " + tableName + + " WHERE BSON_VALUE(COL, 'result[1].location.coordinates.longitude', 'DOUBLE') = ?"); + indexPs.setDouble(1, 52.3736); + indexRs = indexPs.executeQuery(); + assertTrue(indexRs.next()); + assertEquals("pk1011", indexRs.getString(1)); + assertFalse(indexRs.next()); + validateExplainPlan(indexPs, indexName2, "RANGE SCAN "); + testCDCAfterFirstUpsert(conn, cdcName, ts1, ts2, bsonDocument1, bsonDocument2, bsonDocument3); ts1 = new Timestamp(System.currentTimeMillis()); @@ -312,14 +368,24 @@ public void testBsonOpsWithSqlConditionsUpdateFailure() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); String tableName = generateUniqueName(); String cdcName = generateUniqueName(); + String indexName1 = "IDX1_" + tableName; + String indexName2 = "IDX2_" + tableName; try (Connection conn = DriverManager.getConnection(getUrl(), props)) { String ddl = "CREATE TABLE " + tableName + " (PK1 VARCHAR NOT NULL, C1 VARCHAR, COL BSON" + " CONSTRAINT pk PRIMARY KEY(PK1))"; String cdcDdl = "CREATE CDC " + cdcName + " ON " + tableName; + String indexDdl1 = "CREATE INDEX " + indexName1 + " ON " + tableName + + "(BSON_VALUE(COL, 'rather[3].outline.clock', 'VARCHAR')) INCLUDE(COL) " + + "WHERE BSON_VALUE(COL, 'rather[3].outline.clock', 'VARCHAR') IS NOT NULL " + + "CONSISTENCY = EVENTUAL"; + String indexDdl2 = "CREATE UNCOVERED INDEX " + indexName2 + " ON " + tableName + + "(BSON_VALUE(COL, 'result[1].location.coordinates.longitude', 'DOUBLE')) " + + "WHERE BSON_VALUE(COL, 'result[1].location.coordinates.longitude', 'DOUBLE') IS NOT NULL " + + "CONSISTENCY = EVENTUAL"; conn.createStatement().execute(ddl); conn.createStatement().execute(cdcDdl); - IndexToolIT.runIndexTool(false, "", tableName, - "\"" + CDCUtil.getCDCIndexName(cdcName) + "\""); + conn.createStatement().execute(indexDdl1); + conn.createStatement().execute(indexDdl2); Timestamp ts1 = new Timestamp(System.currentTimeMillis()); Thread.sleep(100); @@ -352,6 +418,27 @@ public void testBsonOpsWithSqlConditionsUpdateFailure() throws Exception { Thread.sleep(100); Timestamp ts2 = new Timestamp(System.currentTimeMillis()); + Thread.sleep(11000); + PreparedStatement indexPs = conn.prepareStatement("SELECT PK1, COL FROM " + tableName + + " WHERE BSON_VALUE(COL, 'rather[3].outline.clock', 'VARCHAR') = ?"); + indexPs.setString(1, "personal"); + ResultSet indexRs = indexPs.executeQuery(); + assertTrue(indexRs.next()); + assertEquals("pk1010", indexRs.getString(1)); + BsonDocument actualDoc = (BsonDocument) indexRs.getObject(2); + assertEquals(bsonDocument2, actualDoc); + assertFalse(indexRs.next()); + validateExplainPlan(indexPs, indexName1, "RANGE SCAN "); + + indexPs = conn.prepareStatement("SELECT PK1 FROM " + tableName + + " WHERE BSON_VALUE(COL, 'result[1].location.coordinates.longitude', 'DOUBLE') = ?"); + indexPs.setDouble(1, 52.3736); + indexRs = indexPs.executeQuery(); + assertTrue(indexRs.next()); + assertEquals("pk1011", indexRs.getString(1)); + assertFalse(indexRs.next()); + validateExplainPlan(indexPs, indexName2, "RANGE SCAN "); + testCDCAfterFirstUpsert(conn, cdcName, ts1, ts2, bsonDocument1, bsonDocument2, bsonDocument3); Thread.sleep(100); @@ -654,4 +741,13 @@ private static void testCDCUpdateOneRowChange(Connection conn, String cdcName, T } } + private static void validateExplainPlan(PreparedStatement ps, String tableName, String scanType) + throws SQLException { + ExplainPlan plan = ps.unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); + ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); + assertEquals(tableName, explainPlanAttributes.getTableName()); + assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + assertEquals(scanType, explainPlanAttributes.getExplainScanType()); + } + } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredEventualGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredEventualGenerateIT.java new file mode 100644 index 00000000000..0088a38a645 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredEventualGenerateIT.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_BATCH_SIZE; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import org.apache.phoenix.coprocessor.PhoenixMasterObserver; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class ConcurrentMutationsCoveredEventualGenerateIT + extends ConcurrentMutationsExtendedIndexIT { + + private static final int MAX_LOOKBACK_AGE = 1000000; + + public ConcurrentMutationsCoveredEventualGenerateIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(10); + props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(MAX_LOOKBACK_AGE)); + props.put("hbase.rowlock.wait.duration", "100"); + props.put("phoenix.index.concurrent.wait.duration.ms", "10"); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(INDEX_CDC_CONSUMER_BATCH_SIZE, Integer.toString(4500)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(5000)); + props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(200)); + props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName()); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); + props.put(INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS, Integer.toString(1000)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList(new Object[][] { { false, true } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredEventualIT.java new file mode 100644 index 00000000000..f83da1bab10 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredEventualIT.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import java.util.Arrays; +import java.util.Collection; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +@Category(NeedsOwnMiniClusterTest.class) +public class ConcurrentMutationsCoveredEventualIT extends ConcurrentMutationsExtendedIndexIT { + + public ConcurrentMutationsCoveredEventualIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); + } + + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList(new Object[][] { { false, true } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredNonEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredNonEventualIT.java new file mode 100644 index 00000000000..7d7c2c1e5b1 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsCoveredNonEventualIT.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import java.util.Arrays; +import java.util.Collection; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +@Category(NeedsOwnMiniClusterTest.class) +public class ConcurrentMutationsCoveredNonEventualIT extends ConcurrentMutationsExtendedIndexIT { + + public ConcurrentMutationsCoveredNonEventualIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); + } + + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList(new Object[][] { { false, false } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedGenerateIT.java new file mode 100644 index 00000000000..7a5cd7fb671 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedGenerateIT.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_BATCH_SIZE; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import org.apache.phoenix.coprocessor.PhoenixMasterObserver; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class ConcurrentMutationsExtendedGenerateIT extends ConcurrentMutationsExtendedIT { + + public ConcurrentMutationsExtendedGenerateIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(10); + props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(MAX_LOOKBACK_AGE)); + props.put("hbase.rowlock.wait.duration", "100"); + props.put("phoenix.index.concurrent.wait.duration.ms", "10"); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(INDEX_CDC_CONSUMER_BATCH_SIZE, Integer.toString(4500)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2500)); + // props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(200)); + props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName()); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); + props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList(new Object[][] { { false, true }, { true, true } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java index 03ca9e1a684..4e10284a32a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java @@ -18,6 +18,10 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.end2end.IndexToolIT.verifyIndexTable; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_BATCH_SIZE; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -41,6 +45,7 @@ import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.coprocessor.PhoenixMasterObserver; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryServices; @@ -64,6 +69,7 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT { private static final Logger LOGGER = LoggerFactory.getLogger(ConcurrentMutationsExtendedIT.class); private final boolean uncovered; + private final boolean eventual; private static final Random RAND = new Random(5); private static final String MVCC_LOCK_TEST_TABLE_PREFIX = "MVCCLOCKTEST_"; private static final String LOCK_TEST_TABLE_PREFIX = "LOCKTEST_"; @@ -71,8 +77,9 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT { protected static final int MAX_LOOKBACK_AGE = 1000000; private final Object lock = new Object(); - public ConcurrentMutationsExtendedIT(boolean uncovered) { + public ConcurrentMutationsExtendedIT(boolean uncovered, boolean eventual) { this.uncovered = uncovered; + this.eventual = eventual; } @BeforeClass @@ -88,12 +95,20 @@ public static synchronized void doSetup() throws Exception { // The following sets the wait duration for the previous concurrent batch to 10 ms to test // the code path handling timeouts props.put("phoenix.index.concurrent.wait.duration.ms", "10"); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(INDEX_CDC_CONSUMER_BATCH_SIZE, Integer.toString(4500)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(1000)); + props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName()); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.TRUE.toString()); + props.put(INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS, Integer.toString(1000)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } - @Parameterized.Parameters(name = "uncovered={0}") - public static synchronized Collection data() { - return Arrays.asList(true, false); + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList( + new Object[][] { { false, false }, { false, true }, { true, false }, { true, true } }); } @Test @@ -105,7 +120,7 @@ public void testSynchronousDeletesAndUpsertValues() throws Exception { + "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 INTEGER, CONSTRAINT pk PRIMARY KEY (k1,k2)) COLUMN_ENCODED_BYTES = 0"); TestUtil.addCoprocessor(conn, tableName, DelayingRegionObserver.class); conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " - + indexName + " ON " + tableName + "(v1)"); + + indexName + " ON " + tableName + "(v1)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); final CountDownLatch doneSignal = new CountDownLatch(2); Runnable r1 = new Runnable() { @@ -167,6 +182,9 @@ public void run() { t2.start(); doneSignal.await(60, TimeUnit.SECONDS); + if (eventual) { + Thread.sleep(35000); + } verifyIndexTable(tableName, indexName, conn); } @@ -180,11 +198,11 @@ public void testConcurrentDeletesAndUpsertValues() throws Exception { + "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 INTEGER, CONSTRAINT pk PRIMARY KEY (k1,k2))"); TestUtil.addCoprocessor(conn, tableName, DelayingRegionObserver.class); conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " - + indexName + " ON " + tableName + "(v1)"); + + indexName + " ON " + tableName + "(v1)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); conn.createStatement() .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " + singleCellindexName - + " ON " + tableName - + "(v1) IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=2"); + + " ON " + tableName + "(v1)" + (eventual ? " CONSISTENCY = EVENTUAL," : "") + + " IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS," + " COLUMN_ENCODED_BYTES=2"); final CountDownLatch doneSignal = new CountDownLatch(2); Runnable r1 = new Runnable() { @@ -236,6 +254,9 @@ public void run() { t2.start(); doneSignal.await(60, TimeUnit.SECONDS); + if (eventual) { + Thread.sleep(35000); + } verifyIndexTable(tableName, indexName, conn); verifyIndexTable(tableName, singleCellindexName, conn); } @@ -252,51 +273,56 @@ public void testConcurrentUpserts() throws Exception { conn.createStatement().execute("CREATE TABLE " + tableName + "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, a.v1 INTEGER, b.v2 INTEGER, c.v3 INTEGER, d.v4 INTEGER," + "CONSTRAINT pk PRIMARY KEY (k1,k2)) COLUMN_ENCODED_BYTES = 0, VERSIONS=1"); - conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " - + indexName + " ON " + tableName + "(v1)" + (uncovered ? "" : "INCLUDE(v2, v3)")); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " + indexName + " ON " + + tableName + "(v1)" + (uncovered ? "" : " INCLUDE(v2, v3)") + + (eventual ? " CONSISTENCY = EVENTUAL" : "")); final CountDownLatch doneSignal = new CountDownLatch(nThreads); Runnable[] runnables = new Runnable[nThreads]; long startTime = EnvironmentEdgeManager.currentTimeMillis(); - for (int i = 0; i < nThreads; i++) { - runnables[i] = new Runnable() { - - @Override - public void run() { - try { - Connection conn = DriverManager.getConnection(getUrl()); - for (int i = 0; i < 10000; i++) { - conn.createStatement() - .execute("UPSERT INTO " + tableName + " VALUES (" + (i % nRows) + ", 0, " - + (RAND.nextBoolean() ? null : (RAND.nextInt() % nIndexValues)) + ", " - + (RAND.nextBoolean() ? null : RAND.nextInt()) + ", " - + (RAND.nextBoolean() ? null : RAND.nextInt()) + ", " - + (RAND.nextBoolean() ? null : RAND.nextInt()) + ")"); - if ((i % batchSize) == 0) { - conn.commit(); - } - } - conn.commit(); - } catch (SQLException e) { - LOGGER.warn("Exception during upsert : " + e); - } finally { - doneSignal.countDown(); - } - } - - }; - } + buildRunnables(nThreads, runnables, tableName, nRows, nIndexValues, batchSize, doneSignal, + 10000); for (int i = 0; i < nThreads; i++) { Thread t = new Thread(runnables[i]); t.start(); } - assertTrue("Ran out of time", doneSignal.await(120, TimeUnit.SECONDS)); LOGGER.info( "Total upsert time in ms : " + (EnvironmentEdgeManager.currentTimeMillis() - startTime)); + if (eventual) { + Thread.sleep(350000); + } long actualRowCount = verifyIndexTable(tableName, indexName, conn); assertEquals(nRows, actualRowCount); } + static void buildRunnables(int nThreads, Runnable[] runnables, String tableName, int nRows, + int nIndexValues, int batchSize, CountDownLatch doneSignal, int totalUpserts) { + for (int i = 0; i < nThreads; i++) { + runnables[i] = () -> { + try { + Connection conn = DriverManager.getConnection(getUrl()); + for (int j = 0; j < totalUpserts; j++) { + conn.createStatement() + .execute("UPSERT INTO " + tableName + " VALUES (" + (j % nRows) + ", 0, " + + (RAND.nextBoolean() ? null : (RAND.nextInt() % nIndexValues)) + ", " + + (RAND.nextBoolean() ? null : RAND.nextInt()) + ", " + + (RAND.nextBoolean() ? null : RAND.nextInt()) + ", " + + (RAND.nextBoolean() ? null : RAND.nextInt()) + ")"); + if ((j % batchSize) == 0) { + conn.commit(); + } + } + conn.commit(); + } catch (SQLException e) { + LOGGER.warn("Exception during upsert : " + e); + } finally { + doneSignal.countDown(); + } + }; + } + } + @Test public void testRowLockDuringPreBatchMutateWhenIndexed() throws Exception { final String tableName = LOCK_TEST_TABLE_PREFIX + generateUniqueName(); @@ -307,7 +333,7 @@ public void testRowLockDuringPreBatchMutateWhenIndexed() throws Exception { "CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v INTEGER) COLUMN_ENCODED_BYTES = 0"); TestUtil.addCoprocessor(conn, tableName, DelayingRegionObserver.class); conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " - + indexName + " ON " + tableName + "(v)"); + + indexName + " ON " + tableName + "(v)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); final CountDownLatch doneSignal = new CountDownLatch(2); final String[] failedMsg = new String[1]; Runnable r1 = new Runnable() { @@ -353,6 +379,9 @@ public void run() { doneSignal.await(ROW_LOCK_WAIT_TIME + 5000, TimeUnit.SECONDS); assertNull(failedMsg[0], failedMsg[0]); + if (eventual) { + Thread.sleep(35000); + } long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, tableName, indexName); assertEquals(1, actualRowCount); } @@ -365,7 +394,7 @@ public void testLockUntilMVCCAdvanced() throws Exception { conn.createStatement().execute( "CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v INTEGER) COLUMN_ENCODED_BYTES = 0"); conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " - + indexName + " ON " + tableName + "(v,k)"); + + indexName + " ON " + tableName + "(v,k)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('foo',0)"); conn.commit(); TestUtil.addCoprocessor(conn, tableName, DelayingRegionObserver.class); @@ -411,6 +440,9 @@ public void run() { t2.start(); doneSignal.await(ROW_LOCK_WAIT_TIME + 5000, TimeUnit.SECONDS); + if (eventual) { + Thread.sleep(35000); + } long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, tableName, indexName); assertEquals(1, actualRowCount); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIndexIT.java new file mode 100644 index 00000000000..35fcbf17130 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIndexIT.java @@ -0,0 +1,438 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.end2end.IndexToolIT.verifyIndexTable; +import static org.apache.phoenix.hbase.index.metrics.MetricsIndexCDCConsumerSource.CDC_BATCH_COUNT; +import static org.apache.phoenix.hbase.index.metrics.MetricsIndexCDCConsumerSource.CDC_BATCH_PROCESS_TIME; +import static org.apache.phoenix.hbase.index.metrics.MetricsIndexCDCConsumerSource.CDC_MUTATION_COUNT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import org.apache.phoenix.hbase.index.metrics.MetricsIndexCDCConsumerSourceImpl; +import org.apache.phoenix.hbase.index.metrics.MetricsIndexerSourceFactory; +import org.apache.phoenix.util.EnvironmentEdgeManager; +import org.apache.phoenix.util.TestUtil; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(Parameterized.class) +public abstract class ConcurrentMutationsExtendedIndexIT extends ParallelStatsDisabledIT { + + private static final Logger LOGGER = + LoggerFactory.getLogger(ConcurrentMutationsExtendedIndexIT.class); + + private final boolean uncovered; + private final boolean eventual; + + protected ConcurrentMutationsExtendedIndexIT(boolean uncovered, boolean eventual) { + this.uncovered = uncovered; + this.eventual = eventual; + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + ConcurrentMutationsExtendedIT.doSetup(); + } + + // This test is heavy and it might exhaust jenkins resources + @Test(timeout = 1800000) + public void testConcurrentUpsertsWithTableSplits() throws Exception { + Map metricsBefore = getCdcMetricValues(); + int nThreads = 12; + final int batchSize = 100; + final int nRows = 777; + final int nIndexValues = 23; + final int nSplits = 3; + final String schemaName = "SchemaN_01-" + generateUniqueName(); + final String tableName = "Table.Name-00_001_" + generateUniqueName(); + final String indexName1 = "IndexTable.Name-00_001_" + generateUniqueName(); + final String indexName2 = "IndexTable.Name-00_002_" + generateUniqueName(); + final String indexName3 = "IndexTable.Name-00_003_" + generateUniqueName(); + final String indexName4 = "IndexTable.Name-00_004_" + generateUniqueName(); + final String indexName5 = "IndexTable.Name-00_005_" + generateUniqueName(); + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("CREATE TABLE " + "\"" + schemaName + "\".\"" + tableName + "\"" + + "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 INTEGER, v2 INTEGER, v3 INTEGER, v4 INTEGER," + + "CONSTRAINT pk PRIMARY KEY (k1,k2)) \"phoenix.index.cdc.mutations.compress.enabled\"=true"); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " + "\"" + indexName1 + "\"" + + " ON " + "\"" + schemaName + "\".\"" + tableName + "\"" + "(v1)" + + (uncovered ? "" : " INCLUDE(v2, v3)") + + (eventual ? " CONSISTENCY = EVENTUAL" : " CONSISTENCY = STRONG")); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " + "\"" + indexName2 + "\"" + + " ON " + "\"" + schemaName + "\".\"" + tableName + "\"" + "(v2)" + + (uncovered ? "" : " INCLUDE(v1, v4)") + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " + "\"" + indexName3 + "\"" + + " ON " + "\"" + schemaName + "\".\"" + tableName + "\"" + "(v3)" + + (uncovered ? "" : " INCLUDE(v1, v2)") + + (eventual ? " CONSISTENCY = EVENTUAL" : " CONSISTENCY = STRONG")); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " + "\"" + indexName4 + "\"" + + " ON " + "\"" + schemaName + "\".\"" + tableName + "\"" + "(v4)" + + (uncovered ? "" : " INCLUDE(v2, v3)") + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + "INDEX " + "\"" + indexName5 + "\"" + + " ON " + "\"" + schemaName + "\".\"" + tableName + "\"" + "(v1, v2)" + + (uncovered ? "" : " INCLUDE(v3, v4)") + + (eventual ? " CONSISTENCY = EVENTUAL" : " CONSISTENCY = STRONG")); + + final CountDownLatch doneSignal = new CountDownLatch(nThreads); + Runnable[] runnables = new Runnable[nThreads]; + Thread.sleep(3000); + long startTime = EnvironmentEdgeManager.currentTimeMillis(); + + ConcurrentMutationsExtendedIT.buildRunnables(nThreads, runnables, + "\"" + schemaName + "\".\"" + tableName + "\"", nRows, nIndexValues, batchSize, doneSignal, + 10000); + Thread splitThread = new Thread(() -> TestUtil.splitTable(getUrl(), + schemaName + "." + tableName, nSplits, eventual ? 8000 : 2000)); + splitThread.start(); + for (int i = 0; i < nThreads; i++) { + if (i >= (nThreads - 4)) { + Thread.sleep(12000); + } + Thread t = new Thread(runnables[i]); + t.start(); + } + assertTrue("Ran out of time", doneSignal.await(350, TimeUnit.SECONDS)); + splitThread.join(10000); + LOGGER.info( + "Total upsert time in ms : " + (EnvironmentEdgeManager.currentTimeMillis() - startTime)); + Thread.sleep(20000); + List allIndexes = + new ArrayList<>(Arrays.asList(indexName1, indexName2, indexName3, indexName4, indexName5)); + verifyRandomIndexes(allIndexes, schemaName, tableName, conn, nRows); + assertCdcMetrics(metricsBefore); + } + + private void verifyRandomIndexes(List allIndexes, String schemaName, String tableName, + Connection conn, int nRows) throws Exception { + Collections.shuffle(allIndexes, ThreadLocalRandom.current()); + if (!eventual) { + verifyIndexes(allIndexes, schemaName, tableName, conn, nRows, true); + return; + } + boolean isComplete; + int tries = 0; + do { + Thread.sleep(10000); + try { + verifyIndexes(allIndexes, schemaName, tableName, conn, nRows, false); + isComplete = true; + } catch (AssertionError e) { + isComplete = false; + LOGGER.error("Index verification failed. Tries: {}", tries, e); + } finally { + dumpTableRows(conn, "SYSTEM.CDC_STREAM"); + dumpTableRows(conn, "SYSTEM.IDX_CDC_TRACKER"); + } + tries++; + } while (!isComplete); + LOGGER.info("Index verification complete. Tries: {}", tries); + } + + private static void verifyIndexes(List allIndexes, String schemaName, String tableName, + Connection conn, int nRows, boolean dumpTables) throws Exception { + LOGGER.info("Randomly selected indexes to verify: {}, {}", allIndexes.get(0), + allIndexes.get(1)); + long actualRowCount = verifyIndexTable("\"" + schemaName + "\".\"" + tableName + "\"", + "\"" + schemaName + "\".\"" + allIndexes.get(0) + "\"", conn, dumpTables); + assertEquals(nRows, actualRowCount); + actualRowCount = verifyIndexTable("\"" + schemaName + "\".\"" + tableName + "\"", + "\"" + schemaName + "\".\"" + allIndexes.get(1) + "\"", conn, dumpTables); + assertEquals(nRows, actualRowCount); + } + + // This test is heavy and it might exhaust jenkins resources + @Test(timeout = 1800000) + @Ignore("too aggressive for jenkins builds") + public void testConcurrentUpsertsWithTableSplitsMerges() throws Exception { + Assume.assumeFalse(uncovered); + Map metricsBefore = getCdcMetricValues(); + int nThreads = 13; + final int batchSize = 100; + final int nRows = 1451; + final int nIndexValues = 28; + final int nSplits = 3; + final String schemaName = "SchemaN_01-" + generateUniqueName(); + final String tableName = "Table.Name-00_001_" + generateUniqueName(); + final String indexName1 = "IndexTable.Name-00_001_" + generateUniqueName(); + final String indexName2 = "IndexTable.Name-00_002_" + generateUniqueName(); + final String indexName3 = "IndexTable.Name-00_003_" + generateUniqueName(); + final String indexName4 = "IndexTable.Name-00_004_" + generateUniqueName(); + final String indexName5 = "IndexTable.Name-00_005_" + generateUniqueName(); + final String indexName6 = "IndexTable.Name-00_006_" + generateUniqueName(); + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("CREATE TABLE " + "\"" + schemaName + "\".\"" + tableName + "\"" + + "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 INTEGER, v2 INTEGER, v3 INTEGER, v4 INTEGER," + + "CONSTRAINT pk PRIMARY KEY (k1,k2)) \"phoenix.index.cdc.mutations.compress.enabled\"=true"); + conn.createStatement() + .execute("CREATE INDEX " + "\"" + indexName1 + "\"" + " ON " + "\"" + schemaName + "\".\"" + + tableName + "\"" + "(v1)" + " INCLUDE(v2, v3)" + + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + conn.createStatement() + .execute("CREATE UNCOVERED INDEX " + "\"" + indexName2 + "\"" + " ON " + "\"" + schemaName + + "\".\"" + tableName + "\"" + "(v2)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + conn.createStatement() + .execute("CREATE INDEX " + "\"" + indexName3 + "\"" + " ON " + "\"" + schemaName + "\".\"" + + tableName + "\"" + "(v3)" + " INCLUDE(v1, v2)" + + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + conn.createStatement() + .execute("CREATE UNCOVERED INDEX " + "\"" + indexName4 + "\"" + " ON " + "\"" + schemaName + + "\".\"" + tableName + "\"" + "(v4)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + conn.createStatement() + .execute("CREATE INDEX " + "\"" + indexName5 + "\"" + " ON " + "\"" + schemaName + "\".\"" + + tableName + "\"" + "(v1, v2)" + " INCLUDE(v3, v4)" + + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + conn.createStatement() + .execute("CREATE UNCOVERED INDEX " + "\"" + indexName6 + "\"" + " ON " + "\"" + schemaName + + "\".\"" + tableName + "\"" + "(v3, v1)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + + final CountDownLatch doneSignal = new CountDownLatch(nThreads); + Runnable[] runnables = new Runnable[nThreads]; + Thread.sleep(3000); + long startTime = EnvironmentEdgeManager.currentTimeMillis(); + + ConcurrentMutationsExtendedIT.buildRunnables(nThreads, runnables, + "\"" + schemaName + "\".\"" + tableName + "\"", nRows, nIndexValues, batchSize, doneSignal, + 10000); + Thread splitThread = new Thread(() -> TestUtil.splitTable(getUrl(), + schemaName + "." + tableName, nSplits, eventual ? 3500 : 2000)); + splitThread.start(); + for (int i = 0; i < (nThreads - 5); i++) { + if (i >= (nThreads - 8)) { + Thread.sleep(9000); + } + Thread t = new Thread(runnables[i]); + t.start(); + } + splitThread.join(10000); + Thread mergeThread = + new Thread(() -> TestUtil.mergeTableRegions(getUrl(), schemaName + "." + tableName, 4, 1000)); + for (int i = nThreads - 5; i < nThreads; i++) { + Thread t = new Thread(runnables[i]); + t.start(); + if (i == nThreads - 3) { + mergeThread.start(); + } + } + mergeThread.join(10000); + assertTrue("Ran out of time", doneSignal.await(1500, TimeUnit.SECONDS)); + LOGGER.info( + "Total upsert time in ms : " + (EnvironmentEdgeManager.currentTimeMillis() - startTime)); + Thread.sleep(20000); + List allIndexes = new ArrayList<>( + Arrays.asList(indexName1, indexName2, indexName3, indexName4, indexName5, indexName6)); + verifyRandomIndexes(allIndexes, schemaName, tableName, conn, nRows); + assertCdcMetrics(metricsBefore); + } + + @Test(timeout = 1800000) + public void testConcurrentMutationsWithNonIndexedColumnUpdates() throws Exception { + Assume.assumeFalse(uncovered); + Map metricsBefore = getCdcMetricValues(); + final int nThreads = 3; + final int batchSize = 100; + final int nRows = 500; + final int nIndexValues = 23; + final String tableName = generateUniqueName(); + final String coveredIndexName = generateUniqueName(); + final String uncoveredIndexName = generateUniqueName(); + Connection conn = DriverManager.getConnection(getUrl()); + + conn.createStatement() + .execute("CREATE TABLE " + tableName + "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, " + + "v1 INTEGER, v2 INTEGER, v3 INTEGER, v4 INTEGER, v5 INTEGER, v6 INTEGER, v7 INTEGER, " + + "CONSTRAINT pk PRIMARY KEY (k1,k2))"); + + conn.createStatement().execute("CREATE INDEX " + coveredIndexName + " ON " + tableName + + "(v1) INCLUDE(v2)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + + conn.createStatement().execute("CREATE UNCOVERED INDEX " + uncoveredIndexName + " ON " + + tableName + "(v3)" + (eventual ? " CONSISTENCY = EVENTUAL" : "")); + + runMutationPhase(tableName, nThreads, 15000, batchSize, nRows, nIndexValues, false); + Thread.sleep(5000); + + runMutationPhase(tableName, nThreads, 13334, batchSize, nRows, nIndexValues, true); + Thread.sleep(5000); + + runMutationPhase(tableName, nThreads, 15000, batchSize, nRows, nIndexValues, false); + Thread.sleep(5000); + + runMutationPhase(tableName, nThreads, 15000, batchSize, nRows, nIndexValues, true); + Thread.sleep(20000); + + if (eventual) { + boolean verified = false; + for (int attempt = 0; attempt < 40 && !verified; attempt++) { + Thread.sleep(10000); + try { + long coveredCount = verifyIndexTable(tableName, coveredIndexName, conn, false); + assertEquals(nRows, coveredCount); + long uncoveredCount = verifyIndexTable(tableName, uncoveredIndexName, conn, false); + assertEquals(nRows, uncoveredCount); + verified = true; + } catch (AssertionError e) { + LOGGER.info("Verification attempt {} failed: {}", attempt, e.getMessage()); + dumpTableRows(conn, "SYSTEM.CDC_STREAM"); + dumpTableRows(conn, "SYSTEM.IDX_CDC_TRACKER"); + } + } + assertTrue("Index verification failed after retries", verified); + } else { + long coveredCount = verifyIndexTable(tableName, coveredIndexName, conn); + assertEquals(nRows, coveredCount); + long uncoveredCount = verifyIndexTable(tableName, uncoveredIndexName, conn); + assertEquals(nRows, uncoveredCount); + } + assertCdcMetrics(metricsBefore); + } + + private void runMutationPhase(String tableName, int nThreads, int mutationsPerThread, + int batchSize, int nRows, int nIndexValues, boolean includeIndexedColumns) throws Exception { + CountDownLatch doneSignal = new CountDownLatch(nThreads); + for (int t = 0; t < nThreads; t++) { + new Thread(() -> { + try { + Connection c = DriverManager.getConnection(getUrl()); + ThreadLocalRandom rand = ThreadLocalRandom.current(); + for (int j = 0; j < mutationsPerThread; j++) { + if (includeIndexedColumns) { + c.createStatement() + .execute("UPSERT INTO " + tableName + " VALUES (" + (j % nRows) + ", 0, " + + randVal(rand, nIndexValues) + ", " + randVal(rand) + ", " + + randVal(rand, nIndexValues) + ", " + randVal(rand) + ", " + randVal(rand) + ", " + + randVal(rand) + ", " + randVal(rand) + ")"); + } else { + c.createStatement() + .execute("UPSERT INTO " + tableName + " (k1, k2, v4, v5, v6, v7) VALUES (" + + (j % nRows) + ", 0, " + randVal(rand) + ", " + randVal(rand) + ", " + + randVal(rand) + ", " + randVal(rand) + ")"); + } + if ((j % batchSize) == 0) { + c.commit(); + } + } + c.commit(); + } catch (SQLException e) { + LOGGER.warn("Mutation phase exception: {}", e.getMessage()); + } finally { + doneSignal.countDown(); + } + }).start(); + } + assertTrue("Mutation phase timed out", doneSignal.await(350, TimeUnit.SECONDS)); + } + + private static String randVal(ThreadLocalRandom rand) { + return rand.nextBoolean() ? "null" : Integer.toString(rand.nextInt()); + } + + private static String randVal(ThreadLocalRandom rand, int bound) { + return rand.nextBoolean() ? "null" : Integer.toString(rand.nextInt() % bound); + } + + private Map getCdcMetricValues() { + MetricsIndexCDCConsumerSourceImpl source = + (MetricsIndexCDCConsumerSourceImpl) MetricsIndexerSourceFactory.getInstance() + .getIndexCDCConsumerSource(); + Map values = new HashMap<>(); + values.put(CDC_BATCH_COUNT, source.getMetricsRegistry().getCounter(CDC_BATCH_COUNT, 0).value()); + values.put(CDC_MUTATION_COUNT, + source.getMetricsRegistry().getCounter(CDC_MUTATION_COUNT, 0).value()); + values.put(CDC_BATCH_PROCESS_TIME, + source.getMetricsRegistry().getHistogram(CDC_BATCH_PROCESS_TIME).getCount()); + return values; + } + + private void assertCdcMetrics(Map before) { + Map after = getCdcMetricValues(); + long batchDelta = after.get(CDC_BATCH_COUNT) - before.get(CDC_BATCH_COUNT); + long mutationDelta = after.get(CDC_MUTATION_COUNT) - before.get(CDC_MUTATION_COUNT); + long histogramDelta = after.get(CDC_BATCH_PROCESS_TIME) - before.get(CDC_BATCH_PROCESS_TIME); + LOGGER.info( + "CDC metrics — before: {}, after: {}, delta — batchCount: {}, " + + "mutationCount: {}, batchProcessTime samples: {}", + before, after, batchDelta, mutationDelta, histogramDelta); + if (eventual) { + assertTrue("Expected cdcBatchCount to increase by at least 1, got " + batchDelta, + batchDelta >= 1); + assertTrue("Expected cdcMutationCount to increase by at least 1000, got " + mutationDelta, + mutationDelta >= 1000); + assertTrue("Expected cdcBatchProcessTime histogram samples to increase by at least 1, got " + + histogramDelta, histogramDelta >= 1); + } else { + assertEquals("Expected no CDC batch processing for synchronous indexes", 0, batchDelta); + assertEquals("Expected no CDC mutations for synchronous indexes", 0, mutationDelta); + assertEquals("Expected no CDC batch process time samples for synchronous indexes", 0, + histogramDelta); + } + } + + private void dumpTableRows(Connection conn, String tableName) throws SQLException { + LOGGER.info("Dumping {} table:", tableName); + try (ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + tableName)) { + ResultSetMetaData metaData = rs.getMetaData(); + int columnCount = metaData.getColumnCount(); + StringBuilder header = new StringBuilder(); + for (int i = 1; i <= columnCount; i++) { + if (i > 1) { + header.append("\t|\t"); + } + header.append(metaData.getColumnName(i)); + } + LOGGER.info("{}", header); + LOGGER.info("_____________________________________________________________________________"); + while (rs.next()) { + StringBuilder row = new StringBuilder(); + for (int i = 1; i <= columnCount; i++) { + if (i > 1) { + row.append("\t|\t"); + } + row.append(rs.getString(i)); + } + LOGGER.info("{}", row); + } + } + } + +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsLazyPostBatchWriteIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsLazyPostBatchWriteIT.java index 761c8658eb2..3b49514b99e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsLazyPostBatchWriteIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsLazyPostBatchWriteIT.java @@ -17,7 +17,12 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_BATCH_SIZE; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + import java.util.Map; +import org.apache.phoenix.coprocessor.PhoenixMasterObserver; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.query.QueryServices; @@ -34,20 +39,27 @@ @Category(NeedsOwnMiniClusterTest.class) public class ConcurrentMutationsLazyPostBatchWriteIT extends ConcurrentMutationsExtendedIT { - public ConcurrentMutationsLazyPostBatchWriteIT(boolean uncovered) { - super(uncovered); + public ConcurrentMutationsLazyPostBatchWriteIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); Assume.assumeFalse("Only covered index supports lazy post batch write mode", uncovered); } @BeforeClass public static synchronized void doSetup() throws Exception { - Map props = Maps.newHashMapWithExpectedSize(4); + Map props = Maps.newHashMapWithExpectedSize(8); props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); props.put(IndexRegionObserver.INDEX_LAZY_POST_BATCH_WRITE, "true"); props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(MAX_LOOKBACK_AGE)); props.put("hbase.rowlock.wait.duration", "100"); props.put("phoenix.index.concurrent.wait.duration.ms", "10"); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + props.put(INDEX_CDC_CONSUMER_BATCH_SIZE, Integer.toString(4500)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2500)); + props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName()); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredEventualGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredEventualGenerateIT.java new file mode 100644 index 00000000000..24511b46dbb --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredEventualGenerateIT.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_BATCH_SIZE; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import org.apache.phoenix.coprocessor.PhoenixMasterObserver; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class ConcurrentMutationsUncoveredEventualGenerateIT + extends ConcurrentMutationsExtendedIndexIT { + + private static final int MAX_LOOKBACK_AGE = 1000000; + + public ConcurrentMutationsUncoveredEventualGenerateIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(10); + props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(MAX_LOOKBACK_AGE)); + props.put("hbase.rowlock.wait.duration", "100"); + props.put("phoenix.index.concurrent.wait.duration.ms", "10"); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(INDEX_CDC_CONSUMER_BATCH_SIZE, Integer.toString(4500)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(5000)); + props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(200)); + props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName()); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); + props.put(INDEX_CDC_CONSUMER_PARENT_PROGRESS_PAUSE_MS, Integer.toString(1000)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList(new Object[][] { { true, true } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredEventualIT.java new file mode 100644 index 00000000000..9887f8c5464 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredEventualIT.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import java.util.Arrays; +import java.util.Collection; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +@Category(NeedsOwnMiniClusterTest.class) +public class ConcurrentMutationsUncoveredEventualIT extends ConcurrentMutationsExtendedIndexIT { + + public ConcurrentMutationsUncoveredEventualIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); + } + + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList(new Object[][] { { true, true } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredNonEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredNonEventualIT.java new file mode 100644 index 00000000000..70bb2ac995b --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsUncoveredNonEventualIT.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import java.util.Arrays; +import java.util.Collection; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +@Category(NeedsOwnMiniClusterTest.class) +public class ConcurrentMutationsUncoveredNonEventualIT extends ConcurrentMutationsExtendedIndexIT { + + public ConcurrentMutationsUncoveredNonEventualIT(boolean uncovered, boolean eventual) { + super(uncovered, eventual); + } + + @Parameterized.Parameters(name = "uncovered={0}, eventual={1}") + public static synchronized Collection data() { + return Arrays.asList(new Object[][] { { true, false } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexMaintainerConsistencyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexMaintainerConsistencyIT.java new file mode 100644 index 00000000000..26d231f0ea3 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexMaintainerConsistencyIT.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.List; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.phoenix.coprocessor.generated.ServerCachingProtos; +import org.apache.phoenix.index.IndexMaintainer; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.RowKeySchema; +import org.apache.phoenix.schema.types.IndexConsistency; +import org.apache.phoenix.util.CDCUtil; +import org.apache.phoenix.util.PropertiesUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * IndexMaintainer consistency tests. + */ +@Category(ParallelStatsDisabledTest.class) +public class IndexMaintainerConsistencyIT extends ParallelStatsDisabledIT { + + @After + public synchronized void afterTest() throws Exception { + boolean refCountLeaked = isAnyStoreRefCountLeaked(); + Assert.assertFalse("refCount leaked", refCountLeaked); + } + + @Test + public void testIndexMaintainerConsistencyFromPTable() throws Exception { + String dataTableName = "XYZ.\"tablENam1.1_0-001\""; + String dataTableNameWithoutQuotes = "XYZ.tablENam1.1_0-001"; + String strongIndexName1 = "\"idXNam1.12._0-001\""; + String strongIndexName2 = "\"idXNam1.12._0-002\""; + String eventualIndexName1 = "\"idXNam2.12._0-002\""; + String eventualIndexName2 = "\"idXNam3.12._0-003\""; + + try (Connection conn = + DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) { + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + conn.createStatement().execute( + "CREATE TABLE " + dataTableName + " (id INTEGER PRIMARY KEY, name VARCHAR, city VARCHAR)" + + " IS_STRICT_TTL=false, UPDATE_CACHE_FREQUENCY=2, " + + "\"phoenix.max.lookback.age.seconds\"=97200"); + conn.createStatement().execute( + "CREATE INDEX " + strongIndexName1 + " ON " + dataTableName + " (name) INCLUDE (city)"); + conn.createStatement().execute("CREATE INDEX " + strongIndexName2 + " ON " + dataTableName + + " (name) INCLUDE (city) CONSISTENCY=STRONG"); + conn.createStatement().execute("CREATE INDEX " + eventualIndexName1 + " ON " + dataTableName + + " (name) INCLUDE (city) CONSISTENCY=EVENTUAL"); + conn.createStatement().execute("CREATE UNCOVERED INDEX " + eventualIndexName2 + " ON " + + dataTableName + " (city, name) CONSISTENCY=EVENTUAL"); + PTable strongIndex1 = + pconn.getTable(new PTableKey(pconn.getTenantId(), "XYZ.idXNam1.12._0-001")); + PTable strongIndex2 = + pconn.getTable(new PTableKey(pconn.getTenantId(), "XYZ.idXNam1.12._0-002")); + PTable eventualIndex1 = + pconn.getTable(new PTableKey(pconn.getTenantId(), "XYZ.idXNam2.12._0-002")); + PTable eventualIndex2 = + pconn.getTable(new PTableKey(pconn.getTenantId(), "XYZ.idXNam3.12._0-003")); + PTable dataTable = + pconn.getTable(new PTableKey(pconn.getTenantId(), dataTableNameWithoutQuotes)); + assertNull(strongIndex1.getIndexConsistency()); + assertEquals(IndexConsistency.STRONG, strongIndex2.getIndexConsistency()); + assertEquals(IndexConsistency.EVENTUAL, eventualIndex1.getIndexConsistency()); + assertEquals(IndexConsistency.EVENTUAL, eventualIndex2.getIndexConsistency()); + assertEquals(2, strongIndex1.getUpdateCacheFrequency()); + assertEquals(2, strongIndex2.getUpdateCacheFrequency()); + assertEquals(2, eventualIndex1.getUpdateCacheFrequency()); + assertEquals(2, eventualIndex2.getUpdateCacheFrequency()); + assertEquals(2, dataTable.getUpdateCacheFrequency()); + assertFalse(eventualIndex1.isStrictTTL()); + assertFalse(eventualIndex2.isStrictTTL()); + assertFalse(strongIndex1.isStrictTTL()); + assertFalse(strongIndex2.isStrictTTL()); + assertFalse(dataTable.isStrictTTL()); + assertNull(dataTable.getIndexConsistency()); + + verifyTableDescriptor(pconn, strongIndex1, strongIndex2, eventualIndex1, eventualIndex2, + dataTable); + + IndexMaintainer strongMaintainer1 = strongIndex1.getIndexMaintainer(dataTable, pconn); + IndexMaintainer strongMaintainer2 = strongIndex2.getIndexMaintainer(dataTable, pconn); + IndexMaintainer eventualMaintainer1 = eventualIndex1.getIndexMaintainer(dataTable, pconn); + IndexMaintainer eventualMaintainer2 = eventualIndex2.getIndexMaintainer(dataTable, pconn); + + assertNotNull(strongMaintainer1); + assertNotNull(strongMaintainer2); + assertNotNull(eventualMaintainer1); + assertNotNull(eventualMaintainer2); + + verifyIndexMaintainerConsistency(strongMaintainer1, null, dataTable); + verifyIndexMaintainerConsistency(strongMaintainer2, IndexConsistency.STRONG, dataTable); + verifyIndexMaintainerConsistency(eventualMaintainer1, IndexConsistency.EVENTUAL, dataTable); + verifyIndexMaintainerConsistency(eventualMaintainer2, IndexConsistency.EVENTUAL, dataTable); + + RowKeySchema schema = RowKeySchema.EMPTY_SCHEMA; + IndexMaintainer maintainer = new IndexMaintainer(schema, false) { + }; + + assertNull(maintainer.getIndexConsistency()); + + assertTrue("Data table should have CDC index after creating eventually consistent index", + CDCUtil.hasCDCIndex(dataTable)); + assertTrue("Data table should have active CDC index", CDCUtil.hasActiveCDCIndex(dataTable)); + + String cdcObjectName = CDCUtil.getCDCObjectName(dataTable, false); + assertNotNull("CDC object name should be retrievable from data table", cdcObjectName); + + String expectedCDCName = "CDC_tablENam1.1_0-001"; + assertEquals("CDC object name should follow expected convention", expectedCDCName, + cdcObjectName); + + PTable cdcIndex = CDCUtil.getActiveCDCIndex(dataTable); + String expectedCDCIndexName = CDCUtil.getCDCIndexName(cdcObjectName); + assertEquals("CDC index name should follow expected convention", expectedCDCIndexName, + cdcIndex.getTableName().getString()); + + PTable cdcTable = pconn.getTable(new PTableKey(pconn.getTenantId(), "XYZ." + cdcObjectName)); + assertEquals("CDC table should have CDC type", PTableType.CDC, cdcTable.getType()); + + List indexes = dataTable.getIndexes(); + assertEquals("Data table should have total 5 indexes", 5, indexes.size()); + } + } + + @Test + public void testAlterIndexConsistency() throws Exception { + String dataTableName = "\"ScheMa1234-_\".\"alterTable1.1_0-001\""; + String dataTableNameWithoutQuotes = "ScheMa1234-_.alterTable1.1_0-001"; + String indexName = "\"alterIdx1.12._0-001\""; + String fullIdxNameWithoutQuotes = "ScheMa1234-_.alterIdx1.12._0-001"; + + try (Connection conn = + DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) { + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + conn.createStatement().execute( + "CREATE TABLE " + dataTableName + " (id INTEGER PRIMARY KEY, name VARCHAR, city VARCHAR)"); + conn.createStatement() + .execute("CREATE INDEX " + indexName + " ON " + dataTableName + " (name) INCLUDE (city)"); + + PTable strongIndex = + pconn.getTable(new PTableKey(pconn.getTenantId(), fullIdxNameWithoutQuotes)); + PTable dataTable = + pconn.getTable(new PTableKey(pconn.getTenantId(), dataTableNameWithoutQuotes)); + assertNull(strongIndex.getIndexConsistency()); + assertFalse("Data table should not have CDC index initially", CDCUtil.hasCDCIndex(dataTable)); + verifyTableDescriptor(pconn, strongIndex, dataTable); + + IndexMaintainer strongMaintainer = strongIndex.getIndexMaintainer(dataTable, pconn); + verifyIndexMaintainerConsistency(strongMaintainer, null, dataTable); + + conn.createStatement() + .execute("ALTER INDEX " + indexName + " ON " + dataTableName + " CONSISTENCY=EVENTUAL"); + PTable eventualIndex = + pconn.getTable(new PTableKey(pconn.getTenantId(), fullIdxNameWithoutQuotes)); + dataTable = pconn.getTable(new PTableKey(pconn.getTenantId(), dataTableNameWithoutQuotes)); + + assertEquals(IndexConsistency.EVENTUAL, eventualIndex.getIndexConsistency()); + assertTrue("Data table should have CDC index after altering to eventual consistency", + CDCUtil.hasCDCIndex(dataTable)); + verifyTableDescriptor(pconn, eventualIndex, dataTable); + + IndexMaintainer eventualMaintainer = eventualIndex.getIndexMaintainer(dataTable, pconn); + verifyIndexMaintainerConsistency(eventualMaintainer, IndexConsistency.EVENTUAL, dataTable); + conn.createStatement() + .execute("ALTER INDEX " + indexName + " ON " + dataTableName + " CONSISTENCY=STRONG"); + PTable strongIdx = + pconn.getTable(new PTableKey(pconn.getTenantId(), fullIdxNameWithoutQuotes)); + assertEquals(IndexConsistency.STRONG, strongIdx.getIndexConsistency()); + verifyTableDescriptor(pconn, strongIdx, dataTable); + IndexMaintainer strongIdxMaintainer = strongIdx.getIndexMaintainer(dataTable, pconn); + verifyIndexMaintainerConsistency(strongIdxMaintainer, IndexConsistency.STRONG, dataTable); + } + } + + private void verifyTableDescriptor(PhoenixConnection pconn, PTable... tables) throws Exception { + try (Admin admin = pconn.getQueryServices().getAdmin()) { + for (PTable table : tables) { + TableDescriptor td = + admin.getDescriptor(TableName.valueOf(table.getPhysicalName().getBytes())); + assertNull( + "CONSISTENCY should be present in table descriptor for " + table.getName().getString(), + td.getValue("CONSISTENCY")); + } + } + } + + private void verifyIndexMaintainerConsistency(IndexMaintainer maintainer, + IndexConsistency expectedConsistency, PTable dataTable) throws IOException { + assertEquals(expectedConsistency, maintainer.getIndexConsistency()); + ServerCachingProtos.IndexMaintainer proto = IndexMaintainer.toProto(maintainer); + if (expectedConsistency != null) { + assertTrue(proto.hasIndexConsistency()); + assertEquals(expectedConsistency.toString(), proto.getIndexConsistency()); + } else { + assertFalse(proto.hasIndexConsistency()); + } + IndexMaintainer deserializedMaintainer = IndexMaintainer.fromProto(proto, + dataTable.getRowKeySchema(), dataTable.getBucketNum() != null); + assertEquals(expectedConsistency, deserializedMaintainer.getIndexConsistency()); + } + +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexEventualGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexEventualGenerateIT.java new file mode 100644 index 00000000000..533427174d3 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexEventualGenerateIT.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class IndexToolForNonTxGlobalIndexEventualGenerateIT extends IndexToolForNonTxGlobalIndexIT { + + public IndexToolForNonTxGlobalIndexEventualGenerateIT(boolean mutable, boolean singleCell) { + super(mutable, singleCell); + if (indexDDLOptions.trim().isEmpty()) { + indexDDLOptions = " CONSISTENCY=EVENTUAL"; + } else { + indexDDLOptions = " CONSISTENCY=EVENTUAL," + indexDDLOptions; + } + } + + @Override + protected void waitForEventualConsistency() throws Exception { + Thread.sleep(18000); + } + + @BeforeClass + public static synchronized void setup() throws Exception { + Map serverProps = Maps.newHashMapWithExpectedSize(13); + serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20)); + serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, + Long.toString(5)); + serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, + QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8)); + serverProps.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Long.toString(MAX_LOOKBACK_AGE)); + serverProps.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + serverProps.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + serverProps.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, + Long.toString(0)); + serverProps.put("hbase.regionserver.rpc.retry.interval", Long.toString(0)); + serverProps.put("hbase.procedure.remote.dispatcher.delay.msec", Integer.toString(0)); + serverProps.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2000)); + serverProps.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(5)); + serverProps.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); + serverProps.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + Map clientProps = Maps.newHashMapWithExpectedSize(5); + clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true)); + clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5)); + clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); + clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString()); + clientProps.put(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB, + Boolean.TRUE.toString()); + destroyDriver(); + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), + new ReadOnlyProps(clientProps.entrySet().iterator())); + getUtility().getConfiguration().set(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, "1"); + } + + @Parameterized.Parameters(name = "mutable={0}, singleCellIndex={1}") + public static synchronized Collection data() { + return Arrays.asList( + new Object[][] { { true, true }, { true, false }, { false, true }, { false, false } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexEventualIT.java new file mode 100644 index 00000000000..737be134cd9 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexEventualIT.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class IndexToolForNonTxGlobalIndexEventualIT extends IndexToolForNonTxGlobalIndexIT { + + public IndexToolForNonTxGlobalIndexEventualIT(boolean mutable, boolean singleCell) { + super(mutable, singleCell); + if (indexDDLOptions.trim().isEmpty()) { + indexDDLOptions = " CONSISTENCY=EVENTUAL"; + } else { + indexDDLOptions = " CONSISTENCY=EVENTUAL," + indexDDLOptions; + } + } + + @Override + protected void waitForEventualConsistency() throws Exception { + Thread.sleep(15000); + } + + @BeforeClass + public static synchronized void setup() throws Exception { + Map serverProps = Maps.newHashMapWithExpectedSize(12); + serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20)); + serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, + Long.toString(5)); + serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, + QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8)); + serverProps.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Long.toString(MAX_LOOKBACK_AGE)); + serverProps.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + serverProps.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + serverProps.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, + Long.toString(0)); + serverProps.put("hbase.regionserver.rpc.retry.interval", Long.toString(0)); + serverProps.put("hbase.procedure.remote.dispatcher.delay.msec", Integer.toString(0)); + serverProps.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2000)); + serverProps.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(5)); + serverProps.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + serverProps.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.TRUE.toString()); + Map clientProps = Maps.newHashMapWithExpectedSize(5); + clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true)); + clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5)); + clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); + clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString()); + clientProps.put(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB, + Boolean.TRUE.toString()); + destroyDriver(); + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), + new ReadOnlyProps(clientProps.entrySet().iterator())); + getUtility().getConfiguration().set(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, "1"); + } + + @Parameterized.Parameters(name = "mutable={0}, singleCellIndex={1}") + public static synchronized Collection data() { + return Arrays.asList( + new Object[][] { { true, true }, { true, false }, { false, true }, { false, false } }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java index 6d3a47d961e..1129c78d03e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java @@ -130,11 +130,11 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseTest { public static final int MAX_LOOKBACK_AGE = 3600; - private final String tableDDLOptions; + protected final String tableDDLOptions; private final boolean useSnapshot = false; - private final boolean mutable; - private final String indexDDLOptions; + protected final boolean mutable; + protected String indexDDLOptions; private boolean singleCell; @Rule @@ -196,6 +196,9 @@ public static synchronized void setup() throws Exception { getUtility().getConfiguration().set(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, "1"); } + protected void waitForEventualConsistency() throws Exception { + } + @After public void cleanup() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -240,15 +243,18 @@ public void testWithSetNull() throws Exception { assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue()); assertTrue("Index rebuild failed!", indexTool.getJob().isSuccessful()); TestUtil.assertIndexState(conn, indexTableFullName, PIndexState.ACTIVE, null); + waitForEventualConsistency(); long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(NROWS, actualRowCount); IndexToolIT.setEveryNthRowWithNull(NROWS, 5, stmt); conn.commit(); + waitForEventualConsistency(); actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(NROWS, actualRowCount); IndexToolIT.setEveryNthRowWithNull(NROWS, 7, stmt); conn.commit(); + waitForEventualConsistency(); actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(NROWS, actualRowCount); actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); @@ -445,6 +451,7 @@ public void testBuildSecondaryIndexAndScrutinize() throws Exception { } assertEquals(0, indexTool.getJob().getCounters() .findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue()); + waitForEventualConsistency(); long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(NROWS, actualRowCount); @@ -454,6 +461,7 @@ public void testBuildSecondaryIndexAndScrutinize() throws Exception { IndexToolIT.upsertRow(stmt1, i); } conn.commit(); + waitForEventualConsistency(); indexTool = IndexToolIT.runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BOTH, new String[0]); assertEquals(2 * NROWS, @@ -492,6 +500,7 @@ public void testBuildSecondaryIndexAndScrutinize() throws Exception { .findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters() .findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue()); + waitForEventualConsistency(); actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(2 * NROWS, actualRowCount); } @@ -554,6 +563,7 @@ public void testCleanUpOldDesignIndexRows() throws Exception { .findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters() .findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue()); + waitForEventualConsistency(); long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(N_ROWS, actualRowCount); @@ -580,6 +590,7 @@ public void testCleanUpOldDesignIndexRows() throws Exception { null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]); assertEquals(0, indexTool.getJob().getCounters() .findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue()); + waitForEventualConsistency(); actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(N_ROWS, actualRowCount); } @@ -587,6 +598,8 @@ public void testCleanUpOldDesignIndexRows() throws Exception { @Test public void testIndexToolVerifyBeforeAndBothOptions() throws Exception { + Assume.assumeFalse("View indexes do not support CONSISTENCY=EVENTUAL", + indexDDLOptions.contains("CONSISTENCY=EVENTUAL")); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { String schemaName = generateUniqueName(); @@ -630,6 +643,8 @@ public void testIndexToolVerifyBeforeAndBothOptions() throws Exception { @Test public void testIndexToolVerifyAfterOption() throws Exception { + Assume.assumeFalse("View indexes do not support CONSISTENCY=EVENTUAL", + indexDDLOptions.contains("CONSISTENCY=EVENTUAL")); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { String schemaName = generateUniqueName(); @@ -841,6 +856,8 @@ public void testIndexToolForIncrementalRebuild() throws Exception { @Test public void testIndexToolForIncrementalVerify() throws Exception { + Assume.assumeFalse("View indexes do not support CONSISTENCY=EVENTUAL", + indexDDLOptions.contains("CONSISTENCY=EVENTUAL")); ManualEnvironmentEdge customEdge = new ManualEnvironmentEdge(); String schemaName = generateUniqueName(); String dataTableName = generateUniqueName(); @@ -976,6 +993,8 @@ public void testIndexToolForIncrementalVerify() throws Exception { @Test public void testIndexToolForIncrementalVerify_viewIndex() throws Exception { + Assume.assumeFalse("View indexes do not support CONSISTENCY=EVENTUAL", + indexDDLOptions.contains("CONSISTENCY=EVENTUAL")); ManualEnvironmentEdge customeEdge = new ManualEnvironmentEdge(); String schemaName = generateUniqueName(); String dataTableName = generateUniqueName(); @@ -1383,6 +1402,8 @@ public void testIncrementalRebuildWithPageSize() throws Exception { @Test public void testUpdatablePKFilterViewIndexRebuild() throws Exception { + Assume.assumeFalse("View indexes do not support CONSISTENCY=EVENTUAL", + indexDDLOptions.contains("CONSISTENCY=EVENTUAL")); if (!mutable) { return; } @@ -1452,6 +1473,8 @@ public void testUpdatablePKFilterViewIndexRebuild() throws Exception { @Test public void testUpdatableNonPkFilterViewIndexRebuild() throws Exception { + Assume.assumeFalse("View indexes do not support CONSISTENCY=EVENTUAL", + indexDDLOptions.contains("CONSISTENCY=EVENTUAL")); if (!mutable) { return; } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java index d5c12b0bbde..362520674b2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java @@ -1118,15 +1118,32 @@ public static IndexTool runIndexTool(Configuration conf, boolean useSnapshot, St public static long verifyIndexTable(String fullTableName, String fullIndexName, Connection conn) throws Exception { + return verifyIndexTable(fullTableName, fullIndexName, conn, true); + } + + public static long verifyIndexTable(String fullTableName, String fullIndexName, Connection conn, + boolean dumpTables) throws Exception { String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName); String tableName = SchemaUtil.getTableNameFromFullName(fullTableName); String indexName = SchemaUtil.getTableNameFromFullName(fullIndexName); + String fullTableNameNormalized = null; + String fullIndexNameNormalized = null; + if (schemaName != null && !schemaName.isEmpty()) { + fullTableNameNormalized = SchemaUtil.normalizeIdentifier(schemaName) + "." + + SchemaUtil.normalizeIdentifier(tableName); + fullIndexNameNormalized = SchemaUtil.normalizeIdentifier(schemaName) + "." + + SchemaUtil.normalizeIdentifier(indexName); + } try { // This checks the state of every raw index row without rebuilding any row IndexTool indexTool = IndexToolIT.runIndexTool(false, schemaName, tableName, indexName, null, 0, IndexTool.IndexVerifyType.ONLY); TestUtil.dumpTable(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME)); + if (dumpTables) { + TestUtil.dumpTable(conn, + TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME)); + } Counters counters = indexTool.getJob().getCounters(); LOGGER.info(counters.toString()); assertEquals(0, counters.findCounter(REBUILT_INDEX_ROW_COUNT).getValue()); @@ -1156,7 +1173,8 @@ public static long verifyIndexTable(String fullTableName, String fullIndexName, // The index scrutiny run will trigger index repair on all unverified rows, and they will be // repaired or // deleted (since the age threshold is set to zero ms for these tests - PTable pIndexTable = conn.unwrap(PhoenixConnection.class).getTable(fullIndexName); + PTable pIndexTable = conn.unwrap(PhoenixConnection.class) + .getTable(fullIndexNameNormalized != null ? fullIndexNameNormalized : fullIndexName); if (pIndexTable.getIndexType() != PTable.IndexType.UNCOVERED_GLOBAL) { assertEquals(0, counters.findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue()); } @@ -1195,14 +1213,19 @@ public static long verifyIndexTable(String fullTableName, String fullIndexName, counters.findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue()); assertEquals(0, counters.findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue()); - pConn.getQueryServices().clearTableRegionCache(TableName.valueOf(fullIndexName)); + pConn.getQueryServices().clearTableRegionCache(TableName + .valueOf(fullIndexNameNormalized != null ? fullIndexNameNormalized : fullIndexName)); long actualRowCountAfterCompaction = IndexScrutiny.scrutinizeIndex(conn, fullTableName, fullIndexName); assertEquals(actualRowCount, actualRowCountAfterCompaction); return actualRowCount; } catch (AssertionError e) { - TestUtil.dumpTable(conn, TableName.valueOf(fullTableName)); - TestUtil.dumpTable(conn, TableName.valueOf(fullIndexName)); + if (dumpTables) { + TestUtil.dumpTable(conn, TableName + .valueOf(fullTableNameNormalized != null ? fullTableNameNormalized : fullTableName)); + TestUtil.dumpTable(conn, TableName + .valueOf(fullIndexNameNormalized != null ? fullIndexNameNormalized : fullIndexName)); + } throw e; } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MetadataServerConnectionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MetadataServerConnectionsIT.java index ecebf38016f..bebabc9bbfd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MetadataServerConnectionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MetadataServerConnectionsIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.query.QueryServices.DISABLE_VIEW_SUBTREE_VALIDATION; import static org.apache.phoenix.query.QueryServicesTestImpl.DEFAULT_HCONNECTION_POOL_CORE_SIZE; import static org.apache.phoenix.query.QueryServicesTestImpl.DEFAULT_HCONNECTION_POOL_MAX_SIZE; @@ -81,6 +82,7 @@ public static synchronized void doSetup() throws Exception { Map props = Maps.newHashMapWithExpectedSize(1); props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); props.put(DISABLE_VIEW_SUBTREE_VALIDATION, "true"); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props)); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java index 9109e837f52..9004703ef3a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java @@ -63,14 +63,14 @@ @Category(NeedsOwnMiniClusterTest.class) public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest { - private static final Set PHOENIX_SYSTEM_TABLES = - new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", - "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", - "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM")); - private static final Set PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = - new HashSet<>(Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", - "SYSTEM:FUNCTION", "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", - "SYSTEM:TRANSFORM", "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM")); + private static final Set PHOENIX_SYSTEM_TABLES = new HashSet<>( + Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", + "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM", + "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER")); + private static final Set PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>( + Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", + "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM", + "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:IDX_CDC_TRACKER")); private static final String SCHEMA_NAME = "MIGRATETEST"; private static final String TABLE_NAME = SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiTenantEventualIndexGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiTenantEventualIndexGenerateIT.java new file mode 100644 index 00000000000..857abf5e815 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiTenantEventualIndexGenerateIT.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_BATCH_SIZE; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Map; +import org.apache.phoenix.coprocessor.PhoenixMasterObserver; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class MultiTenantEventualIndexGenerateIT extends MultiTenantEventualIndexIT { + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(12); + props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(MAX_LOOKBACK_AGE)); + props.put(INDEX_CDC_CONSUMER_BATCH_SIZE, Integer.toString(3500)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(4000)); + props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Long.toString(10)); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB, Boolean.TRUE.toString()); + props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName()); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); + props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiTenantEventualIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiTenantEventualIndexIT.java new file mode 100644 index 00000000000..e0a4ab2a4f6 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiTenantEventualIndexIT.java @@ -0,0 +1,626 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.end2end.IndexToolIT.verifyIndexTable; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_BATCH_SIZE; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.coprocessor.PhoenixMasterObserver; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.EnvironmentEdgeManager; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class MultiTenantEventualIndexIT extends ParallelStatsDisabledIT { + + private static final Logger LOG = LoggerFactory.getLogger(MultiTenantEventualIndexIT.class); + protected static final int MAX_LOOKBACK_AGE = 1000000; + private static final long WAIT_MS = 25000; + private static final int ROWS_PER_TENANT_PER_PHASE = 10; + private static final String[] TENANT_PREFIXES = { "AA_", "BB_", "CC_", "DD_", "EE_", "FF_" }; + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(12); + props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(MAX_LOOKBACK_AGE)); + props.put(INDEX_CDC_CONSUMER_BATCH_SIZE, Integer.toString(3500)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(4000)); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB, Boolean.TRUE.toString()); + props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName()); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.TRUE.toString()); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + private Connection getTenantConnection(String tenantId) throws SQLException { + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); + return DriverManager.getConnection(getUrl(), props); + } + + private Connection getGlobalConnection() throws SQLException { + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + return DriverManager.getConnection(getUrl(), props); + } + + private void waitForEventualConsistency() throws InterruptedException { + Thread.sleep(WAIT_MS); + } + + private String[] createTenants() { + String[] tenants = new String[TENANT_PREFIXES.length]; + for (int i = 0; i < TENANT_PREFIXES.length; i++) { + tenants[i] = TENANT_PREFIXES[i] + generateUniqueName(); + } + return tenants; + } + + private void insertRows(String[] tenants, String tableName, String phase, int startRow, + int endRow) throws Exception { + for (String tenant : tenants) { + try (Connection conn = getTenantConnection(tenant)) { + for (int j = startRow; j <= endRow; j++) { + conn.createStatement().execute( + String.format("UPSERT INTO %s(PK2, V1, V2) VALUES ('%s_r%d', '%s_v%d', '%s_d%d')", + tableName, phase, j, phase, j, phase, j)); + } + conn.commit(); + } + } + } + + private void updateRows(String[] tenants, String tableName, String phase, int startRow, + int endRow, String suffix) throws Exception { + for (String tenant : tenants) { + try (Connection conn = getTenantConnection(tenant)) { + for (int j = startRow; j <= endRow; j++) { + conn.createStatement() + .execute(String.format( + "UPSERT INTO %s(PK2, V1, V2) VALUES ('%s_r%d', '%s_v%d_%s', '%s_d%d_%s')", tableName, + phase, j, phase, j, suffix, phase, j, suffix)); + } + conn.commit(); + } + } + } + + private void deleteRows(String[] tenants, String tableName, String phase, int startRow, + int endRow) throws Exception { + for (String tenant : tenants) { + try (Connection conn = getTenantConnection(tenant)) { + for (int j = startRow; j <= endRow; j++) { + conn.createStatement() + .execute(String.format("DELETE FROM %s WHERE PK2 = '%s_r%d'", tableName, phase, j)); + } + conn.commit(); + } + } + } + + private void verifyRowCount(String[] tenants, String tableName, int expectedCount, String phase) + throws Exception { + for (String tenant : tenants) { + try (Connection conn = getTenantConnection(tenant)) { + ResultSet rs = conn.createStatement() + .executeQuery("SELECT COUNT(*) FROM " + tableName + " WHERE V1 IS NOT NULL"); + assertTrue(rs.next()); + assertEquals(phase + ": tenant " + tenant, expectedCount, rs.getInt(1)); + } + } + } + + private void verifyIndexLookup(String[] tenants, String tableName, String v1Value, + String expectedPk2, String expectedV2) throws Exception { + for (String tenant : tenants) { + try (Connection conn = getTenantConnection(tenant)) { + ResultSet rs = conn.createStatement() + .executeQuery("SELECT PK2, V2 FROM " + tableName + " WHERE V1 = '" + v1Value + "'"); + assertTrue("Row with V1=" + v1Value + " not found for " + tenant, rs.next()); + assertEquals(expectedPk2, rs.getString(1)); + if (expectedV2 != null) { + assertEquals(expectedV2, rs.getString(2)); + } + assertFalse(rs.next()); + } + } + } + + private void verifyNoResult(String[] tenants, String tableName, String v1Value, String message) + throws Exception { + for (String tenant : tenants) { + try (Connection conn = getTenantConnection(tenant)) { + ResultSet rs = conn.createStatement() + .executeQuery("SELECT PK2 FROM " + tableName + " WHERE V1 = '" + v1Value + "'"); + assertFalse(message + " for " + tenant, rs.next()); + } + } + } + + private int getRegionCount(Connection conn, String tableName) throws Exception { + List regions = TestUtil.getAllTableRegions(conn, tableName); + return regions.size(); + } + + @Test + public void testBasicMultiTenantEventualIndex() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + String tenantA = "TENANT_A_" + generateUniqueName(); + String tenantB = "TENANT_B_" + generateUniqueName(); + + try (Connection globalConn = getGlobalConnection()) { + globalConn.createStatement() + .execute("CREATE TABLE " + tableName + + " (TENANT_ID VARCHAR NOT NULL, PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR" + + " CONSTRAINT pk PRIMARY KEY (TENANT_ID, PK2))" + + " MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0"); + globalConn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + + "(V1) INCLUDE (V2) CONSISTENCY=EVENTUAL"); + } + + try (Connection connA = getTenantConnection(tenantA)) { + connA.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r1', 'a1', 'x1')"); + connA.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r2', 'a2', 'x2')"); + connA.commit(); + } + + try (Connection connB = getTenantConnection(tenantB)) { + connB.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r1', 'b1', 'y1')"); + connB.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r2', 'b2', 'y2')"); + connB.commit(); + } + + waitForEventualConsistency(); + + try (Connection connA = getTenantConnection(tenantA)) { + ResultSet rs = connA.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'a1'"); + assertTrue(rs.next()); + assertEquals("r1", rs.getString(1)); + assertEquals("a1", rs.getString(2)); + assertEquals("x1", rs.getString(3)); + assertFalse(rs.next()); + + rs = connA.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'a2'"); + assertTrue(rs.next()); + assertEquals("r2", rs.getString(1)); + assertEquals("a2", rs.getString(2)); + assertEquals("x2", rs.getString(3)); + assertFalse(rs.next()); + } + + try (Connection connB = getTenantConnection(tenantB)) { + ResultSet rs = connB.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'b1'"); + assertTrue(rs.next()); + assertEquals("r1", rs.getString(1)); + assertEquals("b1", rs.getString(2)); + assertEquals("y1", rs.getString(3)); + assertFalse(rs.next()); + } + + try (Connection globalConn = getGlobalConnection()) { + long rowCount = verifyIndexTable(tableName, indexName, globalConn); + assertEquals(4, rowCount); + } + } + + @Test + public void testMultiTenantDeleteAndUpsert() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + String tenantA = "TENANT_A_" + generateUniqueName(); + String tenantB = "TENANT_B_" + generateUniqueName(); + + try (Connection globalConn = getGlobalConnection()) { + globalConn.createStatement() + .execute("CREATE TABLE " + tableName + + " (TENANT_ID VARCHAR NOT NULL, PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR" + + " CONSTRAINT pk PRIMARY KEY (TENANT_ID, PK2))" + + " MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0"); + globalConn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + + "(V1) INCLUDE (V2) CONSISTENCY=EVENTUAL"); + } + + try (Connection connA = getTenantConnection(tenantA)) { + connA.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r1', 'a1', 'x1')"); + connA.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r2', 'a2', 'x2')"); + connA.commit(); + } + + try (Connection connB = getTenantConnection(tenantB)) { + connB.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r1', 'b1', 'y1')"); + connB.commit(); + } + + waitForEventualConsistency(); + + try (Connection connA = getTenantConnection(tenantA)) { + connA.createStatement().execute("DELETE FROM " + tableName + " WHERE PK2 = 'r1'"); + connA.commit(); + } + + try (Connection connB = getTenantConnection(tenantB)) { + connB.createStatement().execute( + "UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r1', 'b1_updated', 'y1_updated')"); + connB.commit(); + } + + waitForEventualConsistency(); + + try (Connection connA = getTenantConnection(tenantA)) { + ResultSet rs = connA.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'a1'"); + assertFalse("Deleted row should not be visible", rs.next()); + + rs = connA.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'a2'"); + assertTrue(rs.next()); + assertEquals("r2", rs.getString(1)); + assertFalse(rs.next()); + } + + try (Connection connB = getTenantConnection(tenantB)) { + ResultSet rs = connB.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'b1_updated'"); + assertTrue(rs.next()); + assertEquals("r1", rs.getString(1)); + assertEquals("y1_updated", rs.getString(3)); + assertFalse(rs.next()); + } + + try (Connection globalConn = getGlobalConnection()) { + long rowCount = verifyIndexTable(tableName, indexName, globalConn); + assertEquals(2, rowCount); + } + } + + @Test + public void testMultiTenantUncoveredEventualIndex() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + String tenantA = "TENANT_A_" + generateUniqueName(); + String tenantB = "TENANT_B_" + generateUniqueName(); + + try (Connection globalConn = getGlobalConnection()) { + globalConn.createStatement() + .execute("CREATE TABLE " + tableName + + " (TENANT_ID VARCHAR NOT NULL, PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR" + + " CONSTRAINT pk PRIMARY KEY (TENANT_ID, PK2))" + + " MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0"); + globalConn.createStatement().execute( + "CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + "(V1) CONSISTENCY=EVENTUAL"); + } + + try (Connection connA = getTenantConnection(tenantA)) { + connA.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r1', 'a1', 'x1')"); + connA.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r2', 'a2', 'x2')"); + connA.commit(); + } + + try (Connection connB = getTenantConnection(tenantB)) { + connB.createStatement() + .execute("UPSERT INTO " + tableName + "(PK2, V1, V2) VALUES ('r1', 'b1', 'y1')"); + connB.commit(); + } + + waitForEventualConsistency(); + + try (Connection connA = getTenantConnection(tenantA)) { + ResultSet rs = connA.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'a1'"); + assertTrue(rs.next()); + assertEquals("r1", rs.getString(1)); + assertEquals("a1", rs.getString(2)); + assertEquals("x1", rs.getString(3)); + assertFalse(rs.next()); + } + + try (Connection connB = getTenantConnection(tenantB)) { + ResultSet rs = connB.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'b1'"); + assertTrue(rs.next()); + assertEquals("r1", rs.getString(1)); + assertEquals("b1", rs.getString(2)); + assertEquals("y1", rs.getString(3)); + assertFalse(rs.next()); + } + + try (Connection connA = getTenantConnection(tenantA)) { + connA.createStatement().execute("DELETE FROM " + tableName + " WHERE PK2 = 'r1'"); + connA.commit(); + } + + waitForEventualConsistency(); + + try (Connection connA = getTenantConnection(tenantA)) { + ResultSet rs = connA.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'a1'"); + assertFalse("Deleted row should not appear for tenant A", rs.next()); + } + + try (Connection connB = getTenantConnection(tenantB)) { + ResultSet rs = connB.createStatement() + .executeQuery("SELECT PK2, V1, V2 FROM " + tableName + " WHERE V1 = 'b1'"); + assertTrue("Tenant B row should still be visible", rs.next()); + assertEquals("r1", rs.getString(1)); + assertFalse(rs.next()); + } + + try (Connection globalConn = getGlobalConnection()) { + long rowCount = verifyIndexTable(tableName, indexName, globalConn); + assertEquals(2, rowCount); + } + } + + @Test + public void testMultiTenantCoveredIndexWithSplits() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + String[] tenants = createTenants(); + + try (Connection globalConn = getGlobalConnection()) { + globalConn.createStatement() + .execute("CREATE TABLE " + tableName + + " (TENANT_ID VARCHAR NOT NULL, PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR" + + " CONSTRAINT pk PRIMARY KEY (TENANT_ID, PK2))" + + " MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0"); + globalConn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + + "(V1) INCLUDE (V2) CONSISTENCY=EVENTUAL"); + } + + // Phase 1: Insert 10 rows per tenant (60 total), single region + insertRows(tenants, tableName, "p1", 1, ROWS_PER_TENANT_PER_PHASE); + waitForEventualConsistency(); + verifyRowCount(tenants, tableName, ROWS_PER_TENANT_PER_PHASE, "Phase 1"); + verifyIndexLookup(tenants, tableName, "p1_v5", "p1_r5", "p1_d5"); + + try (Connection globalConn = getGlobalConnection()) { + TestUtil.splitTable(globalConn, tableName, Bytes.toBytes("CC")); + assertEquals("Region count after split 1", 2, getRegionCount(globalConn, tableName)); + } + + // Phase 2: Insert 10 more rows per tenant after first split + insertRows(tenants, tableName, "p2", 1, ROWS_PER_TENANT_PER_PHASE); + waitForEventualConsistency(); + verifyRowCount(tenants, tableName, 2 * ROWS_PER_TENANT_PER_PHASE, "Phase 2"); + verifyIndexLookup(tenants, tableName, "p1_v3", "p1_r3", "p1_d3"); + verifyIndexLookup(tenants, tableName, "p2_v7", "p2_r7", "p2_d7"); + + try (Connection globalConn = getGlobalConnection()) { + TestUtil.splitTable(globalConn, tableName, Bytes.toBytes("EE")); + assertEquals("Region count after split 2", 3, getRegionCount(globalConn, tableName)); + } + + insertRows(tenants, tableName, "p3", 1, 5); + updateRows(tenants, tableName, "p1", 1, 5, "upd"); + waitForEventualConsistency(); + // 10 (p1, 5 updated) + 10 (p2) + 5 (p3) = 25 unique PKs + verifyRowCount(tenants, tableName, 2 * ROWS_PER_TENANT_PER_PHASE + 5, "Phase 3"); + + verifyIndexLookup(tenants, tableName, "p1_v1_upd", "p1_r1", "p1_d1_upd"); + verifyIndexLookup(tenants, tableName, "p1_v5_upd", "p1_r5", "p1_d5_upd"); + verifyNoResult(tenants, tableName, "p1_v1", "Old p1_v1 should not exist"); + verifyNoResult(tenants, tableName, "p1_v5", "Old p1_v5 should not exist"); + verifyIndexLookup(tenants, tableName, "p1_v6", "p1_r6", "p1_d6"); + + deleteRows(tenants, tableName, "p2", 1, 2); + waitForEventualConsistency(); + verifyRowCount(tenants, tableName, 2 * ROWS_PER_TENANT_PER_PHASE + 3, "Phase 4"); + verifyNoResult(tenants, tableName, "p2_v1", "Deleted p2_r1 should not be visible via index"); + verifyNoResult(tenants, tableName, "p2_v2", "Deleted p2_r2 should not be visible via index"); + verifyIndexLookup(tenants, tableName, "p2_v3", "p2_r3", "p2_d3"); + + try (Connection globalConn = getGlobalConnection()) { + long rowCount = verifyIndexTable(tableName, indexName, globalConn); + assertEquals(23 * tenants.length, rowCount); + } + } + + @Test + public void testMultiTenantUncoveredIndexWithSplits() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + String[] tenants = createTenants(); + + try (Connection globalConn = getGlobalConnection()) { + globalConn.createStatement() + .execute("CREATE TABLE " + tableName + + " (TENANT_ID VARCHAR NOT NULL, PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR" + + " CONSTRAINT pk PRIMARY KEY (TENANT_ID, PK2))" + + " MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0"); + globalConn.createStatement().execute( + "CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + "(V1) CONSISTENCY=EVENTUAL"); + } + + insertRows(tenants, tableName, "p1", 1, ROWS_PER_TENANT_PER_PHASE); + waitForEventualConsistency(); + verifyRowCount(tenants, tableName, ROWS_PER_TENANT_PER_PHASE, "Phase 1"); + verifyIndexLookup(tenants, tableName, "p1_v5", "p1_r5", "p1_d5"); + + try (Connection globalConn = getGlobalConnection()) { + TestUtil.splitTable(globalConn, tableName, Bytes.toBytes("CC")); + assertEquals(2, getRegionCount(globalConn, tableName)); + } + + insertRows(tenants, tableName, "p2", 1, ROWS_PER_TENANT_PER_PHASE); + waitForEventualConsistency(); + verifyRowCount(tenants, tableName, 2 * ROWS_PER_TENANT_PER_PHASE, "Phase 2"); + + try (Connection globalConn = getGlobalConnection()) { + TestUtil.splitTable(globalConn, tableName, Bytes.toBytes("EE")); + assertEquals(3, getRegionCount(globalConn, tableName)); + } + + insertRows(tenants, tableName, "p3", 1, 5); + updateRows(tenants, tableName, "p1", 1, 5, "upd"); + waitForEventualConsistency(); + verifyRowCount(tenants, tableName, 2 * ROWS_PER_TENANT_PER_PHASE + 5, "Phase 3"); + verifyIndexLookup(tenants, tableName, "p1_v3_upd", "p1_r3", "p1_d3_upd"); + verifyNoResult(tenants, tableName, "p1_v3", "Old value should not exist after update"); + + deleteRows(tenants, tableName, "p2", 1, 2); + waitForEventualConsistency(); + verifyRowCount(tenants, tableName, 2 * ROWS_PER_TENANT_PER_PHASE + 3, "Phase 4"); + verifyNoResult(tenants, tableName, "p2_v1", "Deleted row should not be visible"); + verifyIndexLookup(tenants, tableName, "p2_v5", "p2_r5", "p2_d5"); + + try (Connection globalConn = getGlobalConnection()) { + long rowCount = verifyIndexTable(tableName, indexName, globalConn); + assertEquals(23 * tenants.length, rowCount); + } + } + + @Test(timeout = 1800000) + @Ignore("too aggressive for jenkins builds") + public void testConcurrentUpsertsWithTableSplits() throws Exception { + int nThreads = 8; + final int batchSize = 100; + final int nRows = 777; + final int nIndexValues = 23; + final int nSplits = 3; + final int totalUpserts = 10000; + final String tableName = generateUniqueName(); + final String indexName1 = generateUniqueName(); + final String indexName2 = generateUniqueName(); + final String indexName3 = generateUniqueName(); + final String indexName4 = generateUniqueName(); + final String indexName5 = generateUniqueName(); + final String[] tenants = createTenants(); + + try (Connection globalConn = getGlobalConnection()) { + globalConn.createStatement() + .execute("CREATE TABLE " + tableName + + "(TENANT_ID VARCHAR NOT NULL, K2 INTEGER NOT NULL, V1 INTEGER, V2 INTEGER," + + " V3 INTEGER, V4 INTEGER," + " CONSTRAINT pk PRIMARY KEY (TENANT_ID, K2))" + + " MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0"); + globalConn.createStatement().execute("CREATE INDEX " + indexName1 + " ON " + tableName + + "(V1) INCLUDE (V2, V3) CONSISTENCY=EVENTUAL"); + globalConn.createStatement().execute( + "CREATE UNCOVERED INDEX " + indexName2 + " ON " + tableName + "(V2) CONSISTENCY=EVENTUAL"); + globalConn.createStatement().execute("CREATE INDEX " + indexName3 + " ON " + tableName + + "(V3) INCLUDE (V1, V2) CONSISTENCY=EVENTUAL"); + globalConn.createStatement().execute( + "CREATE UNCOVERED INDEX " + indexName4 + " ON " + tableName + "(V4) CONSISTENCY=EVENTUAL"); + globalConn.createStatement().execute("CREATE INDEX " + indexName5 + " ON " + tableName + + "(V1, V2) INCLUDE (V3, V4) CONSISTENCY=EVENTUAL"); + } + + final CountDownLatch doneSignal = new CountDownLatch(nThreads); + Runnable[] runnables = new Runnable[nThreads]; + Thread.sleep(3000); + long startTime = EnvironmentEdgeManager.currentTimeMillis(); + + for (int i = 0; i < nThreads; i++) { + final String tenant = tenants[i % tenants.length]; + runnables[i] = () -> { + try (Connection conn = getTenantConnection(tenant)) { + ThreadLocalRandom rand = ThreadLocalRandom.current(); + for (int j = 0; j < totalUpserts; j++) { + conn.createStatement() + .execute("UPSERT INTO " + tableName + " VALUES (" + (j % nRows) + ", " + + (rand.nextBoolean() ? null : (rand.nextInt() % nIndexValues)) + ", " + + (rand.nextBoolean() ? null : rand.nextInt()) + ", " + + (rand.nextBoolean() ? null : rand.nextInt()) + ", " + + (rand.nextBoolean() ? null : rand.nextInt()) + ")"); + if ((j % batchSize) == 0) { + conn.commit(); + } + } + conn.commit(); + } catch (SQLException e) { + LOG.warn("Exception during concurrent upsert for tenant {}", tenant, e); + } finally { + doneSignal.countDown(); + } + }; + } + + Thread splitThread = new Thread(() -> TestUtil.splitTable(getUrl(), tableName, nSplits, 8000)); + splitThread.start(); + for (int i = 0; i < nThreads; i++) { + if (i >= (nThreads - 4)) { + Thread.sleep(12000); + } + Thread t = new Thread(runnables[i]); + t.start(); + } + assertTrue("Ran out of time", doneSignal.await(350, TimeUnit.SECONDS)); + splitThread.join(10000); + LOG.info("Total upsert time: {} ms", EnvironmentEdgeManager.currentTimeMillis() - startTime); + + int expectedTotal = tenants.length * nRows; + List allIndexes = + new ArrayList<>(Arrays.asList(indexName1, indexName2, indexName3, indexName4, indexName5)); + Collections.shuffle(allIndexes, ThreadLocalRandom.current()); + LOG.info("Randomly selected indexes to verify: {}, {}", allIndexes.get(0), allIndexes.get(1)); + try (Connection globalConn = getGlobalConnection()) { + Thread.sleep(500000); + long rowCount = verifyIndexTable(tableName, allIndexes.get(0), globalConn, false); + assertEquals("Index " + allIndexes.get(0), expectedTotal, rowCount); + rowCount = verifyIndexTable(tableName, allIndexes.get(1), globalConn, false); + assertEquals("Index " + allIndexes.get(1), expectedTotal, rowCount); + } + } + +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java index 4a4bd754f9f..5de095496fe 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -117,6 +118,7 @@ public static synchronized void doSetup() throws Exception { props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1)); props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS, TestScanningResultPostDummyResultCaller.class.getName()); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java index 7d839f4c16c..53adfd85543 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java @@ -189,6 +189,11 @@ public void testTableMetadataScan() throws SQLException { assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE")); assertTrue(rs.next()); assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM")); + assertEquals(PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_TABLE, + rs.getString("TABLE_NAME")); + assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE")); + assertTrue(rs.next()); + assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM")); assertEquals(PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, rs.getString("TABLE_NAME")); assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE")); assertTrue(rs.next()); @@ -425,7 +430,7 @@ public void testShowTables() throws SQLException { tables.add(rs.getString("TABLE_NAME")); assertEquals("SYSTEM", rs.getString("TABLE_SCHEM")); } - assertEquals(11, tables.size()); + assertEquals(12, tables.size()); assertTrue(tables.contains("CATALOG")); assertTrue(tables.contains("FUNCTION")); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java index 4541a510540..361a104e325 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java @@ -64,10 +64,10 @@ public class SkipSystemTablesExistenceCheckIT { private static HBaseTestingUtility hbaseTestUtil; private static Configuration conf; private Set hbaseTables; - private static final Set PHOENIX_SYSTEM_TABLES = - new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", - "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", - "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM")); + private static final Set PHOENIX_SYSTEM_TABLES = new HashSet<>( + Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", + "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM", + "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER")); private static class PhoenixSystemTablesCreationTestDriver extends PhoenixTestDriver { private static ConnectionQueryServices cqs; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java index 819feebcaa4..ac2b03276cf 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java @@ -101,15 +101,15 @@ public class SystemTablesCreationOnConnectionIT { private static final String UPSERT_STMT = "UPSERT INTO %s VALUES ('A', 'B')"; private static final String QUERY_SYSTEM_CATALOG = "SELECT * FROM SYSTEM.CATALOG LIMIT 1"; - private static final Set PHOENIX_SYSTEM_TABLES = - new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", - "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", - "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM")); - - private static final Set PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = - new HashSet<>(Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", - "SYSTEM:FUNCTION", "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", - "SYSTEM:TRANSFORM", "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM")); + private static final Set PHOENIX_SYSTEM_TABLES = new HashSet<>( + Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", + "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM", + "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER")); + + private static final Set PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>( + Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", + "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM", + "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:IDX_CDC_TRACKER")); private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java index bf396d25d9a..07ca7495754 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java @@ -578,6 +578,9 @@ public void testTableMetadataScan() throws Exception { assertTrue(rs.next()); assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, SYSTEM_FUNCTION_TABLE, SYSTEM); assertTrue(rs.next()); + assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, + PhoenixDatabaseMetaData.SYSTEM_IDX_CDC_TRACKER_TABLE, PTableType.SYSTEM); + assertTrue(rs.next()); assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, PTableType.SYSTEM); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithServerMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithServerMetadataIT.java index 6a19bec2e69..fe299294569 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithServerMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithServerMetadataIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.junit.Assert.assertEquals; import com.google.protobuf.RpcCallback; @@ -100,6 +101,7 @@ public void addServerCache(RpcController controller, AddServerCacheRequest reque public static synchronized void doSetup() throws Exception { Map props = new HashMap<>(1); props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props)); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java index 588e08b6b6b..1bf090cdae1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.apache.phoenix.util.PhoenixRuntime.UPSERT_BATCH_SIZE_ATTRIB; @@ -88,6 +89,7 @@ public static synchronized void doSetup() throws Exception { // any addition to GLOBAL_OPEN_PHOENIX_CONNECTIONS props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java index 9ad2a9bad5c..c2ffbc30a57 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.apache.phoenix.util.PhoenixRuntime.UPSERT_BATCH_SIZE_ATTRIB; import static org.apache.phoenix.util.TestUtil.A_VALUE; @@ -94,6 +95,7 @@ public static synchronized void doSetup() throws Exception { props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1)); props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS, TestScanningResultPostDummyResultCaller.class.getName()); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java index c5213df866c..6b05b7dd0a5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import java.sql.Connection; @@ -24,40 +27,80 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.Map; import java.util.Properties; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -@Category(ParallelStatsDisabledTest.class) +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) @RunWith(Parameterized.class) public class VarBinaryEncoded2IT extends ParallelStatsDisabledIT { + private static final long EVENTUAL_CONSISTENCY_WAIT_MS = 17000; + private final boolean columnEncoded; private final boolean coveredIndex; private final boolean isBindStatement; + private final boolean eventual; - public VarBinaryEncoded2IT(boolean columnEncoded, boolean coveredIndex, boolean isBindStatement) { + public VarBinaryEncoded2IT(boolean columnEncoded, boolean coveredIndex, boolean isBindStatement, + boolean eventual) { this.columnEncoded = columnEncoded; this.coveredIndex = coveredIndex; this.isBindStatement = isBindStatement; + this.eventual = eventual; } - @Parameterized.Parameters(name = "VarBinary2IT_columnEncoded={0}, coveredIndex={1}") + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(1); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(60 * 60)); + props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2000)); + props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(5)); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Parameterized.Parameters( + name = "VarBinary2IT_columnEncoded={0}, coveredIndex={1}, isBindStatement={2}, eventual={3}") public static synchronized Collection data() { - return Arrays.asList(new Object[][] { { false, false, false }, { false, true, false }, - { true, false, false }, { true, true, false }, { false, false, true }, { false, true, true }, - { true, false, true }, { true, true, true } }); + List params = new ArrayList<>(); + for (boolean columnEncoded : new boolean[] { false, true }) { + for (boolean coveredIndex : new boolean[] { false, true }) { + for (boolean isBindStatement : new boolean[] { false, true }) { + for (boolean eventual : new boolean[] { false, true }) { + params.add(new Object[] { columnEncoded, coveredIndex, isBindStatement, eventual }); + } + } + } + } + return params; + } + + private void waitForEventualConsistency() throws InterruptedException { + if (eventual) { + Thread.sleep(EVENTUAL_CONSISTENCY_WAIT_MS); + } } @Test @@ -73,12 +116,13 @@ public void testVarBinaryPkSchema1() throws Exception { + " COL3 VARBINARY_ENCODED CONSTRAINT pk PRIMARY KEY(PK1, PK2, PK3)) " + (this.columnEncoded ? "" : "COLUMN_ENCODED_BYTES=0")); + String consistencyClause = eventual ? " CONSISTENCY = EVENTUAL" : " CONSISTENCY = STRONG"; if (this.coveredIndex) { - conn.createStatement().execute( - "CREATE INDEX " + indexName + " ON " + tableName + " (COL1, COL2) INCLUDE (COL3)"); + conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + + " (COL1, COL2) INCLUDE (COL3)" + consistencyClause); } else { - conn.createStatement() - .execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + " (COL1, COL2)"); + conn.createStatement().execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + + " (COL1, COL2)" + consistencyClause); } byte[] b1 = new byte[] { 1, 1, 19, -28, 24, 1, 1, -11, -21, 1 }; @@ -152,6 +196,7 @@ public void testVarBinaryPkSchema1() throws Exception { upsertRow(preparedStatement, b16, b26, b36, b46, b56, b66); } conn.commit(); + waitForEventualConsistency(); PreparedStatement preparedStatement = conn.prepareStatement("SELECT * FROM " + tableName + " WHERE COL1 = ? AND COL2 = ?"); @@ -367,12 +412,13 @@ public void testVarBinaryPkSchema2() throws Exception { + " COL3 VARBINARY_ENCODED CONSTRAINT pk PRIMARY KEY(PK1, PK2, PK3)) " + (this.columnEncoded ? "" : "COLUMN_ENCODED_BYTES=0")); + String consistencyClause = eventual ? " CONSISTENCY = EVENTUAL" : " CONSISTENCY = STRONG"; if (this.coveredIndex) { - conn.createStatement().execute( - "CREATE INDEX " + indexName + " ON " + tableName + " (COL1, COL2) INCLUDE (COL3)"); + conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + + " (COL1, COL2) INCLUDE (COL3)" + consistencyClause); } else { - conn.createStatement() - .execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + " (COL1, COL2)"); + conn.createStatement().execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + + " (COL1, COL2)" + consistencyClause); } byte[] b1 = new byte[] { 1, 1, 19, -28, 24, 1, 1, -11, -21, 1 }; @@ -449,6 +495,7 @@ public void testVarBinaryPkSchema2() throws Exception { upsertRow(preparedStatement, b16, b26, b36, b46, b56, b66); } conn.commit(); + waitForEventualConsistency(); PreparedStatement preparedStatement = conn.prepareStatement("SELECT * FROM " + tableName + " WHERE COL1 = ? AND COL2 = ?"); @@ -664,12 +711,13 @@ public void testVarBinaryPkSchema3() throws Exception { + " COL3 VARBINARY_ENCODED CONSTRAINT pk PRIMARY KEY(PK1, PK2, PK3)) " + (this.columnEncoded ? "" : "COLUMN_ENCODED_BYTES=0")); + String consistencyClause = eventual ? " CONSISTENCY = EVENTUAL" : " CONSISTENCY = STRONG"; if (this.coveredIndex) { - conn.createStatement().execute( - "CREATE INDEX " + indexName + " ON " + tableName + " (COL1, COL2) INCLUDE (COL3)"); + conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + + " (COL1, COL2) INCLUDE (COL3)" + consistencyClause); } else { - conn.createStatement() - .execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + " (COL1, COL2)"); + conn.createStatement().execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + + " (COL1, COL2)" + consistencyClause); } byte[] b1 = new byte[] { 1, 1, 19, -28, 24, 1, 1, -11, -21, 1 }; @@ -743,6 +791,7 @@ public void testVarBinaryPkSchema3() throws Exception { upsertRow(preparedStatement, b16, b26, b36, b46, b56, b66); } conn.commit(); + waitForEventualConsistency(); PreparedStatement preparedStatement = conn.prepareStatement("SELECT * FROM " + tableName + " WHERE COL1 = ? AND COL2 = ?"); @@ -951,12 +1000,13 @@ public void testVarBinaryPkSchema4() throws Exception { + " COL3 VARBINARY_ENCODED CONSTRAINT pk PRIMARY KEY(PK1, PK2, PK3)) " + (this.columnEncoded ? "" : "COLUMN_ENCODED_BYTES=0")); + String consistencyClause = eventual ? " CONSISTENCY = EVENTUAL" : " CONSISTENCY = STRONG"; if (this.coveredIndex) { - conn.createStatement().execute( - "CREATE INDEX " + indexName + " ON " + tableName + " (COL1, COL2) INCLUDE (COL3)"); + conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + + " (COL1, COL2) INCLUDE (COL3)" + consistencyClause); } else { - conn.createStatement() - .execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + " (COL1, COL2)"); + conn.createStatement().execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName + + " (COL1, COL2)" + consistencyClause); } byte[] b1 = new byte[] { 1, 1, 19, -28, 24, 1, 1, -11, -21, 1 }; @@ -1027,6 +1077,7 @@ public void testVarBinaryPkSchema4() throws Exception { upsertRow(preparedStatement, b16, b26, b36, b46, b56, b66); } conn.commit(); + waitForEventualConsistency(); PreparedStatement preparedStatement = conn.prepareStatement("SELECT * FROM " + tableName + " WHERE COL1 = ? AND COL2 = ?"); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java new file mode 100644 index 00000000000..a862df07adf --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.index; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class GlobalIndexCheckerEventualGenerateIT extends GlobalIndexCheckerIT { + + public GlobalIndexCheckerEventualGenerateIT(boolean async, boolean encoded) { + super(async, encoded); + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(10); + props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2000)); + props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(5)); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.FALSE.toString()); + props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Before + public void appendEventualConsistency() { + if (indexDDLOptions.trim().isEmpty()) { + indexDDLOptions = " CONSISTENCY=EVENTUAL"; + } else { + indexDDLOptions = " CONSISTENCY=EVENTUAL," + indexDDLOptions; + } + } + + @After + public void after() throws Exception { + } + + @Override + protected void waitForEventualConsistency() throws Exception { + Thread.sleep(18000); + } + + @Parameterized.Parameters(name = "async={0},encoded={1}") + public static synchronized Collection data() { + List list = Lists.newArrayListWithExpectedSize(4); + boolean[] Booleans = new boolean[] { true, false }; + for (boolean async : Booleans) { + for (boolean encoded : Booleans) { + list.add(new Object[] { async, encoded }); + } + } + return list; + } + +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java new file mode 100644 index 00000000000..d7b5a35e1c6 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.index; + +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; + +@Category(NeedsOwnMiniClusterTest.class) +public class GlobalIndexCheckerEventualIT extends GlobalIndexCheckerIT { + + public GlobalIndexCheckerEventualIT(boolean async, boolean encoded) { + super(async, encoded); + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(8); + props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS, Integer.toString(2000)); + props.put(INDEX_CDC_CONSUMER_RETRY_PAUSE_MS, Integer.toString(5)); + props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Integer.toString(-1)); + props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(2)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(1)); + props.put(PHOENIX_INDEX_CDC_MUTATION_SERIALIZE, Boolean.TRUE.toString()); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Before + public void appendEventualConsistency() { + if (indexDDLOptions.trim().isEmpty()) { + indexDDLOptions = " CONSISTENCY=EVENTUAL"; + } else { + indexDDLOptions = " CONSISTENCY=EVENTUAL," + indexDDLOptions; + } + } + + @After + public void after() throws Exception { + } + + @Override + protected void waitForEventualConsistency() throws Exception { + Thread.sleep(15000); + } + + @Parameterized.Parameters(name = "async={0},encoded={1}") + public static synchronized Collection data() { + List list = Lists.newArrayListWithExpectedSize(4); + boolean[] Booleans = new boolean[] { true, false }; + for (boolean async : Booleans) { + for (boolean encoded : Booleans) { + list.add(new Object[] { async, encoded }); + } + } + return list; + } + +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java index 67ccf665e2f..8d7d4c546ad 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java @@ -85,8 +85,8 @@ public class GlobalIndexCheckerIT extends BaseTest { private final boolean async; - private String indexDDLOptions; - private String tableDDLOptions; + protected String indexDDLOptions; + protected String tableDDLOptions; private StringBuilder optionBuilder; private StringBuilder indexOptionBuilder; private final boolean encoded; @@ -266,6 +266,7 @@ public void testPhoenixRowTimestamp() throws Exception { + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table assertExplainPlan(conn, query, dataTableName, indexTableName); + waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -300,6 +301,7 @@ public void testPhoenixRowTimestamp() throws Exception { "SELECT val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table assertExplainPlan(conn, query, dataTableName, indexTableName); + waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("de", rs.getString(1)); @@ -322,6 +324,7 @@ public void testPhoenixRowTimestamp() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " values ('e', 'ae', 'efg', 'efgh')"); conn.commit(); + waitForEventualConsistency(); // Write a query to get all the rows in the order of their timestamps query = "SELECT val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString() @@ -365,6 +368,7 @@ public void testPhoenixRowTimestamp() throws Exception { + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; assertExplainPlan(conn, query, dataTableName, indexTableName); + waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("ab", rs.getString(1)); @@ -565,6 +569,7 @@ public void testSimulateConcurrentUpdates() throws Exception { conn.commit(); // Now the expected state of the index table is {('ab', 'a', 'abcc' , null), ('ab', 'b', null, // 'bcde')} + waitForEventualConsistency(); ResultSet rs = conn.createStatement().executeQuery("SELECT * from " + indexTableName); assertTrue(rs.next()); assertEquals("ab", rs.getString(1)); @@ -606,6 +611,7 @@ public void testFailPostIndexDeleteUpdate() throws Exception { String dml = "DELETE from " + dataTableName + " WHERE id = 'a'"; assertEquals(1, conn.createStatement().executeUpdate(dml)); conn.commit(); + waitForEventualConsistency(); // The index rows are actually not deleted yet because IndexRegionObserver failed delete // operation. However, they are @@ -653,6 +659,7 @@ public void testPartialRowUpdateForMutable() throws Exception { String selectSql = "SELECT * from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the index table assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + waitForEventualConsistency(); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); @@ -666,6 +673,7 @@ public void testPartialRowUpdateForMutable() throws Exception { conn.commit(); // Verify that we will read from the index table assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + waitForEventualConsistency(); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); @@ -795,6 +803,7 @@ public void testFailPostIndexRowUpdate() throws Exception { // run the index MR job. IndexToolIT.runIndexTool(false, null, dataTableName, indexTableName); } + waitForEventualConsistency(); // Configure IndexRegionObserver to fail the last write phase (i.e., the post index update // phase) where the verify flag is set // to true and/or index rows are deleted and check that this does not impact the correctness @@ -805,6 +814,7 @@ public void testFailPostIndexRowUpdate() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " (id, val1, val2) values ('c', 'cd','cde')"); conn.commit(); + waitForEventualConsistency(); IndexTool indexTool = IndexToolIT.runIndexTool(false, "", dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.ONLY); assertEquals(3, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue()); @@ -824,8 +834,10 @@ public void testFailPostIndexRowUpdate() throws Exception { .findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters() .findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue()); - assertEquals(2, indexTool.getJob().getCounters() - .findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue()); + if (!indexDDLOptions.contains("CONSISTENCY=EVENTUAL")) { + assertEquals(2, indexTool.getJob().getCounters() + .findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue()); + } assertEquals(0, indexTool.getJob().getCounters() .findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters() @@ -872,6 +884,7 @@ public void testUnverifiedValuesAreNotVisible() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " (id, val1, val2) values ('a', 'ab','abc')"); conn.commit(); + waitForEventualConsistency(); // At this moment val3 in the data table row has null value String selectSql = "SELECT val3 from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the index table @@ -919,6 +932,7 @@ public void testUnverifiedRowRepair() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " values ('a', 'ab','abc', 'abcd')"); conn.commit(); + waitForEventualConsistency(); // At this moment val3 in the data table row should not have null value selectSql = "SELECT val3 from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the index table @@ -932,6 +946,7 @@ public void testUnverifiedRowRepair() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " (id, val1, val3) values ('a', 'ab','abcde')"); commitWithException(conn); + waitForEventualConsistency(); // The above upsert will create an unverified index row // Configure IndexRegionObserver to allow the data write phase IndexRegionObserver.setFailDataTableUpdatesForTesting(false); @@ -983,6 +998,7 @@ public void testOnePhaseOverwiteFollowingTwoPhaseWrite() throws Exception { String selectSql = "SELECT val2, val3 from " + dataTableName + " WHERE val1 = 'cd'"; // Verify that we will read from the first index table assertExplainPlan(conn, selectSql, dataTableName, indexTableName + "1"); + waitForEventualConsistency(); // Verify the first write is visible but the second one is not ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); @@ -1120,6 +1136,7 @@ public void testFailDataTableAndPostIndexRowUpdate() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " (id, val3) values ('a', 'abcdd')"); conn.commit(); + waitForEventualConsistency(); String selectSql = "SELECT val2, val3 from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the first index table assertExplainPlan(conn, selectSql, dataTableName, indexName + "1"); @@ -1326,6 +1343,7 @@ public void testUnverifiedIndexRowWithSkipScanFilter2() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " " + "values ('g', 'val1', 'val2g', null)"); conn.commit(); + waitForEventualConsistency(); // Fail phase 3 IndexRegionObserver.setFailPostIndexUpdatesForTesting(false); String selectSql = "SELECT id from " + dataTableName @@ -1390,6 +1408,7 @@ public void testReadRepairWithDistinctPrefixFilter() throws Exception { conn.createStatement().execute("upsert into " + dataTableName + " " + "values ('a2', 'a1', 'val1', 'val2a', 'val31', 'val4')"); conn.commit(); + waitForEventualConsistency(); // create an unverified update to the index row pointing to an existing data row IndexRegionObserver.setFailDataTableUpdatesForTesting(true); @@ -1400,6 +1419,7 @@ public void testReadRepairWithDistinctPrefixFilter() throws Exception { conn.createStatement().execute("upsert into " + dataTableName + " " + "values ('a2', 'a2', 'val1', 'val1b', 'val3', 'val4')"); conn.commit(); + waitForEventualConsistency(); ArrayList expectedValues = Lists.newArrayList("a1", "a2"); String selectSql = @@ -1415,6 +1435,7 @@ public void testReadRepairWithDistinctPrefixFilter() throws Exception { conn.createStatement().execute("upsert into " + dataTableName + " " + "values ('a3', 'a2', 'val1', 'val2a', 'val3', 'val4')"); conn.commit(); + waitForEventualConsistency(); IndexRegionObserver.setFailPostIndexUpdatesForTesting(false); expectedValues = Lists.newArrayList("a1", "a2", "a3"); selectSql = @@ -1431,6 +1452,7 @@ public void testReadRepairWithDistinctPrefixFilter() throws Exception { conn.createStatement().execute("upsert into " + dataTableName + " " + "values ('a5', 'a1', 'val1_4', 'val1_4', 'val1_4', 'val1_4')"); conn.commit(); + waitForEventualConsistency(); IndexRegionObserver.setFailPostIndexUpdatesForTesting(false); expectedValues = Lists.newArrayList("a4", "a5"); selectSql = @@ -1475,6 +1497,7 @@ public void testUncoveredIndexWithDistinctPrefixFilter() throws Exception { conn.createStatement().execute("upsert into " + dataTableName + " " + "values ('a2', 'a1', 'val1a', 'val2a', 'val31', 'val4')"); conn.commit(); + waitForEventualConsistency(); ArrayList expectedValues = Lists.newArrayList("a1", "a2"); // condition on val1 in WHERE clause so that query will use the uncovered index @@ -1499,6 +1522,7 @@ public void testUncoveredIndexWithDistinctPrefixFilter() throws Exception { conn.createStatement().execute("upsert into " + dataTableName + " " + "values ('a4', 'a1', 'val1b', 'val2a', 'val31', 'val4')"); conn.commit(); + waitForEventualConsistency(); expectedValues = Lists.newArrayList("a3", "a4"); selectSql = "SELECT distinct(id1) from " + dataTableName + " WHERE val1 = 'val1b'"; verifyDistinctQueryOnIndex(conn, uncoveredIndex1, selectSql, expectedValues); @@ -1589,6 +1613,7 @@ public void testIndexRowWithNullIncludedColumnAndFilter() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " (id, val3) values ('a', null)"); conn.commit(); + waitForEventualConsistency(); String dql = String.format("select id, val2 from %s where val1='ab' and val3='abcd'", dataTableName); @@ -1613,6 +1638,7 @@ public void testIndexRowWithNullIncludedColumnAndFilter() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " values ('a', 'ac', null, null)"); conn.commit(); + waitForEventualConsistency(); dql = String.format("select id, val2 from %s where val1='ac' and val3 is null", dataTableName); @@ -1651,6 +1677,7 @@ public void testIndexToolWithNullIncludedColumn() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " (id, val1, val2) values ('c', 'cd', 'cde')"); conn.commit(); + waitForEventualConsistency(); IndexRegionObserver.setIgnoreWritingDeleteColumnsToIndex(false); IndexTool it = IndexToolIT.runIndexTool(false, null, dataTableName, indexName, null, 0, IndexTool.IndexVerifyType.BEFORE); @@ -1689,6 +1716,7 @@ public void testIndexToolWithMultipleDeleteFamilyMarkers() throws Exception { String delete = String.format("DELETE FROM %s where id = 'a'", dataTableName); conn.createStatement().execute(delete); conn.commit(); + waitForEventualConsistency(); // skip phase2, inserts an unverified row in index IndexRegionObserver.setFailDataTableUpdatesForTesting(true); String dml = "upsert into " + dataTableName + " (id, val1, val3) values ('a', 'ab', ?)"; @@ -1715,6 +1743,7 @@ public void testIndexToolWithMultipleDeleteFamilyMarkers() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " (id, val1, val3) values ('a', 'ab', null)"); conn.commit(); + waitForEventualConsistency(); IndexTool it = IndexToolIT.runIndexTool(false, null, dataTableName, indexName, null, 0, IndexTool.IndexVerifyType.ONLY); CounterGroup mrJobCounters = IndexToolIT.getMRJobCounters(it); @@ -1737,6 +1766,8 @@ public void testViewIndexRowUpdate() throws Exception { // No need to run the same test twice one for async = true and the other for async = false return; } + Assume.assumeFalse("View indexes do not support CONSISTENCY=EVENTUAL", + indexDDLOptions.contains("CONSISTENCY=EVENTUAL")); try (Connection conn = DriverManager.getConnection(getUrl())) { // Create a base table String dataTableName = generateUniqueName(); @@ -1813,6 +1844,7 @@ public void testOnDuplicateKeyWithIndex() throws Exception { + "val1 = val1 || val1, val2 = val2 || val2"; conn.createStatement().execute(upsertSql); conn.commit(); + waitForEventualConsistency(); String selectSql = "SELECT * from " + dataTableName + " WHERE val1 = 'abab'"; assertExplainPlan(conn, selectSql, dataTableName, indexTableName); ResultSet rs = conn.createStatement().executeQuery(selectSql); @@ -1849,6 +1881,7 @@ public void testWithDistinctPrefixFilter() throws Exception { ps.executeUpdate(); } conn.commit(); + waitForEventualConsistency(); String distinctQuery = "SELECT DISTINCT val1 FROM " + dataTableName; try (ResultSet rs = conn.createStatement().executeQuery(distinctQuery)) { PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); @@ -1876,14 +1909,18 @@ public static void commitWithException(Connection conn) { } } - static private void verifyTableHealth(Connection conn, String dataTableName, - String indexTableName) throws Exception { + protected void waitForEventualConsistency() throws Exception { + } + + protected void verifyTableHealth(Connection conn, String dataTableName, String indexTableName) + throws Exception { // Add two rows and check everything is still okay conn.createStatement() .execute("upsert into " + dataTableName + " values ('a', 'ab', 'abc', 'abcd')"); conn.createStatement() .execute("upsert into " + dataTableName + " values ('z', 'za', 'zab', 'zabc')"); conn.commit(); + waitForEventualConsistency(); String selectSql = "SELECT * from " + dataTableName + " WHERE val1 = 'ab'"; /// Verify that we will read from the index table assertExplainPlan(conn, selectSql, dataTableName, indexTableName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java index 8632eaf0137..6583df2d84b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.mapreduce.PhoenixJobCounters.INPUT_RECORDS; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT; @@ -142,6 +143,7 @@ public static synchronized void doSetup() throws Exception { props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS, TestScanningResultPostDummyResultCaller.class.getName()); props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java index 4db0b3c5bdd..6b6f3059b33 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_STARTUP_DELAY_MS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -100,6 +101,7 @@ public static synchronized void setupMiniCluster(Long threshold, Long rows, Long Configuration conf = HBaseConfiguration.create(); Map props = Maps.newHashMapWithExpectedSize(1); props.put(QueryServices.CLIENT_INDEX_ASYNC_THRESHOLD, Long.toString(threshold)); + props.put(INDEX_CDC_CONSUMER_STARTUP_DELAY_MS, Integer.toString(Integer.MAX_VALUE)); url = setUpTestCluster(conf, new ReadOnlyProps(props.entrySet().iterator())); driver = initAndRegisterTestDriver(url, new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index 2a46162e8b1..8184f923962 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -18,7 +18,7 @@ package org.apache.phoenix.end2end.index; import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_INDEX_SYSTEM_TABLE; -import static org.apache.phoenix.exception.SQLExceptionCode.MISMATCHED_TOKEN; +import static org.apache.phoenix.exception.SQLExceptionCode.PARSER_ERROR; import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.COLUMN_TYPES; import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_COLUMNS; import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED; @@ -907,7 +907,7 @@ public void testIndexesOnOtherSystemTables() throws Exception { "CREATE INDEX IF NOT EXISTS SYS_INDEX_FUNCTION_IDX ON SYSTEM.FUNCTION(CLASS_NAME,JAR_PATH) WHERE FUNCTION_NAME = '4'"); fail(); } catch (SQLException sqle) { - Assert.assertEquals(MISMATCHED_TOKEN.getErrorCode(), sqle.getErrorCode()); + Assert.assertEquals(PARSER_ERROR.getErrorCode(), sqle.getErrorCode()); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReplicationWithWALAnnotationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReplicationWithWALAnnotationIT.java index 3293a9525aa..9848452bd86 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReplicationWithWALAnnotationIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReplicationWithWALAnnotationIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_STARTUP_DELAY_MS; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -151,6 +152,7 @@ private static void setupConfigsAndStartCluster() throws Exception { ReplicationSinkEndpoint.class.getName()); conf1.setBoolean(IndexRegionObserver.PHOENIX_APPEND_METADATA_TO_WAL, true); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + conf1.set(INDEX_CDC_CONSUMER_STARTUP_DELAY_MS, Integer.toString(Integer.MAX_VALUE)); setUpConfigForMiniCluster(conf1); utility1 = new HBaseTestingUtility(conf1); @@ -165,6 +167,7 @@ private static void setupConfigsAndStartCluster() throws Exception { conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); conf2.setBoolean("dfs.support.append", true); conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false); + conf2.set(INDEX_CDC_CONSUMER_STARTUP_DELAY_MS, Integer.toString(Integer.MAX_VALUE)); utility2 = new HBaseTestingUtility(conf2); utility2.startMiniZKCluster(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java index a008150d6b6..555f87ff0cb 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java @@ -19,6 +19,7 @@ import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlan; import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlanWithLimit; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -133,6 +134,7 @@ public static synchronized void doSetup() throws Exception { props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0)); props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS, TestScanningResultPostDummyResultCaller.class.getName()); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java index 7e348a0902d..68221e63406 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.jdbc; import static org.apache.hadoop.test.GenericTestUtils.waitFor; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneZKDown; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection; @@ -76,6 +77,11 @@ public class FailoverPhoenixConnection2IT { @BeforeClass public static void setUpBeforeClass() throws Exception { + CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); + CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); + CLUSTERS.start(); DriverManager.registerDriver(PhoenixDriver.INSTANCE); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java index f633294df2d..7ee3f49b9b3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.test.GenericTestUtils.waitFor; import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION; +import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_STARTUP_DELAY_MS; import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR; import static org.apache.phoenix.jdbc.HighAvailabilityGroup.URLS; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair; @@ -99,6 +100,10 @@ public class FailoverPhoenixConnectionIT { @BeforeClass public static void setUpBeforeClass() throws Exception { + CLUSTERS.getHBaseCluster1().getConfiguration().setInt(INDEX_CDC_CONSUMER_STARTUP_DELAY_MS, + Integer.MAX_VALUE); + CLUSTERS.getHBaseCluster2().getConfiguration().setInt(INDEX_CDC_CONSUMER_STARTUP_DELAY_MS, + Integer.MAX_VALUE); CLUSTERS.start(); DriverManager.registerDriver(PhoenixDriver.INSTANCE); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAConnectionWithMasterAndRPCRegistryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAConnectionWithMasterAndRPCRegistryIT.java index 1464db36df6..46674b61db8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAConnectionWithMasterAndRPCRegistryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAConnectionWithMasterAndRPCRegistryIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.jdbc; import static org.apache.hadoop.test.GenericTestUtils.waitFor; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneHBaseDown; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection; @@ -86,6 +87,10 @@ public class HAConnectionWithMasterAndRPCRegistryIT { @BeforeClass public static void setUpBeforeClass() throws Exception { + CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); + CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); CLUSTERS.start(); DriverManager.registerDriver(PhoenixDriver.INSTANCE); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java index 1564beda213..7453c599032 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.jdbc; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneHBaseDown; @@ -81,6 +82,10 @@ public HighAvailabilityTestingUtilityIT(ClusterRoleRecord.RegistryType registryT @BeforeClass public static void setUpBeforeClass() throws Exception { + CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); + CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); CLUSTERS.start(); DriverManager.registerDriver(PhoenixDriver.INSTANCE); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFallbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFallbackIT.java index 076e418636b..3a1ae7d3119 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFallbackIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFallbackIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.jdbc; import static org.apache.hadoop.test.GenericTestUtils.waitFor; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR; import static org.apache.phoenix.jdbc.HighAvailabilityPolicy.PARALLEL; import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair; @@ -57,6 +58,10 @@ public class ParallelPhoenixConnectionFallbackIT { @BeforeClass public static void setUpBeforeClass() throws Exception { haGroupName = ParallelPhoenixConnectionFallbackIT.class.getSimpleName(); + CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); + CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); CLUSTERS.start(); DriverManager.registerDriver(PhoenixDriver.INSTANCE); PROPERTIES.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java index d14f00ae8ad..04b385f7625 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.jdbc; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.jdbc.ClusterRoleRecordGeneratorTool.PHOENIX_HA_GROUP_STORE_PEER_ID_DEFAULT; import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR; import static org.apache.phoenix.jdbc.HighAvailabilityPolicy.PARALLEL; @@ -72,6 +73,10 @@ public class ParallelPhoenixNullComparingResultSetIT { @BeforeClass public static void setUpBeforeClass() throws Exception { String haGroupName = ParallelPhoenixNullComparingResultSetIT.class.getSimpleName(); + CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); + CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, + false); CLUSTERS.start(); DriverManager.registerDriver(PhoenixDriver.INSTANCE); PROPERTIES.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java index 05538f5da42..fe059f8451f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.monitoring; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.monitoring.MetricType.DELETE_BATCH_FAILED_COUNTER; import static org.apache.phoenix.monitoring.MetricType.DELETE_BATCH_FAILED_SIZE; import static org.apache.phoenix.monitoring.MetricType.MUTATION_COMMIT_TIME; @@ -65,6 +66,7 @@ public static synchronized void doSetup() throws Exception { props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true)); // disable renewing leases as this will force spooling to happen. props.put(QueryServices.RENEW_LEASE_ENABLED, String.valueOf(false)); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); // need the non-test driver for some tests that check number of hconnections, etc. DriverManager.registerDriver(PhoenixDriver.INSTANCE); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java index fdaf8a86390..4ca910964a3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java @@ -20,6 +20,7 @@ import static org.apache.phoenix.exception.SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY; import static org.apache.phoenix.exception.SQLExceptionCode.GET_TABLE_REGIONS_FAIL; import static org.apache.phoenix.exception.SQLExceptionCode.OPERATION_TIMED_OUT; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_MUTATION_BYTES; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_TIME; @@ -158,6 +159,7 @@ public class PhoenixTableLevelMetricsIT extends BaseTest { @BeforeClass public static void doSetup() throws Exception { final Configuration conf = HBaseConfiguration.create(); + conf.set(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpConfigForMiniCluster(conf); conf.set(QueryServices.TABLE_LEVEL_METRICS_ENABLED, String.valueOf(true)); conf.set(QueryServices.METRIC_PUBLISHER_ENABLED, String.valueOf(true)); @@ -189,6 +191,7 @@ public Configuration getConfiguration(Configuration confToClone) { Map props = Maps.newHashMapWithExpectedSize(1); props.put(BaseTest.DRIVER_CLASS_NAME_ATTRIB, PhoenixMetricsTestingDriver.class.getName()); props.put(ENABLE_SERVER_UPSERT_SELECT, "true"); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); initAndRegisterTestDriver(url, new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/connectionqueryservice/ConnectionQueryServicesMetricsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/connectionqueryservice/ConnectionQueryServicesMetricsIT.java index 54d53afdf6d..57791072cda 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/connectionqueryservice/ConnectionQueryServicesMetricsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/connectionqueryservice/ConnectionQueryServicesMetricsIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.monitoring.connectionqueryservice; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.monitoring.MetricType.OPEN_INTERNAL_PHOENIX_CONNECTIONS_COUNTER; import static org.apache.phoenix.monitoring.MetricType.OPEN_PHOENIX_CONNECTIONS_COUNTER; import static org.apache.phoenix.monitoring.MetricType.PHOENIX_CONNECTIONS_THROTTLED_COUNTER; @@ -98,6 +99,7 @@ public Configuration getConfiguration() { // counter won't be increased at all. So we need to set max allowed connection count conf.set(CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS, "2"); conf.set(INTERNAL_CONNECTION_MAX_ALLOWED_CONNECTIONS, "1"); + conf.set(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); return conf; } @@ -109,6 +111,7 @@ public Configuration getConfiguration(Configuration confToClone) { // counter won't be increased at all. So we need to set max allowed connection count conf.set(CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS, "2"); conf.set(INTERNAL_CONNECTION_MAX_ALLOWED_CONNECTIONS, "1"); + conf.set(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); Configuration copy = new Configuration(conf); copy.addResource(confToClone); return copy; @@ -117,6 +120,7 @@ public Configuration getConfiguration(Configuration confToClone) { Configuration conf = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(); conf.set(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); conf.set(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); + conf.set(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); hbaseTestUtil = new HBaseTestingUtility(conf); setUpConfigForMiniCluster(conf); conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java index e9df1d00182..2a28c61e78d 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.query; import static org.apache.phoenix.exception.SQLExceptionCode.NEW_INTERNAL_CONNECTION_THROTTLED; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_OPEN_INTERNAL_PHOENIX_CONNECTIONS; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS; import static org.apache.phoenix.query.QueryServices.CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS; @@ -60,6 +61,7 @@ public class MaxConcurrentConnectionsIT extends BaseTest { @BeforeClass public static void setUp() throws Exception { final Configuration conf = HBaseConfiguration.create(); + conf.set(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpConfigForMiniCluster(conf); hbaseTestUtil = new HBaseTestingUtility(conf); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java index 03e70ddf77f..c84d62c76a0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.query; +import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.junit.Assert.*; import java.sql.Connection; @@ -67,6 +68,7 @@ public static synchronized void doSetup() throws Exception { props.put(QueryServices.CLIENT_CACHE_ENCODING, "object"); props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); + props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/parse/IndexConsistencyParseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/parse/IndexConsistencyParseTest.java new file mode 100644 index 00000000000..559a57d3f8b --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/parse/IndexConsistencyParseTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.parse; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import org.apache.phoenix.schema.types.IndexConsistency; +import org.junit.Test; + +/** + * Test class for parsing CREATE INDEX statements with CONSISTENCY clause. + */ +public class IndexConsistencyParseTest { + + @Test + public void testCreateIndexWithStrongConsistency() throws Exception { + String sql = "CREATE INDEX idx ON mytable (col1) CONSISTENCY=STRONG"; + SQLParser parser = new SQLParser(sql); + CreateIndexStatement stmt = (CreateIndexStatement) parser.parseStatement(); + + assertNotNull("Statement should not be null", stmt); + assertEquals("Should have STRONG consistency", IndexConsistency.STRONG, + stmt.getIndexConsistency()); + } + + @Test + public void testCreateIndexWithEventualConsistency() throws Exception { + String sql = "CREATE INDEX idx ON mytable (col1) CONSISTENCY=EVENTUAL"; + SQLParser parser = new SQLParser(sql); + CreateIndexStatement stmt = (CreateIndexStatement) parser.parseStatement(); + + assertNotNull("Statement should not be null", stmt); + assertEquals("Should have EVENTUAL consistency", IndexConsistency.EVENTUAL, + stmt.getIndexConsistency()); + } + + @Test + public void testCreateIndexWithoutConsistency() throws Exception { + String sql = "CREATE INDEX idx ON mytable (col1)"; + SQLParser parser = new SQLParser(sql); + CreateIndexStatement stmt = (CreateIndexStatement) parser.parseStatement(); + + assertNotNull("Statement should not be null", stmt); + assertNull("Should default to STRONG consistency", stmt.getIndexConsistency()); + } + + @Test + public void testCreateIndexWithConsistencyAndOtherClauses() throws Exception { + String sql = "CREATE INDEX idx ON mytable (col1) INCLUDE (col2) ASYNC CONSISTENCY=EVENTUAL"; + SQLParser parser = new SQLParser(sql); + CreateIndexStatement stmt = (CreateIndexStatement) parser.parseStatement(); + + assertNotNull("Statement should not be null", stmt); + assertEquals("Should have EVENTUAL consistency", IndexConsistency.EVENTUAL, + stmt.getIndexConsistency()); + assertEquals("Should have include columns", 1, stmt.getIncludeColumns().size()); + assertEquals("Should be async", true, stmt.isAsync()); + } + + @Test + public void testLocalIndexWithConsistency() throws Exception { + String sql = "CREATE LOCAL INDEX idx ON mytable (col1) CONSISTENCY=EVENTUAL"; + SQLParser parser = new SQLParser(sql); + CreateIndexStatement stmt = (CreateIndexStatement) parser.parseStatement(); + + assertNotNull("Statement should not be null", stmt); + assertEquals("Should have EVENTUAL consistency", IndexConsistency.EVENTUAL, + stmt.getIndexConsistency()); + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index 7b14996fba4..e2e3afedef2 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -2070,7 +2070,7 @@ protected synchronized static boolean isAnyStoreRefCountLeaked() throws IOExcept * @throws IOException if something went wrong while connecting to Admin */ public synchronized static boolean isAnyStoreRefCountLeaked(Admin admin) throws IOException { - int retries = 5; + int retries = 15; while (retries > 0) { boolean isStoreRefCountLeaked = isStoreRefCountLeaked(admin); if (!isStoreRefCountLeaked) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java b/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java index 75cc9e3843b..565d7aec7d0 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java @@ -39,7 +39,22 @@ public class IndexScrutiny { public static long scrutinizeIndex(Connection conn, String fullTableName, String fullIndexName) throws SQLException { PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - PTable ptable = pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)); + String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName); + String tableName = SchemaUtil.getTableNameFromFullName(fullTableName); + String indexName = SchemaUtil.getTableNameFromFullName(fullIndexName); + PTable ptable; + PTable pindex; + if (schemaName != null && !schemaName.isEmpty()) { + ptable = + pconn.getTable(new PTableKey(pconn.getTenantId(), SchemaUtil.normalizeIdentifier(schemaName) + + "." + SchemaUtil.normalizeIdentifier(tableName))); + pindex = + pconn.getTable(new PTableKey(pconn.getTenantId(), SchemaUtil.normalizeIdentifier(schemaName) + + "." + SchemaUtil.normalizeIdentifier(indexName))); + } else { + ptable = pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)); + pindex = pconn.getTable(new PTableKey(pconn.getTenantId(), fullIndexName)); + } int tableColumnOffset = 0; List tableColumns = ptable.getColumns(); List tablePKColumns = ptable.getPKColumns(); @@ -48,7 +63,6 @@ public static long scrutinizeIndex(Connection conn, String fullTableName, String tableColumns = tableColumns.subList(tableColumnOffset, tableColumns.size()); tablePKColumns = tablePKColumns.subList(tableColumnOffset, tablePKColumns.size()); } - PTable pindex = pconn.getTable(new PTableKey(pconn.getTenantId(), fullIndexName)); List indexColumns = pindex.getColumns(); int indexColumnOffset = 0; if (pindex.getBucketNum() != null) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index 6541dd7338e..69a191b5c2d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -1694,4 +1695,97 @@ public static String retainSingleQuotes(String input) { return sb.toString(); } + public static void splitTable(String url, String tableName, int nSplits, + long sleepBetweenSplitsMs) { + try ( + PhoenixConnection phoenixConn = + DriverManager.getConnection(url).unwrap(PhoenixConnection.class); + Admin admin = phoenixConn.getQueryServices().getAdmin()) { + TableName hTableName = TableName.valueOf(tableName); + int splitCount = 0; + while (splitCount < nSplits) { + Thread.sleep(sleepBetweenSplitsMs); + try { + int regionsBefore = admin.getRegions(hTableName).size(); + admin.split(hTableName); + int retries = 0; + while (retries < 50) { + int regionsAfter = admin.getRegions(hTableName).size(); + if (regionsAfter >= regionsBefore * 2) { + LOGGER.info("Split {} completed, regions: {} -> {}", splitCount + 1, regionsBefore, + regionsAfter); + break; + } + retries++; + Thread.sleep(100); + } + splitCount++; + } catch (Exception e) { + LOGGER.warn("Exception during split: " + e.getMessage()); + } + } + } catch (Exception e) { + LOGGER.error("Error in split thread", e); + } + } + + /** + * Merges table regions in groups of nMerges consecutive regions. + * @param url connection URL. + * @param tableName name of the table. + * @param nMerges number of consecutive regions to merge together in each group. + * @param sleepBetweenMergesMs time to sleep between merge operations. + */ + public static void mergeTableRegions(String url, String tableName, int nMerges, + long sleepBetweenMergesMs) { + if (nMerges < 2) { + throw new IllegalArgumentException("nMerges must be at least 2"); + } + try ( + PhoenixConnection phoenixConn = + DriverManager.getConnection(url).unwrap(PhoenixConnection.class); + Admin admin = phoenixConn.getQueryServices().getAdmin()) { + TableName hTableName = TableName.valueOf(tableName); + List regions = admin.getRegions(hTableName); + if (regions.size() < 2) { + return; + } + int regionIndex = 0; + while (regionIndex < regions.size()) { + int remainingRegions = regions.size() - regionIndex; + int groupSize = Math.min(nMerges, remainingRegions); + if (groupSize < 2) { + break; + } + byte[][] regionsToMerge = new byte[groupSize][]; + for (int i = 0; i < groupSize; i++) { + regionsToMerge[i] = regions.get(regionIndex + i).getRegionName(); + } + try { + int regionsBefore = admin.getRegions(hTableName).size(); + admin.mergeRegionsAsync(regionsToMerge, false).get(); + int retries = 0; + while (retries < 50) { + List currentRegions = admin.getRegions(hTableName); + if (currentRegions.size() <= regionsBefore - groupSize + 1) { + LOGGER.info("Merge completed, regions: {} -> {}", regionsBefore, + currentRegions.size()); + break; + } + retries++; + Thread.sleep(100); + } + } catch (Exception e) { + LOGGER.warn("Exception during merge: " + e.getMessage(), e); + } + regionIndex += groupSize; + if (regionIndex < regions.size()) { + Thread.sleep(sleepBetweenMergesMs); + } + } + } catch (Exception e) { + LOGGER.error("Error in mergeTableRegions", e); + } + } + } diff --git a/pom.xml b/pom.xml index a4f92dda7d8..627cadbc876 100644 --- a/pom.xml +++ b/pom.xml @@ -115,6 +115,7 @@ 1.0-1 2.0.1 0.5 + 1.1.10.8 1.16.0 3.1.0-incubating 2.10.5 @@ -1242,6 +1243,11 @@ snappy ${snappy.version} + + org.xerial.snappy + snappy-java + ${snappy-java.version} + com.fasterxml.jackson jackson-bom