From 2a8f0c0fe5b0afd2f154bbf0ad0932abfebdeda8 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 13 Apr 2018 14:30:00 -0700 Subject: [PATCH] PHOENIX-4575 Phoenix metadata KEEP_DELETED_CELLS and VERSIONS should be property driven (Chinmay Kulkarni) --- .../phoenix/coprocessor/MetaDataProtocol.java | 4 -- .../query/ConnectionQueryServicesImpl.java | 42 +++++++++++++------ .../ConnectionlessQueryServicesImpl.java | 35 ++++++++++++++-- .../apache/phoenix/query/QueryConstants.java | 14 +++---- .../apache/phoenix/query/QueryServices.java | 5 ++- .../phoenix/query/QueryServicesOptions.java | 3 +- .../org/apache/phoenix/util/UpgradeUtil.java | 4 +- 7 files changed, 74 insertions(+), 33 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java index 4af85bf677a..8f33d414768 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java @@ -72,10 +72,6 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP = 0; public static final String MIGRATION_IN_PROGRESS = "MigrationInProgress"; - public static final int DEFAULT_MAX_META_DATA_VERSIONS = 1000; - public static final boolean DEFAULT_META_DATA_KEEP_DELETED_CELLS = true; - public static final int DEFAULT_MAX_STAT_DATA_VERSIONS = 1; - public static final boolean DEFAULT_STATS_KEEP_DELETED_CELLS = false; public static final int DEFAULT_LOG_VERSIONS = 10; public static final int DEFAULT_LOG_TTL = 7 * 24 * 60 * 60; // 7 days diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index bb94313a2d7..ffce2ea5590 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -726,11 +726,6 @@ public PhoenixConnection connect(String url, Properties info) throws SQLExceptio private ColumnFamilyDescriptor generateColumnFamilyDescriptor(Pair> family, PTableType tableType) throws SQLException { ColumnFamilyDescriptorBuilder columnDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(family.getFirst()); if (tableType != PTableType.VIEW) { - if(props.get(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB) != null){ - columnDescBuilder.setKeepDeletedCells(props.getBoolean(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB, - QueryServicesOptions.DEFAULT_KEEP_DELETED_CELLS) ? KeepDeletedCells.TRUE - : KeepDeletedCells.FALSE); - } columnDescBuilder.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING); columnDescBuilder.setBloomFilterType(BloomType.NONE); for (Entry entry : family.getSecond().entrySet()) { @@ -2494,8 +2489,29 @@ protected void setInitialized(boolean isInitialized) { } // Available for testing - protected String getSystemCatalogDML() { - return QueryConstants.CREATE_TABLE_METADATA; + protected String getSystemCatalogTableDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_TABLE_METADATA); + } + + // Available for testing + protected String getFunctionTableDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_FUNCTION_METADATA); + } + + // Available for testing + protected String getLogTableDDL() { + return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA); + } + + private String setSystemDDLProperties(String ddl) { + return String.format(ddl, + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS), + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)); + } + + private String setSystemLogDDLProperties(String ddl) { + return String.format(ddl, + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)); } @Override @@ -2538,7 +2554,7 @@ public Void call() throws Exception { scnProps, newEmptyMetaData())) { try { metaConnection.setRunningUpgrade(true); - metaConnection.createStatement().executeUpdate(getSystemCatalogDML()); + metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL()); } catch (NewerTableAlreadyExistsException ignore) { // Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed // timestamp. A TableAlreadyExistsException is not thrown, since the table only exists @@ -2695,10 +2711,10 @@ private void createOtherSystemTables(PhoenixConnection metaConnection, Admin hba metaConnection.createStatement().execute(QueryConstants.CREATE_STATS_TABLE_METADATA); } catch (TableAlreadyExistsException ignore) {} try { - metaConnection.createStatement().execute(QueryConstants.CREATE_FUNCTION_METADATA); + metaConnection.createStatement().execute(getFunctionTableDDL()); } catch (TableAlreadyExistsException ignore) {} try { - metaConnection.createStatement().execute(QueryConstants.CREATE_LOG_METADATA); + metaConnection.createStatement().execute(getLogTableDDL()); } catch (TableAlreadyExistsException ignore) {} // Catch the IOException to log the error message and then bubble it up for the client to retry. try { @@ -3003,7 +3019,7 @@ public void upgradeSystemTables(final String url, final Properties props) throws createSysMutexTableIfNotExists(admin); } try { - metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA); + metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL()); } catch (NewerTableAlreadyExistsException ignore) { // Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed // timestamp. A TableAlreadyExistsException is not thrown, since the table only exists @@ -3125,10 +3141,10 @@ public void upgradeSystemTables(final String url, final Properties props) throws } } try { - metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA); + metaConnection.createStatement().executeUpdate(getFunctionTableDDL()); } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {} try { - metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA); + metaConnection.createStatement().executeUpdate(getLogTableDDL()); } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {} // In case namespace mapping is enabled and system table to system namespace mapping is also enabled, diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index 779a60e2769..580da68db28 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -86,11 +86,13 @@ import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.transaction.PhoenixTransactionClient; import org.apache.phoenix.transaction.TransactionFactory.Provider; +import org.apache.phoenix.util.ConfigUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.JDBCUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.SequenceUtil; @@ -108,6 +110,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices implements ConnectionQueryServices { private static ServerName SERVER_NAME = ServerName.parseServerName(HConstants.LOCALHOST + Addressing.HOSTNAME_PORT_SEPARATOR + HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT); + private final ReadOnlyProps props; private PMetaData metaData; private final Map sequenceMap = Maps.newHashMap(); private final String userName; @@ -145,12 +148,38 @@ public ConnectionlessQueryServicesImpl(QueryServices services, ConnectionInfo co // on the server side during testing. this.config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config); this.guidePostsCache = new GuidePostsCache(this, config); + // set replication required parameter + ConfigUtil.setReplicationConfigIfAbsent(this.config); + this.props = new ReadOnlyProps(this.config.iterator()); } private PMetaData newEmptyMetaData() { return new PMetaDataImpl(INITIAL_META_DATA_TABLE_CAPACITY, getProps()); } + protected String getSystemCatalogTableDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_TABLE_METADATA); + } + + protected String getFunctionTableDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_FUNCTION_METADATA); + } + + protected String getLogTableDDL() { + return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA); + } + + private String setSystemDDLProperties(String ddl) { + return String.format(ddl, + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS), + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)); + } + + private String setSystemLogDDLProperties(String ddl) { + return String.format(ddl, + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)); + } + @Override public ConnectionQueryServices getChildQueryServices(ImmutableBytesWritable childId) { return this; // Just reuse the same query services @@ -311,7 +340,7 @@ public void init(String url, Properties props) throws SQLException { metaConnection = new PhoenixConnection(this, globalUrl, scnProps, newEmptyMetaData()); metaConnection.setRunningUpgrade(true); try { - metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA); + metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL()); } catch (TableAlreadyExistsException ignore) { // Ignore, as this will happen if the SYSTEM.TABLE already exists at this fixed timestamp. // A TableAlreadyExistsException is not thrown, since the table only exists *after* this fixed timestamp. @@ -334,11 +363,11 @@ public void init(String url, Properties props) throws SQLException { } try { - metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA); + metaConnection.createStatement().executeUpdate(getFunctionTableDDL()); } catch (NewerTableAlreadyExistsException ignore) { } try { - metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA); + metaConnection.createStatement().executeUpdate(getLogTableDDL()); } catch (NewerTableAlreadyExistsException ignore) {} } catch (SQLException e) { sqlE = e; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index 67cf1b4c2f4..f018e1ac5f8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -344,8 +344,8 @@ public enum JoinType {INNER, LEFT_OUTER} TRANSACTION_PROVIDER + " TINYINT, " + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" + - HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + - ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" + + HConstants.VERSIONS + "=%s,\n" + + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" + // Install split policy to prevent a tenant's metadata from being split across regions. TableDescriptorBuilder.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE; @@ -362,8 +362,6 @@ public enum JoinType {INNER, LEFT_OUTER} "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + PHYSICAL_NAME + "," + COLUMN_FAMILY + ","+ GUIDE_POST_KEY+"))\n" + - HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_STAT_DATA_VERSIONS + ",\n" + - ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_STATS_KEEP_DELETED_CELLS + ",\n" + // Install split policy to prevent a physical table's stats from being split across regions. TableDescriptorBuilder.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE; @@ -383,8 +381,6 @@ public enum JoinType {INNER, LEFT_OUTER} CYCLE_FLAG + " BOOLEAN, \n" + LIMIT_REACHED_FLAG + " BOOLEAN \n" + " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA + "," + SEQUENCE_NAME + "))\n" + - HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + - ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE; public static final String CREATE_SYSTEM_SCHEMA = "CREATE SCHEMA " + SYSTEM_CATALOG_SCHEMA; public static final String UPGRADE_TABLE_SNAPSHOT_PREFIX = "_UPGRADING_TABLE_"; @@ -408,8 +404,8 @@ public enum JoinType {INNER, LEFT_OUTER} MIN_VALUE + " VARCHAR, \n" + MAX_VALUE + " VARCHAR, \n" + " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ", " + FUNCTION_NAME + ", " + TYPE + ", " + ARG_POSITION + "))\n" + - HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + - ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n"+ + HConstants.VERSIONS + "=%s,\n" + + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n"+ // Install split policy to prevent a tenant's metadata from being split across regions. TableDescriptorBuilder.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE; @@ -435,7 +431,7 @@ public enum JoinType {INNER, LEFT_OUTER} SCAN_METRICS_JSON + " VARCHAR, \n" + " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" + HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n" + - ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n"+ + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n"+ // Install split policy to prevent a tenant's metadata from being split across regions. TableDescriptorBuilder.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" + diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index 66ec076a064..0f465b30874 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -202,7 +202,6 @@ public interface QueryServices extends SQLCloseable { public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable"; public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries"; public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay"; - public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells"; public static final String DEFAULT_STORE_NULLS_ATTRIB = "phoenix.table.default.store.nulls"; public static final String DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB = "phoenix.table.istransactional.default"; public static final String DEFAULT_TRANSACTION_PROVIDER_ATTRIB = "phoenix.table.transaction.provider.default"; @@ -250,6 +249,10 @@ public interface QueryServices extends SQLCloseable { public static final String QUERY_SERVER_REMOTEUSEREXTRACTOR_PARAM = "phoenix.queryserver.remoteUserExtractor.param"; public static final String QUERY_SERVER_DISABLE_KERBEROS_LOGIN = "phoenix.queryserver.disable.kerberos.login"; + // metadata configs + public static final String DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB = "phoenix.system.default.keep.deleted.cells"; + public static final String DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB = "phoenix.system.default.max.versions"; + public static final String RENEW_LEASE_ENABLED = "phoenix.scanner.lease.renew.enabled"; public static final String RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS = "phoenix.scanner.lease.renew.interval"; public static final String RENEW_LEASE_THRESHOLD_MILLISECONDS = "phoenix.scanner.lease.threshold"; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 7f344464b91..90e31657f10 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -212,6 +212,8 @@ public class QueryServicesOptions { public static final boolean DEFAULT_ALLOW_LOCAL_INDEX = true; public static final int DEFAULT_INDEX_HANDLER_COUNT = 30; public static final int DEFAULT_METADATA_HANDLER_COUNT = 30; + public static final int DEFAULT_SYSTEM_MAX_VERSIONS = 1; + public static final boolean DEFAULT_SYSTEM_KEEP_DELETED_CELLS = false; // Retries when doing server side writes to SYSTEM.CATALOG // 20 retries with 100 pause = 230 seconds total retry time @@ -256,7 +258,6 @@ public class QueryServicesOptions { public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true; public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10; public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds. - public static final boolean DEFAULT_KEEP_DELETED_CELLS = false; public static final boolean DEFAULT_STORE_NULLS = false; // TODO Change this to true as part of PHOENIX-1543 diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index 6fbe760873d..f57249c2972 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -226,7 +226,7 @@ private static void copyTable(PhoenixConnection conn, byte[] sourceName, byte[] Scan scan = new Scan(); scan.setRaw(true); - scan.setMaxVersions(MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS); + scan.setMaxVersions(); ResultScanner scanner = null; Table source = null; Table target = null; @@ -702,7 +702,7 @@ public static boolean upgradeSequenceTable(PhoenixConnection conn, int nSaltBuck boolean success = false; Scan scan = new Scan(); scan.setRaw(true); - scan.setMaxVersions(MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS); + scan.setMaxVersions(); Table seqTable = conn.getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES); try { boolean committed = false;