Skip to content

Commit

Permalink
PHOENIX-2570 Start transaction manager only when needed and reduce nu…
Browse files Browse the repository at this point in the history 
…mber of threads spawned for tests
  • Loading branch information
Samarth committed Jan 6, 2016
1 parent ddf24b5 commit f3ea392
Show file tree
Hide file tree
Showing 5 changed files with 56 additions and 19 deletions.
3 changes: 3 additions & 0 deletions phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ public interface QueryServices extends SQLCloseable {
public static final String RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS = "phoenix.scanner.lease.renew.interval";
public static final String RENEW_LEASE_THRESHOLD_MILLISECONDS = "phoenix.scanner.lease.threshold";
public static final String RENEW_LEASE_THREAD_POOL_SIZE = "phoenix.scanner.lease.pool.size";
public static final String HCONNECTION_POOL_CORE_SIZE = "hbase.hconnection.threads.core";
public static final String HCONNECTION_POOL_MAX_SIZE = "hbase.hconnection.threads.max";
public static final String HTABLE_MAX_THREADS = "hbase.htable.threads.max";

/**
* Get executor service used for parallel scans
Expand Down
26 changes: 26 additions & 0 deletions phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,4 +615,30 @@ public QueryServicesOptions setEnableRenewLease(boolean enable) {
config.setBoolean(RENEW_LEASE_ENABLED, enable);
return this;
}

public QueryServicesOptions setIndexHandlerCount(int count) {
config.setInt(QueryServices.INDEX_HANDLER_COUNT_ATTRIB, count);
return this;
}

public QueryServicesOptions setMetadataHandlerCount(int count) {
config.setInt(QueryServices.METADATA_HANDLER_COUNT_ATTRIB, count);
return this;
}

public QueryServicesOptions setHConnectionPoolCoreSize(int count) {
config.setInt(QueryServices.HCONNECTION_POOL_CORE_SIZE, count);
return this;
}

public QueryServicesOptions setHConnectionPoolMaxSize(int count) {
config.setInt(QueryServices.HCONNECTION_POOL_MAX_SIZE, count);
return this;
}

public QueryServicesOptions setMaxThreadsPerHTable(int count) {
config.setInt(QueryServices.HTABLE_MAX_THREADS, count);
return this;
}

}
32 changes: 15 additions & 17 deletions phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,12 @@

import javax.annotation.Nonnull;

import co.cask.tephra.TransactionManager;
import co.cask.tephra.TxConstants;
import co.cask.tephra.distributed.TransactionService;
import co.cask.tephra.metrics.TxMetricsCollector;
import co.cask.tephra.persist.InMemoryTransactionStateStorage;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
Expand All @@ -128,6 +134,7 @@
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory;
import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.LocalIndexMerger;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.util.Bytes;
Expand Down Expand Up @@ -163,12 +170,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import co.cask.tephra.TransactionManager;
import co.cask.tephra.TxConstants;
import co.cask.tephra.distributed.TransactionService;
import co.cask.tephra.metrics.TxMetricsCollector;
import co.cask.tephra.persist.InMemoryTransactionStateStorage;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
Expand Down Expand Up @@ -549,7 +550,6 @@ protected static String checkClusterInitialized(ReadOnlyProps overrideProps) thr
if (!clusterInitialized) {
url = setUpTestCluster(config, overrideProps);
clusterInitialized = true;
setupTxManager();
}
return url;
}
Expand All @@ -573,6 +573,7 @@ protected static void destroyDriver() throws Exception {
assertTrue(destroyDriver(driver));
} finally {
driver = null;
teardownTxManager();
}
}
}
Expand All @@ -594,12 +595,8 @@ protected static void tearDownMiniCluster() throws Exception {
utility.shutdownMiniCluster();
}
} finally {
try {
teardownTxManager();
} finally {
utility = null;
clusterInitialized = false;
}
utility = null;
clusterInitialized = false;
}
}
}
Expand All @@ -612,6 +609,9 @@ protected static void setUpTestDriver(ReadOnlyProps serverProps, ReadOnlyProps c
String url = checkClusterInitialized(serverProps);
if (driver == null) {
driver = initAndRegisterDriver(url, clientProps);
if (clientProps.getBoolean(QueryServices.TRANSACTIONS_ENABLED, QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED)) {
setupTxManager();
}
}
}

Expand Down Expand Up @@ -712,9 +712,6 @@ public static Configuration setUpConfigForMiniCluster(Configuration conf, ReadOn
conf.setInt("dfs.datanode.handler.count", 2);
conf.setInt("ipc.server.read.threadpool.size", 2);
conf.setInt("ipc.server.handler.threadpool.size", 2);
conf.setInt("hbase.hconnection.threads.max", 2);
conf.setInt("hbase.hconnection.threads.core", 2);
conf.setInt("hbase.htable.threads.max", 2);
conf.setInt("hbase.regionserver.hlog.syncer.count", 2);
conf.setInt("hbase.hlog.asyncer.number", 2);
conf.setInt("hbase.assignment.zkevent.workers", 5);
Expand All @@ -734,7 +731,8 @@ public static PhoenixTestDriver initAndRegisterDriver(String url, ReadOnlyProps
if (oldDriver != newDriver) {
destroyDriver(oldDriver);
}
Connection conn = newDriver.connect(url, PropertiesUtil.deepCopy(TEST_PROPERTIES));
Properties driverProps = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = newDriver.connect(url, driverProps);
conn.close();
return newDriver;
}
Expand Down
12 changes: 11 additions & 1 deletion phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
public static final boolean DEFAULT_RENEW_LEASE_ENABLED = false;
public static final int DEFAULT_INDEX_HANDLER_COUNT = 5;
public static final int DEFAULT_METADATA_HANDLER_COUNT = 5;
public static final int DEFAULT_HCONNECTION_POOL_CORE_SIZE = 10;
public static final int DEFAULT_HCONNECTION_POOL_MAX_SIZE = 10;
public static final int DEFAULT_HTABLE_MAX_THREADS = 10;


/**
Expand Down Expand Up @@ -99,7 +104,12 @@ private static QueryServicesOptions getDefaultServicesOptions() {
.setExtraJDBCArguments(DEFAULT_EXTRA_JDBC_ARGUMENTS)
.setRunUpdateStatsAsync(DEFAULT_RUN_UPDATE_STATS_ASYNC)
.setCommitStatsAsync(DEFAULT_COMMIT_STATS_ASYNC)
.setEnableRenewLease(DEFAULT_RENEW_LEASE_ENABLED);
.setEnableRenewLease(DEFAULT_RENEW_LEASE_ENABLED)
.setIndexHandlerCount(DEFAULT_INDEX_HANDLER_COUNT)
.setMetadataHandlerCount(DEFAULT_METADATA_HANDLER_COUNT)
.setHConnectionPoolCoreSize(DEFAULT_HCONNECTION_POOL_CORE_SIZE)
.setHConnectionPoolMaxSize(DEFAULT_HCONNECTION_POOL_MAX_SIZE)
.setMaxThreadsPerHTable(DEFAULT_HTABLE_MAX_THREADS);
}

public QueryServicesTestImpl(ReadOnlyProps defaultProps, ReadOnlyProps overrideProps) {
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@

<!-- Plugin options -->
<numForkedUT>3</numForkedUT>
<numForkedIT>5</numForkedIT>
<numForkedIT>7</numForkedIT>

<!-- Set default encoding so multi-byte tests work correctly on the Mac -->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand Down

0 comments on commit f3ea392

Please sign in to comment.