Skip to content
Permalink
Browse files
Merge pull request #149 from DomGarguilo/updateCI
* Add capability to change compaction planner/dispatcher
* Refactored prop name
* Remove external comps
  • Loading branch information
DomGarguilo committed Sep 2, 2021
2 parents a865eef + a51ef07 commit 77c65631fa4c94e29da2e239295feae127e6005a
Showing 3 changed files with 35 additions and 15 deletions.
@@ -39,12 +39,18 @@ test.ci.common.accumulo.num.tablets=20
# Optional authorizations that if specified will be randomly selected by scanners and walkers
# Format: a,b|a,b,c|c
test.ci.common.auths=
# Accumulo tserver properties to set when creating a table
test.ci.common.accumulo.server.props=\
tserver.compaction.major.service.cs1.planner=org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \
tserver.compaction.major.service.cs1.planner.opts.executors=\
[{"name":"small","type":"internal","maxSize":"16M","numThreads":8},\
{"name":"medium","type":"internal","maxSize":"128M","numThreads":4},\
{"name":"large","type":"internal","numThreads":2}]

# Accumulo table properties to set when creating table
test.ci.common.accumulo.table.props=\
table.majc.compaction.strategy=org.apache.accumulo.tserver.compaction.strategies.BasicCompactionStrategy \
table.majc.compaction.strategy.opts.filter.size=250M \
table.majc.compaction.strategy.opts.large.compress.threshold=100M \
table.majc.compaction.strategy.opts.large.compress.type=gz
table.compaction.dispatcher=org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher \
table.compaction.dispatcher.opts.service=cs1

# Ingest
# ------
@@ -56,9 +56,10 @@ public class TestProps {
// Number of tablets that should exist in Accumulo table when created
public static final String CI_COMMON_ACCUMULO_NUM_TABLETS = CI_COMMON + "accumulo.num.tablets";
// Optional authorizations (in CSV format) that if specified will be
// randomly selected by scanners
// and walkers
// randomly selected by scanners and walkers
public static final String CI_COMMON_AUTHS = CI_COMMON + "auths";
// Tserver props to set when a table is created
public static final String CI_COMMON_ACCUMULO_SERVER_PROPS = CI_COMMON + "accumulo.server.props";

/** Ingest **/
// Number of entries each ingest client should write
@@ -24,6 +24,8 @@
import java.util.TreeSet;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.testing.TestProps;
import org.apache.hadoop.io.Text;
@@ -55,9 +57,20 @@ public static void main(String[] args) throws Exception {
System.exit(-1);
}

// retrieve and set tserver props
Map<String,String> props = getProps(env, TestProps.CI_COMMON_ACCUMULO_SERVER_PROPS);
try {
for (Map.Entry<String,String> entry : props.entrySet()) {
client.instanceOperations().setProperty(entry.getKey(), entry.getValue());
}
} catch (AccumuloException | AccumuloSecurityException e) {
log.error("Failed to set tserver props");
throw new Exception(e);
}

SortedSet<Text> splits = new TreeSet<>();
int numSplits = numTablets - 1;
long distance = ((env.getRowMax() - env.getRowMin()) / numTablets) + 1;
final int numSplits = numTablets - 1;
final long distance = ((env.getRowMax() - env.getRowMin()) / numTablets) + 1;
long split = distance;
for (int i = 0; i < numSplits; i++) {
String s = String.format("%016x", split + env.getRowMin());
@@ -70,22 +83,22 @@ public static void main(String[] args) throws Exception {

NewTableConfiguration ntc = new NewTableConfiguration();
ntc.withSplits(splits);
ntc.setProperties(getTableProps(env));
ntc.setProperties(getProps(env, TestProps.CI_COMMON_ACCUMULO_TABLE_PROPS));

client.tableOperations().create(tableName, ntc);

log.info("Created Accumulo table {} with {} tablets", tableName, numTablets);
}
}

private static Map<String,String> getTableProps(ContinuousEnv env) {
String[] props = env.getTestProperty(TestProps.CI_COMMON_ACCUMULO_TABLE_PROPS).split(" ");
Map<String,String> tableProps = new HashMap<>();
for (String prop : props) {
private static Map<String,String> getProps(ContinuousEnv env, String propType) {
String[] propArray = env.getTestProperty(propType).split(" ");
Map<String,String> propMap = new HashMap<>();
for (String prop : propArray) {
log.debug("prop: {}", prop);
String[] kv = prop.split("=");
tableProps.put(kv[0], kv[1]);
propMap.put(kv[0], kv[1]);
}
return tableProps;
return propMap;
}
}

0 comments on commit 77c6563

Please sign in to comment.