Skip to content
Permalink
Browse files
Refactor out Non API code (#150)
Removed or replaced usages of non-API code
  • Loading branch information
DomGarguilo committed Sep 10, 2021
1 parent 77c6563 commit e89cd6dbffe6a5031ae65c52035ed818bff1fa38
Showing 12 changed files with 88 additions and 40 deletions.
@@ -37,18 +37,6 @@
<allow pkg="org.apache.accumulo.core.conf"/>
<allow class="org.apache.accumulo.core.util.threads.ThreadPools"/>

<!-- TODO refactor code to remove the following exceptions -->
<allow class="org.apache.accumulo.core.metadata.MetadataTable"/>
<allow class="org.apache.accumulo.core.replication.ReplicationTable"/>
<allow class="org.apache.accumulo.core.spi.scan.HintScanPrioritizer"/>
<allow class="org.apache.accumulo.core.clientImpl.TabletServerBatchWriter"/>
<allow class="org.apache.accumulo.core.util.FastFormat"/>
<allow class="org.apache.accumulo.core.util.Pair"/>
<allow class="org.apache.accumulo.core.trace.Trace"/>
<allow class="org.apache.accumulo.core.trace.TraceSamplers"/>
<allow class="org.apache.accumulo.core.trace.Span"/>
<allow class="org.apache.accumulo.core.trace.DistributedTrace"/>
<!-- End TODO section -->

<!-- disallow everything else coming from accumulo -->
<disallow pkg="org.apache.accumulo"/>
@@ -46,6 +46,10 @@ public class TestProps {
public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb";
// Number of cores given to each YARN container
public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores";
// Name of metadata table
public static final String METADATA_TABLE_NAME = "accumulo.metadata";
// Name of replication table
public static final String REPLICATION_TABLE_NAME = "accumulo.replication";

/** Continuous ingest test properties **/
/** Common **/
@@ -38,8 +38,8 @@
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.testing.TestProps;
import org.apache.accumulo.testing.util.FastFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -35,7 +35,6 @@
import org.apache.accumulo.core.client.rfile.RFile;
import org.apache.accumulo.core.client.rfile.RFileWriter;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
import org.apache.accumulo.core.clientImpl.TabletServerBatchWriter;
import org.apache.accumulo.core.conf.ClientProperty;
import org.apache.accumulo.core.data.ConstraintViolationSummary;
import org.apache.accumulo.core.data.Key;
@@ -44,8 +43,8 @@
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.testing.cli.ClientOpts;
import org.apache.accumulo.testing.util.FastFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
@@ -184,7 +183,8 @@ public static void main(String[] args) {
try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build()) {

if (opts.debug)
Logger.getLogger(TabletServerBatchWriter.class.getName()).setLevel(Level.TRACE);
Logger.getLogger("org.apache.accumulo.core.clientImpl.TabletServerBatchWriter")
.setLevel(Level.TRACE);

// test batch update

@@ -32,8 +32,6 @@
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
// import org.apache.accumulo.core.trace.DistributedTrace;
// import org.apache.accumulo.core.trace.Trace;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,18 +60,9 @@ public static void main(String[] args) throws Exception {
opts.parseArgs(VerifyIngest.class.getName(), args);
try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build()) {
if (opts.trace) {
/*
* String name = VerifyIngest.class.getSimpleName(); DistributedTrace.enable();
* Trace.on(name); Trace.data("cmdLine", Arrays.asList(args).toString());
*/
throw new UnsupportedOperationException("Tracing is enabled but not currently supported");
}

verifyIngest(client, opts);

} finally {
/*
* Trace.off(); DistributedTrace.disable();
*/
}
}

@@ -21,7 +21,7 @@
import java.util.Arrays;
import java.util.List;

import org.apache.accumulo.core.util.Pair;
import org.apache.commons.math3.util.Pair;

import com.google.common.collect.Iterables;

@@ -22,7 +22,7 @@
import java.util.TreeSet;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.testing.TestProps;
import org.apache.accumulo.testing.performance.Environment;
import org.apache.accumulo.testing.performance.PerformanceTest;
import org.apache.accumulo.testing.performance.Report;
@@ -53,7 +53,7 @@ public Report runTest(final Environment env) throws Exception {

AccumuloClient client = env.getClient();
client.tableOperations().create(TABLE_NAME);
client.tableOperations().addSplits(MetadataTable.NAME, getMetadataTableSplits());
client.tableOperations().addSplits(TestProps.METADATA_TABLE_NAME, getMetadataTableSplits());

SortedSet<Text> splits = getTestTableSplits();

@@ -35,7 +35,6 @@
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.spi.scan.HintScanPrioritizer;
import org.apache.accumulo.testing.performance.Environment;
import org.apache.accumulo.testing.performance.PerformanceTest;
import org.apache.accumulo.testing.performance.Report;
@@ -57,7 +56,7 @@ public class ScanExecutorPT implements PerformanceTest {
private static final int NUM_QUALS = 10;

private static final String SCAN_EXECUTOR_THREADS = "2";
private static final String SCAN_PRIORITIZER = HintScanPrioritizer.class.getName();
private static final String SCAN_PRIORITIZER = "org.apache.accumulo.core.spi.scan.HintScanPrioritizer";

private static final String TEST_DESC = "Scan Executor Test. Test running lots of short scans "
+ "while long scans are running in the background. Each short scan reads a random row and "
@@ -27,8 +27,8 @@
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.testing.TestProps;
import org.apache.accumulo.testing.performance.Environment;
import org.apache.accumulo.testing.performance.PerformanceTest;
import org.apache.accumulo.testing.performance.Report;
@@ -97,7 +97,8 @@ private SortedSet<Text> getSplits() {

private Map<String,Integer> getTablets(final AccumuloClient client) {
Map<String,Integer> tablets = new HashMap<>();
try (Scanner scanner = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
try (Scanner scanner = client.createScanner(TestProps.METADATA_TABLE_NAME,
Authorizations.EMPTY)) {
scanner.fetchColumnFamily(TSERVER_ASSIGNED_TABLETS_COL_FAM);
Range range = new Range(null, false, RESERVED_PREFIX, false);
scanner.setRange(range);
@@ -20,7 +20,7 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.testing.util.FastFormat;

public class TestData {

@@ -46,8 +46,8 @@
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.replication.ReplicationTable;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.testing.TestProps;
import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.randomwalk.State;
import org.apache.accumulo.testing.randomwalk.Test;
@@ -80,13 +80,13 @@ public void visit(State state, RandWalkEnv env, Properties props) throws Excepti
iOps.setProperty(REPLICATION_WORK_PROCESSOR_PERIOD.getKey(), "1s");

// Ensure the replication table is online
ReplicationTable.setOnline(c);
boolean online = ReplicationTable.isOnline(c);
tOps.online(TestProps.REPLICATION_TABLE_NAME, true);
boolean online = tOps.isOnline(TestProps.REPLICATION_TABLE_NAME);
for (int i = 0; i < 10; i++) {
if (online)
break;
sleepUninterruptibly(2, TimeUnit.SECONDS);
online = ReplicationTable.isOnline(c);
online = tOps.isOnline(TestProps.REPLICATION_TABLE_NAME);
}
assertTrue("Replication table was not online", online);

@@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.testing.util;

import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.base.Preconditions;

public class FastFormat {
// Copied over from accumulo
// this 7 to 8 times faster than String.format("%s%06d",prefix, num)
public static byte[] toZeroPaddedString(long num, int width, int radix, byte[] prefix) {
Preconditions.checkArgument(num >= 0);
String strNum = Long.toString(num, radix);
byte[] ret = new byte[Math.max(strNum.length(), width) + prefix.length];
if (toZeroPaddedString(ret, 0, strNum, width, prefix) != ret.length)
throw new RuntimeException(" Did not format to expected width " + num + " " + width + " "
+ radix + " " + new String(prefix, UTF_8));
return ret;
}

public static int toZeroPaddedString(byte[] output, int outputOffset, long num, int width,
int radix, byte[] prefix) {
Preconditions.checkArgument(num >= 0);

String strNum = Long.toString(num, radix);

return toZeroPaddedString(output, outputOffset, strNum, width, prefix);
}

private static int toZeroPaddedString(byte[] output, int outputOffset, String strNum, int width,
byte[] prefix) {

int index = outputOffset;

for (byte b : prefix) {
output[index++] = b;
}

int end = width - strNum.length() + index;

while (index < end)
output[index++] = '0';

for (int i = 0; i < strNum.length(); i++) {
output[index++] = (byte) strNum.charAt(i);
}

return index - outputOffset;
}
}

0 comments on commit e89cd6d

Please sign in to comment.