Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,8 @@ void compact(String tableName, Text start, Text end, List<IteratorSetting> itera

/**
* Starts a full major compaction of the tablets in the range (start, end]. If the config does not
* specify a compaction selector (or a deprecated strategy), then all files in a tablet are
* compacted. The compaction is performed even for tablets that have only one file.
* specify a compaction selector, then all files in a tablet are compacted. The compaction is
* performed even for tablets that have only one file.
*
* <p>
* The following optional settings can only be set by one compact call per table at the same time.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.core.compaction;

import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.accumulo.core.client.admin.compaction.CompactionConfigurer;
import org.apache.accumulo.core.conf.Property;

/**
* The compaction configurer is used by the shell compact command. It exists in accumulo-core, so it
* is on the class path for the shell and servers that run compactions.
*/
public class ShellCompactCommandConfigurer implements CompactionConfigurer {

private Map<String,String> overrides = new HashMap<>();

@Override
public void init(InitParameters iparams) {
Set<Entry<String,String>> es = iparams.getOptions().entrySet();
for (Entry<String,String> entry : es) {

switch (CompactionSettings.valueOf(entry.getKey())) {
case OUTPUT_COMPRESSION_OPT:
overrides.put(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), entry.getValue());
break;
case OUTPUT_BLOCK_SIZE_OPT:
overrides.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), entry.getValue());
break;
case OUTPUT_INDEX_BLOCK_SIZE_OPT:
overrides.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), entry.getValue());
break;
case OUTPUT_HDFS_BLOCK_SIZE_OPT:
overrides.put(Property.TABLE_FILE_BLOCK_SIZE.getKey(), entry.getValue());
break;
case OUTPUT_REPLICATION_OPT:
overrides.put(Property.TABLE_FILE_REPLICATION.getKey(), entry.getValue());
break;
default:
throw new IllegalArgumentException("Unknown option " + entry.getKey());
}
}
}

@Override
public Overrides override(InputParameters params) {
return new Overrides(overrides);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.tserver.compaction.strategies;
package org.apache.accumulo.core.compaction;

import java.util.ArrayList;
import java.util.Collection;
Expand All @@ -32,23 +32,21 @@
import java.util.stream.Collectors;

import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.client.admin.compaction.CompactionConfigurer;
import org.apache.accumulo.core.client.admin.compaction.CompactionSelector;
import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
import org.apache.accumulo.core.client.summary.Summary;
import org.apache.accumulo.core.compaction.CompactionSettings;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.hadoop.fs.Path;

/**
* The compaction strategy used by the shell compact command.
* The compaction selector is used by the shell compact command. It exists in accumulo-core, so it
* is on the class path for the shell and servers that run compactions.
*/
public class ConfigurableCompactionStrategy implements CompactionSelector, CompactionConfigurer {
public class ShellCompactCommandSelector implements CompactionSelector {

private abstract static class Test {
abstract Set<CompactableFile> getFilesToCompact(SelectionParameters params);
Expand Down Expand Up @@ -168,45 +166,9 @@ Set<CompactableFile> getFilesToCompact(SelectionParameters params) {
private List<Test> tests = new ArrayList<>();
private boolean andTest = true;
private int minFiles = 1;
private Map<String,String> overrides = new HashMap<>();

@Override
public void init(
org.apache.accumulo.core.client.admin.compaction.CompactionConfigurer.InitParameters iparams) {
Set<Entry<String,String>> es = iparams.getOptions().entrySet();
for (Entry<String,String> entry : es) {

switch (CompactionSettings.valueOf(entry.getKey())) {
case OUTPUT_COMPRESSION_OPT:
overrides.put(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), entry.getValue());
break;
case OUTPUT_BLOCK_SIZE_OPT:
overrides.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), entry.getValue());
break;
case OUTPUT_INDEX_BLOCK_SIZE_OPT:
overrides.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), entry.getValue());
break;
case OUTPUT_HDFS_BLOCK_SIZE_OPT:
overrides.put(Property.TABLE_FILE_BLOCK_SIZE.getKey(), entry.getValue());
break;
case OUTPUT_REPLICATION_OPT:
overrides.put(Property.TABLE_FILE_REPLICATION.getKey(), entry.getValue());
break;
default:
throw new IllegalArgumentException("Unknown option " + entry.getKey());
}
}

}

@Override
public Overrides override(InputParameters params) {
return new Overrides(overrides);
}

@Override
public void init(
org.apache.accumulo.core.client.admin.compaction.CompactionSelector.InitParameters iparams) {
public void init(InitParameters iparams) {
boolean selectNoSummary = false;
boolean selectExtraSummary = false;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -733,9 +733,6 @@ public enum Property {
"The listening port for the garbage collector's monitor service", "1.3.5"),
GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT,
"The number of threads used to delete RFiles and write-ahead logs", "1.3.5"),
@Deprecated(since = "2.1.1", forRemoval = true)
GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN,
"Do not use the Trash, even if it is configured.", "1.5.0"),
GC_SAFEMODE("gc.safemode", "false", PropertyType.BOOLEAN,
"Provides listing of files to be deleted but does not delete any files", "2.1.0"),
GC_USE_FULL_COMPACTION("gc.post.metadata.action", "flush", PropertyType.GC_POST_ACTION,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,6 @@ OverridesOption fromUrl(URL propertiesFileUrl) {
}

public OverridesOption fromEnv() {
URL siteUrl = SiteConfiguration.class.getClassLoader().getResource("accumulo-site.xml");
if (siteUrl != null) {
throw new IllegalArgumentException("Found deprecated config file 'accumulo-site.xml' on "
+ "classpath. Since 2.0.0, this file was replaced by 'accumulo.properties'. Run the "
+ "following command to convert an old 'accumulo-site.xml' file to the new format: "
+ "accumulo convert-config -x /old/accumulo-site.xml -p /new/accumulo.properties");
}

String configFile = System.getProperty("accumulo.properties", "accumulo.properties");
if (configFile.startsWith("file://")) {
File f;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,6 @@
* to periodically select files to compact. This supports use cases like periodically compacting all
* files because there are too many deletes. See
* {@link org.apache.accumulo.core.client.admin.compaction.CompactionSelector}
* <li><b>Compaction Strategy</b> A deprecated pluggable component replaced by the Selector and
* Configurer. See {@code org.apache.accumulo.core.client.admin.CompactionStrategyConfig}'s own
* documentation for more information about why it was deprecated.
* </ul>
* </ul>
* </ul>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.tserver.compaction.strategies;
package org.apache.accumulo.core.compaction;

import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getFixedMemoryAsBytes;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand All @@ -33,15 +33,12 @@
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.client.admin.compaction.CompactionConfigurer;
import org.apache.accumulo.core.client.admin.compaction.CompactionConfigurer.Overrides;
import org.apache.accumulo.core.compaction.CompactionSettings;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.TabletId;
import org.junit.jupiter.api.Test;

public class ConfigurableCompactionStrategyTest {

// file selection options are adequately tested by ShellServerIT
public class ShellCompactCommandConfigurerTest {

@Test
public void testOutputOptions() throws URISyntaxException {
Expand All @@ -50,7 +47,7 @@ public void testOutputOptions() throws URISyntaxException {
.create(new URI("hdfs://nn1/accumulo/tables/1/t-009/F00001.rf"), 50000, 400));

// test setting no output options
ConfigurableCompactionStrategy ccs = new ConfigurableCompactionStrategy();
ShellCompactCommandConfigurer ccs = new ShellCompactCommandConfigurer();

Map<String,String> opts = new HashMap<>();

Expand Down Expand Up @@ -102,7 +99,7 @@ public PluginEnvironment getEnvironment() {
assertTrue(plan.getOverrides().isEmpty());

// test setting all output options
ccs = new ConfigurableCompactionStrategy();
ccs = new ShellCompactCommandConfigurer();

CompactionSettings.OUTPUT_BLOCK_SIZE_OPT.put(null, opts, "64K");
CompactionSettings.OUTPUT_COMPRESSION_OPT.put(null, opts, "snappy");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.core.compaction;

import java.net.URISyntaxException;

import org.junit.jupiter.api.Test;

public class ShellCompactCommandSelectorTest {

@Test
public void testSelection() throws URISyntaxException {
// file selection options are adequately tested by ShellServerIT, so this is just a placeholder
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -163,17 +163,6 @@ static class DumpConfigCommand {
boolean users = false;
}

private static final String RV_DEPRECATION_MSG =
"Randomizing tablet directories is deprecated and now does nothing. Accumulo now always"
+ " calls the volume chooser for each file created by a tablet, so its no longer "
+ "necessary.";

@Parameters(commandDescription = RV_DEPRECATION_MSG)
static class RandomizeVolumesCommand {
@Parameter(names = {"-t"}, description = "table to update", required = true)
String tableName = null;
}

@Parameters(commandDescription = "Verify all Tablets are assigned to tablet servers")
static class VerifyTabletAssignmentsCommand {
@Parameter(names = {"-v", "--verbose"},
Expand Down Expand Up @@ -306,9 +295,6 @@ public void execute(final String[] args) {
RestoreZooCommand restoreZooOpts = new RestoreZooCommand();
cl.addCommand("restoreZoo", restoreZooOpts);

RandomizeVolumesCommand randomizeVolumesOpts = new RandomizeVolumesCommand();
cl.addCommand("randomizeVolumes", randomizeVolumesOpts);

StopCommand stopOpts = new StopCommand();
cl.addCommand("stop", stopOpts);

Expand Down Expand Up @@ -375,8 +361,6 @@ public void execute(final String[] args) {
printConfig(context, dumpConfigCommand);
} else if (cl.getParsedCommand().equals("volumes")) {
ListVolumesUsed.listVolumes(context);
} else if (cl.getParsedCommand().equals("randomizeVolumes")) {
System.out.println(RV_DEPRECATION_MSG);
} else if (cl.getParsedCommand().equals("verifyTabletAssigns")) {
VerifyTabletAssignments.execute(opts.getClientProps(), verifyTabletAssignmentsOpts.verbose);
} else if (cl.getParsedCommand().equals("changeSecret")) {
Expand Down
Loading