From f89e386148173dbd902765da711c0c553fd37fa6 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Thu, 12 Nov 2015 23:31:22 -0500 Subject: [PATCH 1/3] ACCUMULO-2883 Added locating tablets to API --- .../accumulo/core/client/admin/Locations.java | 49 +++++++ .../core/client/admin/TableOperations.java | 14 ++ .../core/client/impl/TableOperationsImpl.java | 112 +++++++++++++++ .../core/client/mock/MockTableOperations.java | 6 + .../apache/accumulo/core/data/TabletId.java | 7 + .../accumulo/core/data/impl/TabletIdImpl.java | 6 + .../impl/TableOperationsHelperTest.java | 6 + .../org/apache/accumulo/test/LocatorIT.java | 130 ++++++++++++++++++ 8 files changed, 330 insertions(+) create mode 100644 core/src/main/java/org/apache/accumulo/core/client/admin/Locations.java create mode 100644 test/src/main/java/org/apache/accumulo/test/LocatorIT.java diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/Locations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/Locations.java new file mode 100644 index 00000000000..11a02542ec9 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/Locations.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.client.admin; + +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TabletId; + +/** + * A snapshot of metadata information about where a specified set of ranges are located returned by {@link TableOperations#locate(String, java.util.Collection)} + * + * @since 1.8.0 + */ +public interface Locations { + + /** + * For all of the ranges passed to {@link TableOperations#locate(String, java.util.Collection)}, return a map of the tablets each range overlaps. + */ + public Map> groupByRange(); + + /** + * For all of the ranges passed to {@link TableOperations#locate(String, java.util.Collection)}, return a map of the ranges each tablet overlaps. + */ + public Map> groupByTablet(); + + /** + * For any TabletId returned by {@link #getOverlappingTablets(Range)}, the method will return the tablet server location for that tablet. + * + * @return A tablet server location in the form of {@code :} + */ + public String getTabletLocation(TabletId tabletId); +} diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java index fa6fef40ebe..b20de490348 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.TableOfflineException; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.security.Authorizations; @@ -230,6 +231,19 @@ public interface TableOperations { */ Collection listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException; + /** + * Locates the tablet servers and tablets that would service a collections of ranges. If a range covers multiple tablets, it will occur multiple times in the + * returned map. + * + * @param ranges + * The input ranges that should be mapped to tablet servers and tablets. + * + * @throws TableOfflineException + * if the table is offline or goes offline during the operation + * @since 1.8.0 + */ + Locations locate(String tableName, Collection ranges) throws AccumuloException, AccumuloSecurityException, TableNotFoundException; + /** * Finds the max row within a given range. To find the max row in a table, pass null for start and end row. * diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java index 8434f2f5770..b8ca62619e1 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java @@ -66,6 +66,7 @@ import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.DiskUsage; import org.apache.accumulo.core.client.admin.FindMax; +import org.apache.accumulo.core.client.admin.Locations; import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.SamplerConfiguration; import org.apache.accumulo.core.client.admin.TableOperations; @@ -83,8 +84,10 @@ import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.data.impl.KeyExtent; +import org.apache.accumulo.core.data.impl.TabletIdImpl; import org.apache.accumulo.core.iterators.IteratorUtil; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; @@ -109,6 +112,7 @@ import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.volume.VolumeConfiguration; +import org.apache.accumulo.fate.zookeeper.Retry; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -120,6 +124,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.net.HostAndPort; public class TableOperationsImpl extends TableOperationsHelper { @@ -1513,4 +1518,111 @@ public SamplerConfiguration getSamplerConfiguration(String tableName) throws Tab return sci.toSamplerConfiguration(); } + private static class LoctionsImpl implements Locations { + + private Map> groupedByRanges; + private Map> groupedByTablets; + private Map tabletLocations; + + public LoctionsImpl(Map>> binnedRanges) { + groupedByTablets = new HashMap<>(); + groupedByRanges = null; + tabletLocations = new HashMap<>(); + + for (Entry>> entry : binnedRanges.entrySet()) { + String location = entry.getKey(); + + for (Entry> entry2 : entry.getValue().entrySet()) { + TabletIdImpl tabletId = new TabletIdImpl(entry2.getKey()); + tabletLocations.put(tabletId, location); + List prev = groupedByTablets.put(tabletId, Collections.unmodifiableList(entry2.getValue())); + if (prev != null) { + throw new RuntimeException("Unexpected : tablet at multiple locations : " + location + " " + tabletId); + } + } + } + + groupedByTablets = Collections.unmodifiableMap(groupedByTablets); + } + + @Override + public String getTabletLocation(TabletId tabletId) { + return tabletLocations.get(tabletId); + } + + @Override + public Map> groupByRange() { + if (groupedByRanges == null) { + Map> tmp = new HashMap<>(); + + for (Entry> entry : groupedByTablets.entrySet()) { + for (Range range : entry.getValue()) { + List tablets = tmp.get(range); + if (tablets == null) { + tablets = new ArrayList<>(); + tmp.put(range, tablets); + } + + tablets.add(entry.getKey()); + } + } + + Map> tmp2 = new HashMap<>(); + for (Entry> entry : tmp.entrySet()) { + tmp2.put(entry.getKey(), Collections.unmodifiableList(entry.getValue())); + } + + groupedByRanges = Collections.unmodifiableMap(tmp2); + } + + return groupedByRanges; + } + + @Override + public Map> groupByTablet() { + return groupedByTablets; + } + } + + @Override + public Locations locate(String tableName, Collection ranges) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + Preconditions.checkNotNull(tableName, "tableName must be non null"); + Preconditions.checkNotNull(ranges, "ranges must be non null"); + + String tableId = Tables.getTableId(context.getInstance(), tableName); + TabletLocator locator = TabletLocator.getLocator(context, new Text(tableId)); + + List rangeList = null; + if (ranges instanceof List) { + rangeList = (List) ranges; + } else { + rangeList = new ArrayList<>(ranges); + } + + Map>> binnedRanges = new HashMap<>(); + + locator.invalidateCache(); + + Retry retry = new Retry(Long.MAX_VALUE, 100, 100, 2000); + + while (!locator.binRanges(context, rangeList, binnedRanges).isEmpty()) { + + if (!Tables.exists(context.getInstance(), tableId)) + throw new TableNotFoundException(tableId, tableName, null); + if (Tables.getTableState(context.getInstance(), tableId) == TableState.OFFLINE) + throw new TableOfflineException(context.getInstance(), tableId); + + binnedRanges.clear(); + + try { + retry.waitForNextAttempt(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + locator.invalidateCache(); + } + + return new LoctionsImpl(binnedRanges); + } } diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java index 7ca5766a721..41b46039987 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.DiskUsage; import org.apache.accumulo.core.client.admin.FindMax; +import org.apache.accumulo.core.client.admin.Locations; import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.SamplerConfiguration; import org.apache.accumulo.core.client.admin.TimeType; @@ -497,4 +498,9 @@ public void clearSamplerConfiguration(String tableName) throws TableNotFoundExce public SamplerConfiguration getSamplerConfiguration(String tableName) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { throw new UnsupportedOperationException(); } + + @Override + public Locations locate(String tableName, Collection ranges) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + throw new UnsupportedOperationException(); + } } diff --git a/core/src/main/java/org/apache/accumulo/core/data/TabletId.java b/core/src/main/java/org/apache/accumulo/core/data/TabletId.java index 113183d3aca..15fd8d66ed9 100644 --- a/core/src/main/java/org/apache/accumulo/core/data/TabletId.java +++ b/core/src/main/java/org/apache/accumulo/core/data/TabletId.java @@ -30,4 +30,11 @@ public interface TabletId extends Comparable { public Text getEndRow(); public Text getPrevEndRow(); + + /** + * @since 1.8.0 + */ + + public Range toRange(); + } diff --git a/core/src/main/java/org/apache/accumulo/core/data/impl/TabletIdImpl.java b/core/src/main/java/org/apache/accumulo/core/data/impl/TabletIdImpl.java index 61e882a867e..41ff3f523a9 100644 --- a/core/src/main/java/org/apache/accumulo/core/data/impl/TabletIdImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/data/impl/TabletIdImpl.java @@ -17,6 +17,7 @@ package org.apache.accumulo.core.data.impl; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TabletId; import org.apache.hadoop.io.Text; @@ -97,4 +98,9 @@ public boolean equals(Object o) { public String toString() { return ke.toString(); } + + @Override + public Range toRange() { + return ke.toDataRange(); + } } diff --git a/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java b/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java index 7bf9eb17bdc..86857fa9339 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java @@ -35,6 +35,7 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.DiskUsage; +import org.apache.accumulo.core.client.admin.Locations; import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.SamplerConfiguration; import org.apache.accumulo.core.client.admin.TimeType; @@ -243,6 +244,11 @@ public void clearSamplerConfiguration(String tableName) throws TableNotFoundExce public SamplerConfiguration getSamplerConfiguration(String tableName) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { throw new UnsupportedOperationException(); } + + @Override + public Locations locate(String tableName, Collection ranges) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + throw new UnsupportedOperationException(); + } } protected TableOperationsHelper getHelper() { diff --git a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java new file mode 100644 index 00000000000..193d0d4ac54 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.TableOfflineException; +import org.apache.accumulo.core.client.admin.Locations; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TabletId; +import org.apache.accumulo.core.data.impl.KeyExtent; +import org.apache.accumulo.core.data.impl.TabletIdImpl; +import org.apache.accumulo.harness.AccumuloClusterHarness; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +public class LocatorIT extends AccumuloClusterHarness { + + private void assertContains(Locations locations, HashSet tservers, Map> expected1, + Map> expected2) { + + Map> gbr = new HashMap<>(); + for (Entry> entry : locations.groupByRange().entrySet()) { + gbr.put(entry.getKey(), new HashSet<>(entry.getValue())); + } + + Assert.assertEquals(expected1, gbr); + + Map> gbt = new HashMap<>(); + for (Entry> entry : locations.groupByTablet().entrySet()) { + gbt.put(entry.getKey(), new HashSet<>(entry.getValue())); + + TabletId tid = entry.getKey(); + String location = locations.getTabletLocation(tid); + Assert.assertNotNull("Location for " + tid + " was null", location); + Assert.assertTrue("Unknown location " + location, tservers.contains(location)); + Assert.assertTrue("Expected : " + location, location.split(":").length == 2); + + } + + Assert.assertEquals(expected2, gbt); + } + + private static TabletId newTabletId(String tableId, String endRow, String prevRow) { + return new TabletIdImpl(new KeyExtent(new Text(tableId), endRow == null ? null : new Text(endRow), prevRow == null ? null : new Text(prevRow))); + } + + @Test + public void testBasic() throws Exception { + Connector conn = getConnector(); + String tableName = getUniqueNames(1)[0]; + + conn.tableOperations().create(tableName); + + Range r1 = new Range("m"); + Range r2 = new Range("o", "x"); + + String tableId = conn.tableOperations().tableIdMap().get(tableName); + + TabletId t1 = newTabletId(tableId, null, null); + TabletId t2 = newTabletId(tableId, "r", null); + TabletId t3 = newTabletId(tableId, null, "r"); + + ArrayList ranges = new ArrayList<>(); + + HashSet tservers = new HashSet<>(conn.instanceOperations().getTabletServers()); + + ranges.add(r1); + Locations ret = conn.tableOperations().locate(tableName, ranges); + assertContains(ret, tservers, ImmutableMap.of(r1, ImmutableSet.of(t1)), ImmutableMap.of(t1, ImmutableSet.of(r1))); + + ranges.add(r2); + ret = conn.tableOperations().locate(tableName, ranges); + assertContains(ret, tservers, ImmutableMap.of(r1, ImmutableSet.of(t1), r2, ImmutableSet.of(t1)), ImmutableMap.of(t1, ImmutableSet.of(r1, r2))); + + TreeSet splits = new TreeSet(); + splits.add(new Text("r")); + conn.tableOperations().addSplits(tableName, splits); + + ret = conn.tableOperations().locate(tableName, ranges); + assertContains(ret, tservers, ImmutableMap.of(r1, ImmutableSet.of(t2), r2, ImmutableSet.of(t2, t3)), + ImmutableMap.of(t2, ImmutableSet.of(r1, r2), t3, ImmutableSet.of(r2))); + + conn.tableOperations().offline(tableName, true); + + try { + conn.tableOperations().locate(tableName, ranges); + Assert.fail(); + } catch (TableOfflineException e) { + // expected + } + + conn.tableOperations().delete(tableName); + + try { + conn.tableOperations().locate(tableName, ranges); + Assert.fail(); + } catch (TableNotFoundException e) { + // expected + } + } +} From 6c96e2b005ff02c4983eac5efb5dee72292c50ff Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 20 Nov 2015 12:20:01 -0500 Subject: [PATCH 2/3] ACCUMULO-3913 exclude test rfile from rat check --- core/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/core/pom.xml b/core/pom.xml index 5ab52553850..523ef935669 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -155,6 +155,7 @@ src/test/resources/org/apache/accumulo/core/file/rfile/ver_3.rf src/test/resources/org/apache/accumulo/core/file/rfile/ver_4.rf src/test/resources/org/apache/accumulo/core/file/rfile/ver_6.rf + src/test/resources/org/apache/accumulo/core/file/rfile/ver_7.rf src/test/resources/site-cfg.jceks src/test/resources/accumulo.jceks src/test/resources/empty.jceks From cf2fde36d7d2457832ff0a72a3373ec1725e7b24 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 8 Dec 2015 13:28:32 -0500 Subject: [PATCH 3/3] add some javadocs --- core/src/main/java/org/apache/accumulo/core/data/TabletId.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/accumulo/core/data/TabletId.java b/core/src/main/java/org/apache/accumulo/core/data/TabletId.java index 15fd8d66ed9..8680760b87d 100644 --- a/core/src/main/java/org/apache/accumulo/core/data/TabletId.java +++ b/core/src/main/java/org/apache/accumulo/core/data/TabletId.java @@ -32,9 +32,9 @@ public interface TabletId extends Comparable { public Text getPrevEndRow(); /** + * @return a range based on the row range of the tablet. The range will cover {@code (, ]}. * @since 1.8.0 */ - public Range toRange(); }