Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

ACCUMULO-863 reduced block index entry to just one key

git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1426414 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
commit a1afdead746d3ac983a43d962f61ded1b8370332 1 parent e0da4d0
Keith Turner keith-turner authored
30 core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
@@ -57,15 +57,13 @@ private static boolean isPowerOfTwo(int x) {
57 57
58 58 public static class BlockIndexEntry implements Comparable<BlockIndexEntry> {
59 59
60   - private Key key;
61 60 private Key prevKey;
62 61 private int entriesLeft;
63 62 private int pos;
64 63
65   - public BlockIndexEntry(int pos, int entriesLeft, Key key, Key prevKey) {
  64 + public BlockIndexEntry(int pos, int entriesLeft, Key prevKey) {
66 65 this.pos = pos;
67 66 this.entriesLeft = entriesLeft;
68   - this.key = key;
69 67 this.prevKey = prevKey;
70 68 }
71 69
@@ -73,12 +71,10 @@ public BlockIndexEntry(int pos, int entriesLeft, Key key, Key prevKey) {
73 71 * @param key
74 72 */
75 73 public BlockIndexEntry(Key key) {
76   - this.key = key;
  74 + this.prevKey = key;
77 75 }
78 76
79   - public Key getKey() {
80   - return key;
81   - }
  77 +
82 78
83 79 public int getEntriesLeft() {
84 80 return entriesLeft;
@@ -86,12 +82,12 @@ public int getEntriesLeft() {
86 82
87 83 @Override
88 84 public int compareTo(BlockIndexEntry o) {
89   - return key.compareTo(o.key);
  85 + return prevKey.compareTo(o.prevKey);
90 86 }
91 87
92 88 @Override
93 89 public String toString() {
94   - return key + " " + entriesLeft + " " + pos;
  90 + return prevKey + " " + entriesLeft + " " + pos;
95 91 }
96 92
97 93 public Key getPrevKey() {
@@ -116,18 +112,24 @@ public BlockIndexEntry seekBlock(Key startKey, ABlockReader cacheBlock) {
116 112 } else {
117 113 // found exact key in index
118 114 index = pos;
  115 + while (index > 0) {
  116 + if (blockIndex[index].getPrevKey().equals(startKey))
  117 + index--;
  118 + else
  119 + break;
  120 + }
119 121 }
120 122
121 123 // handle case where multiple keys in block are exactly the same, want to find the earliest key in the index
122 124 while (index - 1 > 0) {
123   - if (blockIndex[index].getKey().equals(blockIndex[index - 1].getKey()))
  125 + if (blockIndex[index].getPrevKey().equals(blockIndex[index - 1].getPrevKey()))
124 126 index--;
125 127 else
126 128 break;
127 129
128 130 }
129 131
130   - if (index == 0 && blockIndex[index].getKey().equals(startKey))
  132 + if (index == 0 && blockIndex[index].getPrevKey().equals(startKey))
131 133 return null;
132 134
133 135 BlockIndexEntry bie = blockIndex[index];
@@ -162,7 +164,7 @@ private synchronized void buildIndex(int indexEntries, ABlockReader cacheBlock,
162 164 val.readFields(cacheBlock);
163 165
164 166 if (count > 0 && count % interval == 0) {
165   - index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count, rk.getKey(), myPrevKey));
  167 + index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count, myPrevKey));
166 168 }
167 169
168 170 count++;
@@ -172,4 +174,8 @@ private synchronized void buildIndex(int indexEntries, ABlockReader cacheBlock,
172 174
173 175 cacheBlock.seek(0);
174 176 }
  177 +
  178 + BlockIndexEntry[] getIndexEntries() {
  179 + return blockIndex;
  180 + }
175 181 }
2  core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
@@ -751,7 +751,7 @@ private void _seek(Range range) throws IOException {
751 751 // just consumed one key from the input stream, so subtract one from entries left
752 752 entriesLeft = bie.getEntriesLeft() - 1;
753 753 prevKey = new Key(bie.getPrevKey());
754   - currKey = bie.getKey();
  754 + currKey = tmpRk.getKey();
755 755 }
756 756 }
757 757 }
175 core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java
... ... @@ -0,0 +1,175 @@
  1 +/**
  2 + * Licensed to the Apache Software Foundation (ASF) under one or more
  3 + * contributor license agreements. See the NOTICE file distributed with
  4 + * this work for additional information regarding copyright ownership.
  5 + * The ASF licenses this file to You under the Apache License, Version 2.0
  6 + * (the "License"); you may not use this file except in compliance with
  7 + * the License. You may obtain a copy of the License at
  8 + *
  9 + * http://www.apache.org/licenses/LICENSE-2.0
  10 + *
  11 + * Unless required by applicable law or agreed to in writing, software
  12 + * distributed under the License is distributed on an "AS IS" BASIS,
  13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 + * See the License for the specific language governing permissions and
  15 + * limitations under the License.
  16 + */
  17 +package org.apache.accumulo.core.file.rfile;
  18 +
  19 +import java.io.ByteArrayOutputStream;
  20 +import java.io.DataOutputStream;
  21 +import java.io.IOException;
  22 +
  23 +import org.apache.accumulo.core.data.Key;
  24 +import org.apache.accumulo.core.data.Value;
  25 +import org.apache.accumulo.core.file.blockfile.ABlockReader;
  26 +import org.apache.accumulo.core.file.blockfile.cache.CacheEntry;
  27 +import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
  28 +import org.apache.accumulo.core.file.rfile.BlockIndex.BlockIndexEntry;
  29 +import org.apache.accumulo.core.file.rfile.MultiLevelIndex.IndexEntry;
  30 +import org.junit.Assert;
  31 +import org.junit.Test;
  32 +
  33 +/**
  34 + *
  35 + */
  36 +public class BlockIndexTest {
  37 +
  38 + private static class MyCacheEntry implements CacheEntry {
  39 + Object idx;
  40 + byte[] data;
  41 +
  42 + MyCacheEntry(byte[] d) {
  43 + this.data = d;
  44 + }
  45 +
  46 + @Override
  47 + public void setIndex(Object idx) {
  48 + this.idx = idx;
  49 + }
  50 +
  51 + @Override
  52 + public Object getIndex() {
  53 + return idx;
  54 + }
  55 +
  56 + @Override
  57 + public byte[] getBuffer() {
  58 + return data;
  59 + }
  60 + }
  61 +
  62 + @Test
  63 + public void test1() throws IOException {
  64 + ByteArrayOutputStream baos = new ByteArrayOutputStream();
  65 + DataOutputStream out = new DataOutputStream(baos);
  66 +
  67 + Key prevKey = null;
  68 +
  69 + int num = 1000;
  70 +
  71 + for (int i = 0; i < num; i++) {
  72 + Key key = new Key(RFileTest.nf("", i), "cf1", "cq1");
  73 + new RelativeKey(prevKey, key).write(out);
  74 + new Value(new byte[0]).write(out);
  75 + prevKey = key;
  76 + }
  77 +
  78 + out.close();
  79 + final byte[] data = baos.toByteArray();
  80 +
  81 + CacheEntry ce = new MyCacheEntry(data);
  82 +
  83 + ABlockReader cacheBlock = new CachableBlockFile.CachedBlockRead(ce, data);
  84 + BlockIndex blockIndex = null;
  85 +
  86 + for (int i = 0; i < 129; i++)
  87 + blockIndex = BlockIndex.getIndex(cacheBlock, new IndexEntry(prevKey, num, 0, 0, 0));
  88 +
  89 + BlockIndexEntry[] indexEntries = blockIndex.getIndexEntries();
  90 +
  91 + for (int i = 0; i < indexEntries.length; i++) {
  92 + int row = Integer.parseInt(indexEntries[i].getPrevKey().getRowData().toString());
  93 +
  94 + BlockIndexEntry bie;
  95 +
  96 +
  97 + bie = blockIndex.seekBlock(new Key(RFileTest.nf("", row), "cf1", "cq1"), cacheBlock);
  98 + if (i == 0)
  99 + Assert.assertSame(null, bie);
  100 + else
  101 + Assert.assertSame(indexEntries[i - 1], bie);
  102 +
  103 + Assert.assertSame(bie, blockIndex.seekBlock(new Key(RFileTest.nf("", row - 1), "cf1", "cq1"), cacheBlock));
  104 +
  105 + bie = blockIndex.seekBlock(new Key(RFileTest.nf("", row + 1), "cf1", "cq1"), cacheBlock);
  106 + Assert.assertSame(indexEntries[i], bie);
  107 +
  108 + RelativeKey rk = new RelativeKey();
  109 + rk.setPrevKey(bie.getPrevKey());
  110 + rk.readFields(cacheBlock);
  111 +
  112 + Assert.assertEquals(rk.getKey(), new Key(RFileTest.nf("", row + 1), "cf1", "cq1"));
  113 +
  114 + }
  115 + }
  116 +
  117 + @Test
  118 + public void testSame() throws IOException {
  119 + ByteArrayOutputStream baos = new ByteArrayOutputStream();
  120 + DataOutputStream out = new DataOutputStream(baos);
  121 +
  122 + Key prevKey = null;
  123 +
  124 + int num = 1000;
  125 +
  126 + for (int i = 0; i < num; i++) {
  127 + Key key = new Key(RFileTest.nf("", 1), "cf1", "cq1");
  128 + new RelativeKey(prevKey, key).write(out);
  129 + new Value(new byte[0]).write(out);
  130 + prevKey = key;
  131 + }
  132 +
  133 + for (int i = 0; i < num; i++) {
  134 + Key key = new Key(RFileTest.nf("", 3), "cf1", "cq1");
  135 + new RelativeKey(prevKey, key).write(out);
  136 + new Value(new byte[0]).write(out);
  137 + prevKey = key;
  138 + }
  139 +
  140 + for (int i = 0; i < num; i++) {
  141 + Key key = new Key(RFileTest.nf("", 5), "cf1", "cq1");
  142 + new RelativeKey(prevKey, key).write(out);
  143 + new Value(new byte[0]).write(out);
  144 + prevKey = key;
  145 + }
  146 +
  147 + out.close();
  148 + final byte[] data = baos.toByteArray();
  149 +
  150 + CacheEntry ce = new MyCacheEntry(data);
  151 +
  152 + ABlockReader cacheBlock = new CachableBlockFile.CachedBlockRead(ce, data);
  153 + BlockIndex blockIndex = null;
  154 +
  155 + for (int i = 0; i < 257; i++)
  156 + blockIndex = BlockIndex.getIndex(cacheBlock, new IndexEntry(prevKey, num, 0, 0, 0));
  157 +
  158 + Assert.assertSame(null, blockIndex.seekBlock(new Key(RFileTest.nf("", 0), "cf1", "cq1"), cacheBlock));
  159 + Assert.assertSame(null, blockIndex.seekBlock(new Key(RFileTest.nf("", 1), "cf1", "cq1"), cacheBlock));
  160 +
  161 + for (int i = 2; i < 6; i++) {
  162 + Key seekKey = new Key(RFileTest.nf("", i), "cf1", "cq1");
  163 + BlockIndexEntry bie = blockIndex.seekBlock(seekKey, cacheBlock);
  164 +
  165 + Assert.assertTrue(bie.getPrevKey().compareTo(seekKey) < 0);
  166 +
  167 + RelativeKey rk = new RelativeKey();
  168 + rk.setPrevKey(bie.getPrevKey());
  169 + rk.readFields(cacheBlock);
  170 +
  171 + Assert.assertTrue(rk.getKey().compareTo(seekKey) <= 0);
  172 + }
  173 +
  174 + }
  175 +}

0 comments on commit a1afdea

Please sign in to comment.
Something went wrong with that request. Please try again.