-
Notifications
You must be signed in to change notification settings - Fork 3.3k
/
MemcachedBlockCache.java
286 lines (247 loc) · 9.1 KB
/
MemcachedBlockCache.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
/**
* Copyright The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hadoop.hbase.io.hfile;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.ExecutionException;
import net.spy.memcached.CachedData;
import net.spy.memcached.ConnectionFactoryBuilder;
import net.spy.memcached.FailureMode;
import net.spy.memcached.MemcachedClient;
import net.spy.memcached.transcoders.Transcoder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
import org.apache.hadoop.hbase.trace.TraceUtil;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.htrace.core.TraceScope;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class to store blocks into memcached.
* This should only be used on a cluster of Memcached daemons that are tuned well and have a
* good network connection to the HBase regionservers. Any other use will likely slow down HBase
* greatly.
*/
@InterfaceAudience.Private
public class MemcachedBlockCache implements BlockCache {
private static final Logger LOG = LoggerFactory.getLogger(MemcachedBlockCache.class.getName());
// Some memcache versions won't take more than 1024 * 1024. So set the limit below
// that just in case this client is used with those versions.
public static final int MAX_SIZE = 1020 * 1024;
// Config key for what memcached servers to use.
// They should be specified in a comma sperated list with ports.
// like:
//
// host1:11211,host3:8080,host4:11211
public static final String MEMCACHED_CONFIG_KEY = "hbase.cache.memcached.servers";
public static final String MEMCACHED_TIMEOUT_KEY = "hbase.cache.memcached.timeout";
public static final String MEMCACHED_OPTIMEOUT_KEY = "hbase.cache.memcached.optimeout";
public static final String MEMCACHED_OPTIMIZE_KEY = "hbase.cache.memcached.spy.optimze";
public static final long MEMCACHED_DEFAULT_TIMEOUT = 500;
public static final boolean MEMCACHED_OPTIMIZE_DEFAULT = false;
private final MemcachedClient client;
private final HFileBlockTranscoder tc = new HFileBlockTranscoder();
private final CacheStats cacheStats = new CacheStats("MemcachedBlockCache");
public MemcachedBlockCache(Configuration c) throws IOException {
LOG.info("Creating MemcachedBlockCache");
long opTimeout = c.getLong(MEMCACHED_OPTIMEOUT_KEY, MEMCACHED_DEFAULT_TIMEOUT);
long queueTimeout = c.getLong(MEMCACHED_TIMEOUT_KEY, opTimeout + MEMCACHED_DEFAULT_TIMEOUT);
boolean optimize = c.getBoolean(MEMCACHED_OPTIMIZE_KEY, MEMCACHED_OPTIMIZE_DEFAULT);
ConnectionFactoryBuilder builder = new ConnectionFactoryBuilder()
.setOpTimeout(opTimeout)
.setOpQueueMaxBlockTime(queueTimeout) // Cap the max time before anything times out
.setFailureMode(FailureMode.Redistribute)
.setShouldOptimize(optimize)
.setDaemon(true) // Don't keep threads around past the end of days.
.setUseNagleAlgorithm(false) // Ain't nobody got time for that
.setReadBufferSize(HConstants.DEFAULT_BLOCKSIZE * 4 * 1024); // Much larger just in case
// Assume only the localhost is serving memecached.
// A la mcrouter or co-locating memcached with split regionservers.
//
// If this config is a pool of memecached servers they will all be used according to the
// default hashing scheme defined by the memcache client. Spy Memecache client in this
// case.
String serverListString = c.get(MEMCACHED_CONFIG_KEY,"localhost:11211");
String[] servers = serverListString.split(",");
List<InetSocketAddress> serverAddresses = new ArrayList<>(servers.length);
for (String s:servers) {
serverAddresses.add(Addressing.createInetSocketAddressFromHostAndPortStr(s));
}
client = new MemcachedClient(builder.build(), serverAddresses);
}
@Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
cacheBlock(cacheKey, buf);
}
@Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
if (buf instanceof HFileBlock) {
client.add(cacheKey.toString(), MAX_SIZE, (HFileBlock) buf, tc);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("MemcachedBlockCache can not cache Cacheable's of type "
+ buf.getClass().toString());
}
}
}
@Override
public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching,
boolean repeat, boolean updateCacheMetrics) {
// Assume that nothing is the block cache
HFileBlock result = null;
try (TraceScope traceScope = TraceUtil.createTrace("MemcachedBlockCache.getBlock")) {
result = client.get(cacheKey.toString(), tc);
} catch (Exception e) {
// Catch a pretty broad set of exceptions to limit any changes in the memecache client
// and how it handles failures from leaking into the read path.
if (LOG.isDebugEnabled()) {
LOG.debug("Exception pulling from memcached [ "
+ cacheKey.toString()
+ " ]. Treating as a miss.", e);
}
result = null;
} finally {
// Update stats if this request doesn't have it turned off 100% of the time
if (updateCacheMetrics) {
if (result == null) {
cacheStats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
} else {
cacheStats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
}
}
}
return result;
}
@Override
public boolean evictBlock(BlockCacheKey cacheKey) {
try {
cacheStats.evict();
return client.delete(cacheKey.toString()).get();
} catch (InterruptedException e) {
LOG.warn("Error deleting " + cacheKey.toString(), e);
Thread.currentThread().interrupt();
} catch (ExecutionException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Error deleting " + cacheKey.toString(), e);
}
}
return false;
}
/**
* This method does nothing so that memcached can handle all evictions.
*/
@Override
public int evictBlocksByHfileName(String hfileName) {
return 0;
}
@Override
public CacheStats getStats() {
return cacheStats;
}
@Override
public void shutdown() {
client.shutdown();
}
@Override
public long size() {
return 0;
}
@Override
public long getMaxSize() {
return 0;
}
@Override
public long getFreeSize() {
return 0;
}
@Override
public long getCurrentSize() {
return 0;
}
@Override
public long getCurrentDataSize() {
return 0;
}
@Override
public long getBlockCount() {
return 0;
}
@Override
public long getDataBlockCount() {
return 0;
}
@Override
public Iterator<CachedBlock> iterator() {
return new Iterator<CachedBlock>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public CachedBlock next() {
throw new NoSuchElementException("MemcachedBlockCache can't iterate over blocks.");
}
@Override
public void remove() {
}
};
}
@Override
public BlockCache[] getBlockCaches() {
return null;
}
/**
* Class to encode and decode an HFileBlock to and from memecached's resulting byte arrays.
*/
private static class HFileBlockTranscoder implements Transcoder<HFileBlock> {
@Override
public boolean asyncDecode(CachedData d) {
return false;
}
@Override
public CachedData encode(HFileBlock block) {
ByteBuffer bb = ByteBuffer.allocate(block.getSerializedLength());
block.serialize(bb, true);
return new CachedData(0, bb.array(), CachedData.MAX_SIZE);
}
@Override
public HFileBlock decode(CachedData d) {
try {
ByteBuff buf = new SingleByteBuff(ByteBuffer.wrap(d.getData()));
return (HFileBlock) HFileBlock.BLOCK_DESERIALIZER.deserialize(buf, ByteBuffAllocator.HEAP);
} catch (IOException e) {
LOG.warn("Failed to deserialize data from memcached", e);
}
return null;
}
@Override
public int getMaxSize() {
return MAX_SIZE;
}
}
}