/
Cache.java
469 lines (399 loc) · 17.1 KB
/
Cache.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
// Cache.java
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 26.10.2006 on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.util.MemoryControl;
public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
// this is a combined read cache and write buffer
// we maintain four tables:
// - a read-cache
// - a miss-cache
// - a write buffer for rows that are not contained in the target index
// - a write buffer for rows that are known to be contained in target
// furthermore, if we access a kelondroFlexTable, we can use the ram index of the underlying index
// static object tracker; stores information about object cache usage
private static final TreeMap<String, Cache> objectTracker = new TreeMap<String, Cache>();
private static final long memStopGrow = 40 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available
private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
// class objects
private final ObjectIndex index; // the back-end of the cache
private RowSet readHitCache; // contains a complete copy of the cached objects
private RowSet readMissCache; // contains only the keys of the objects that had been a miss
private Row keyrow;
private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush;
private int hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete;
public Cache(final ObjectIndex backupIndex) {
this.index = backupIndex;
init();
objectTracker.put(backupIndex.filename(), this);
}
private void init() {
this.keyrow = new Row(new Column[]{index.row().column(0)}, index.row().objectOrder);
this.readHitCache = new RowSet(index.row(), 0);
this.readMissCache = new RowSet(this.keyrow, 0);
this.readHit = 0;
this.readMiss = 0;
this.writeUnique = 0;
this.writeDouble = 0;
this.cacheDelete = 0;
this.cacheFlush = 0;
this.hasnotHit = 0;
this.hasnotMiss = 0;
this.hasnotUnique = 0;
this.hasnotDouble = 0;
this.hasnotDelete = 0;
}
public final int cacheObjectChunkSize() {
return index.row().objectsize;
}
public final int writeBufferSize() {
return 0;
}
public final static long getMemStopGrow() {
return memStopGrow ;
}
public final static long getMemStartShrink() {
return memStartShrink ;
}
public static final Iterator<String> filenames() {
// iterates string objects; all file names from record tracker
return objectTracker.keySet().iterator();
}
public static final Map<String, String> memoryStats(final String filename) {
// returns a map for each file in the tracker;
// the map represents properties for each record oobjects,
// i.e. for cache memory allocation
final Cache theObjectsCache = objectTracker.get(filename);
return theObjectsCache.memoryStats();
}
private final Map<String, String> memoryStats() {
// returns statistical data about this object
final HashMap<String, String> map = new HashMap<String, String>();
map.put("objectHitChunkSize", (readHitCache == null) ? "0" : Integer.toString(readHitCache.rowdef.objectsize));
map.put("objectHitCacheCount", (readHitCache == null) ? "0" : Integer.toString(readHitCache.size()));
map.put("objectHitMem", (readHitCache == null) ? "0" : Long.toString(readHitCache.rowdef.objectsize * readHitCache.size()));
map.put("objectHitCacheReadHit", Integer.toString(readHit));
map.put("objectHitCacheReadMiss", Integer.toString(readMiss));
map.put("objectHitCacheWriteUnique", Integer.toString(writeUnique));
map.put("objectHitCacheWriteDouble", Integer.toString(writeDouble));
map.put("objectHitCacheDeletes", Integer.toString(cacheDelete));
map.put("objectHitCacheFlushes", Integer.toString(cacheFlush));
map.put("objectMissChunkSize", (readMissCache == null) ? "0" : Integer.toString(readMissCache.rowdef.objectsize));
map.put("objectMissCacheCount", (readMissCache == null) ? "0" : Integer.toString(readMissCache.size()));
map.put("objectMissMem", (readMissCache == null) ? "0" : Long.toString(readMissCache.rowdef.objectsize * readMissCache.size()));
map.put("objectMissCacheReadHit", Integer.toString(hasnotHit));
map.put("objectMissCacheReadMiss", Integer.toString(hasnotMiss));
map.put("objectMissCacheWriteUnique", Integer.toString(hasnotUnique));
map.put("objectMissCacheWriteDouble", Integer.toString(hasnotDouble));
map.put("objectMissCacheDeletes", Integer.toString(hasnotDelete));
map.put("objectMissCacheFlushes", "0"); // a miss cache flush can only happen if we have a deletion cache (which we dont have)
// future feature .. map.put("objectElderTimeRead", index.profile().)
return map;
}
/**
* checks for space in the miss cache
* @return true if it is allowed to write into this cache
*/
private final boolean checkMissSpace() {
// returns true if it is allowed to write into this cache
if (readMissCache == null) return false;
long available = MemoryControl.available();
if (available - 2 * 1024 * 1024 < readMissCache.memoryNeededForGrow()) {
readMissCache.clear();
}
available = MemoryControl.available();
return (available - 2 * 1024 * 1024 > readMissCache.memoryNeededForGrow());
}
/**
* checks for space in the hit cache
* @return true if it is allowed to write into this cache
*/
private final boolean checkHitSpace() {
// returns true if it is allowed to write into this cache
if (readHitCache == null) return false;
long available = MemoryControl.available();
if (available - 2 * 1024 * 1024 < readHitCache.memoryNeededForGrow()) {
readHitCache.clear();
}
available = MemoryControl.available();
return (available - 2 * 1024 * 1024 > readHitCache.memoryNeededForGrow());
}
public final synchronized void clearCache() {
if (readMissCache != null) readMissCache.clear();
if (readHitCache != null) readHitCache.clear();
}
public final synchronized void close() {
index.close();
readHitCache = null;
readMissCache = null;
}
public final synchronized boolean has(final byte[] key) {
// first look into the miss cache
if (readMissCache != null) {
if (readMissCache.get(key) == null) {
this.hasnotMiss++;
} else {
this.hasnotHit++;
return false;
}
}
// then try the hit cache and the buffers
if (readHitCache != null) {
if (readHitCache.get(key) != null) {
this.readHit++;
return true;
}
}
// finally ask the back-end index
this.readMiss++;
return index.has(key);
}
public final synchronized Row.Entry get(final byte[] key) throws IOException {
// first look into the miss cache
if (readMissCache != null) {
if (readMissCache.get(key) == null) {
this.hasnotMiss++;
} else {
this.hasnotHit++;
return null;
}
}
Row.Entry entry = null;
// then try the hit cache and the buffers
if (readHitCache != null) {
entry = readHitCache.get(key);
if (entry != null) {
this.readHit++;
return entry;
}
}
// finally ask the back-end index
this.readMiss++;
entry = index.get(key);
// learn from result
if (entry == null) {
if (checkMissSpace()) {
final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
return null;
}
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(entry);
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry;
}
public final synchronized void put(final Row.Entry row) throws IOException {
assert (row != null);
assert (row.columns() == row().columns());
//assert (!(serverLog.allZero(row.getColBytes(index.primarykey()))));
final byte[] key = row.getPrimaryKeyBytes();
checkHitSpace();
// remove entry from miss- and hit-cache
if (readMissCache != null) {
if (readMissCache.remove(key) != null) {
this.hasnotHit++;
// the entry did not exist before
index.put(row); // write to back-end
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return;
}
}
// write to the back-end
index.put(row);
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(row); // overwrite old entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
}
public final synchronized Row.Entry replace(final Row.Entry row) throws IOException {
assert (row != null);
assert (row.columns() == row().columns());
//assert (!(serverLog.allZero(row.getColBytes(index.primarykey()))));
final byte[] key = row.getPrimaryKeyBytes();
checkHitSpace();
// remove entry from miss- and hit-cache
if (readMissCache != null) {
if (readMissCache.remove(key) != null) {
this.hasnotHit++;
// the entry does not exist before
index.put(row); // write to backend
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return null;
}
}
Row.Entry entry;
// write to the back-end
entry = index.replace(row);
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry;
}
public final synchronized void addUnique(final Row.Entry row) throws IOException {
assert (row != null);
assert (row.columns() == row().columns());
//assert (!(serverLog.allZero(row.getColBytes(index.primarykey()))));
final byte[] key = row.getPrimaryKeyBytes();
checkHitSpace();
// remove entry from miss- and hit-cache
if (readMissCache != null) {
this.readMissCache.remove(key);
this.hasnotDelete++;
// the entry does not exist before
index.addUnique(row); // write to backend
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return;
}
// the worst case: we must write to the back-end directly
index.addUnique(row);
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
}
public final synchronized void addUnique(final Row.Entry row, final Date entryDate) throws IOException {
if (entryDate == null) {
addUnique(row);
return;
}
assert (row != null);
assert (row.columns() == row().columns());
final byte[] key = row.getPrimaryKeyBytes();
checkHitSpace();
// remove entry from miss- and hit-cache
if (readMissCache != null) {
this.readMissCache.remove(key);
this.hasnotDelete++;
}
// the worst case: we must write to the backend directly
index.addUnique(row);
if (checkHitSpace()) {
final Row.Entry dummy = readHitCache.replace(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
}
public final synchronized void addUnique(final List<Row.Entry> rows) throws IOException {
final Iterator<Row.Entry> i = rows.iterator();
while (i.hasNext()) addUnique(i.next());
}
public final synchronized ArrayList<RowCollection> removeDoubles() throws IOException {
return index.removeDoubles();
// todo: remove reported entries from the cache!!!
}
public final synchronized Row.Entry remove(final byte[] key) throws IOException {
checkMissSpace();
// add entry to miss-cache
if (checkMissSpace()) {
// set the miss cache; if there was already an entry we know that the return value must be null
final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key));
if (dummy == null) {
this.hasnotUnique++;
} else {
this.hasnotHit++;
this.hasnotDouble++;
}
}
// remove entry from hit-cache
if (readHitCache != null) {
final Row.Entry entry = readHitCache.remove(key);
if (entry == null) {
this.readMiss++;
} else {
this.readHit++;
this.cacheDelete++;
}
}
return index.remove(key);
}
public final synchronized Row.Entry removeOne() throws IOException {
checkMissSpace();
final Row.Entry entry = index.removeOne();
if (entry == null) return null;
final byte[] key = entry.getPrimaryKeyBytes();
if (checkMissSpace()) {
final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
if (readHitCache != null) {
final Row.Entry dummy = readHitCache.remove(key);
if (dummy != null) this.cacheDelete++;
}
return entry;
}
public final synchronized Row row() {
return index.row();
}
public final synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
return index.keys(up, firstKey);
}
public final synchronized CloneableIterator<Row.Entry> rows(final boolean up, final byte[] firstKey) throws IOException {
return index.rows(up, firstKey);
}
public final Iterator<Entry> iterator() {
try {
return rows();
} catch (IOException e) {
return null;
}
}
public final synchronized CloneableIterator<Row.Entry> rows() throws IOException {
return index.rows();
}
public final int size() {
return index.size();
}
public final String filename() {
return index.filename();
}
public final void clear() throws IOException {
this.index.clear();
init();
}
public final void deleteOnExit() {
this.index.deleteOnExit();
}
}