/
Msg5.h
309 lines (252 loc) · 8.39 KB
/
Msg5.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
// Matt Wells, copyright Sep 2001
// . get a lists from tree, cache and disk
#ifndef _MSG5_H_
#define _MSG5_H_
#include "Msg3.h"
#include "RdbList.h"
#include "HashTableX.h"
extern int32_t g_numCorrupt;
extern HashTableX g_waitingTable;
extern int32_t g_isDumpingRdbFromMain;
// this is used internally and by Msg0 for generating a cacheKey
// to see if its request can be satisfied by any remote host without
// hitting disk. Multicast uses Msg34 to ask each host that.
//key_t makeCacheKey ( key_t startKey ,
// key_t endKey ,
void makeCacheKey ( char *startKey ,
char *endKey ,
bool includeTree ,
int32_t minRecSizes ,
int32_t startFileNum ,
int32_t numFiles ,
char *cacheKeyPtr ,
char keySize ) ;
inline key_t makeCacheKey ( key_t startKey ,
key_t endKey ,
bool includeTree ,
int32_t minRecSizes ,
int32_t startFileNum ,
int32_t numFiles ) {
key_t k;
makeCacheKey ( (char *)&startKey ,
(char *)&endKey ,
includeTree ,
minRecSizes ,
startFileNum ,
numFiles ,
(char *)&k ,
sizeof(key_t) );
return k;
}
class Msg5 {
public:
Msg5();
~Msg5();
// . set "list" with the asked for records
// . returns false if blocked, true otherwise
// . sets errno on error
// . RdbList will be populated with the records you want
// . we pass ourselves back to callback since requestHandler()s of
// various msg classes often do not have any context/state, and they
// construct us in a C wrapper so they need to delete us when
// they're done with us
// . if maxCacheAge is > 0, we lookup in cache first
bool getList ( //class RdbBase *base ,
char rdbId ,
//char *coll ,
collnum_t collnum ,
RdbList *list ,
//key_t startKey ,
//key_t endKey ,
void *startKey ,
void *endKey ,
int32_t recSizes , // requested scan size(-1 all)
bool includeTree ,
bool addToCache ,
int32_t maxCacheAge , // in secs for cache lookup
int32_t startFileNum , // first file to scan
int32_t numFiles , // rel.to startFileNum,-1 all
void *state , // for callback
void (* callback ) ( void *state ,
RdbList *list ,
Msg5 *msg5 ) ,
int32_t niceness ,
bool doErrorCorrection ,
//key_t *cacheKeyPtr = NULL ,
char *cacheKeyPtr = NULL ,
int32_t retryNum = 0 ,
int32_t maxRetries = -1 ,
bool compensateForMerge = true ,
int64_t syncPoint = -1 ,
class Msg5 *msg5b = NULL ,
bool isRealMerge = false ,
bool allowPageCache = true ,
bool hitDisk = true ,
// if this is false we just return all lists unmerged
bool mergeLists = true );
bool getList ( //class RdbBase *base ,
char rdbId ,
//char *coll ,
collnum_t collnum ,
RdbList *list ,
key_t startKey ,
key_t endKey ,
int32_t recSizes , // requested scan size(-1 all)
bool includeTree ,
bool addToCache ,
int32_t maxCacheAge , // in secs for cache lookup
int32_t startFileNum , // first file to scan
int32_t numFiles , // rel.to startFileNum,-1 all
void *state , // for callback
void (* callback ) ( void *state ,
RdbList *list ,
Msg5 *msg5 ) ,
int32_t niceness ,
bool doErrorCorrection ,
key_t *cacheKeyPtr = NULL ,
int32_t retryNum = 0 ,
int32_t maxRetries = -1 ,
bool compensateForMerge = true ,
int64_t syncPoint = -1 ,
class Msg5 *msg5b = NULL ,
bool isRealMerge = false ,
bool allowPageCache = true ,
bool hitDisk = true ) {
return getList ( rdbId ,
collnum ,
list ,
(char *)&startKey ,
(char *)&endKey ,
recSizes ,
includeTree ,
addToCache ,
maxCacheAge ,
startFileNum ,
numFiles ,
state ,
callback ,
niceness ,
doErrorCorrection ,
(char *)cacheKeyPtr ,
retryNum ,
maxRetries ,
compensateForMerge ,
syncPoint ,
msg5b ,
isRealMerge ,
allowPageCache ,
hitDisk ); };
// need niceness to pass on to others
int32_t getNiceness ( ) { return m_niceness; };
// frees m_treeList, m_diskList (can be quite a lot of mem 2+ megs)
void reset();
// called to read lists from disk using Msg3
bool readList();
// keep public for doneScanningWrapper to call
bool gotList();
bool gotList2();
void copyAndSendBackList ( class RdbList *list );
// does readList() need to be called again due to negative rec
// annihilation?
bool needsRecall();
void repairLists_r ();
void mergeLists_r ();
bool doneMerging ();
// how many dup keys removed from the IndexList merge, if any?
int32_t getDupsRemoved ( ) { return m_dupsRemoved; };
// . when a list is bad we try to patch it by getting a list from
// a host in our redundancy group
// . we also get a local list from ALL files and tree to remove
// recs we already have from the remote list
bool getRemoteList ( );
bool gotRemoteList ( );
// we add our m_finalList(s) to this, the user's list
RdbList *m_list;
// used for testing integrity of the big root list merge algo
RdbList m_list2;
// hold the caller of getList()'s callback here
void (* m_callback )( void *state , RdbList *list , Msg5 *msg );
void *m_state ;
char m_calledCallback;
// private:
// holds all RdbLists from disk
Msg3 m_msg3;
// holds list from tree
RdbList m_treeList;
RdbList m_dummy;
// holds list parms
//key_t m_startKey;
//key_t m_endKey;
char m_startKey[MAX_KEY_BYTES];
char m_endKey[MAX_KEY_BYTES];
bool m_includeTree;
bool m_addToCache;
int32_t m_maxCacheAge;
int32_t m_numFiles;
int32_t m_startFileNum;
int32_t m_minRecSizes;
//RdbBase *m_base;
//char *m_coll;
char m_rdbId;
// . cache may modify these
// . gotLists() may modify these before reading more
//key_t m_fileStartKey;
char m_fileStartKey[MAX_KEY_BYTES];
int32_t m_newMinRecSizes;
int32_t m_round;
int32_t m_totalSize;
int32_t m_lastTotalSize;
int32_t m_treeMinRecSizes;
bool m_readAbsolutelyNothing;
int32_t m_niceness;
// error correction stuff
bool m_doErrorCorrection;
bool m_hadCorruption;
//int64_t m_checkTime;
class Msg0 *m_msg0;
// for timing debug
int64_t m_startTime;
// hold pointers to lists to merge
RdbList *m_listPtrs [ MAX_RDB_FILES + 1 ]; // plus tree
//int32_t m_tfns [ MAX_RDB_FILES + 1 ]; // plus tree
int32_t m_numListPtrs;
//RdbList m_tfndbList;
// cache ptr
//class RdbCache *m_cache;
// key for hitting the cache
//key_t m_cacheKey;
char m_cacheKey[MAX_KEY_BYTES];
bool m_removeNegRecs;
//key_t m_minEndKey;
char m_minEndKey[MAX_KEY_BYTES];
// info for truncating and passing to RdbList::indexMerge_r()
//key_t m_prevKey;
char m_prevKey[MAX_KEY_BYTES];
int32_t m_prevCount;
int32_t m_oldListSize;
// how many dup keys removed from the IndexList merge, if any?
int32_t m_dupsRemoved;
bool m_compensateForMerge;
int32_t m_maxRetries;
//int64_t m_syncPoint;
int32_t m_filtered;
// used for reading a corresponding tfndb list for a titledb read
class Msg5 *m_msg5b;
bool m_isRealMerge;
int64_t m_time1;
int32_t m_indexdbTruncationLimit;
char m_ks;
// for allowing the page cache
bool m_allowPageCache;
bool m_hitDisk;
bool m_mergeLists;
char m_waitingForList;
//char m_waitingForMerge;
collnum_t m_collnum;
// actually part of a different algo than m_waitingForList!
uint64_t m_waitingKey;
// hack parms
void *m_parent;
int32_t m_i;
};
#endif