Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 555 lines (454 sloc) 12.73 kB
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
1 /*
2 * This is a simple 'sstable' implemention, merge all mtable into on-disk indices
3 * BLOCK's LAYOUT:
4 * +--------+--------+--------+--------+
5 * | sst block 1 |
6 * +--------+--------+--------+--------+
7 * | sst block 2 |
8 * +--------+--------+--------+--------+
9 * | ... all the other blocks .. |
10 * +--------+--------+--------+--------+
11 * | sst block N |
12 * +--------+--------+--------+--------+
f80a594 @BohuTANG Modify:README
authored
13 * | footer |
14 * +--------+--------+--------+--------+
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
15 *
f80a594 @BohuTANG Modify:README
authored
16 * FOOTER's LAYOUT:
17 * +--------+--------+--------+--------+
18 * | last key |
19 * +--------+--------+--------+--------+
20 * | block count |
21 * +--------+--------+--------+--------+
22 * | crc |
23 * +--------+--------+--------+--------+
24
239b3e7 @BohuTANG Add: stable log recovery
authored
25 * nessDB storage engine
26 * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
27 * All rights reserved.
28 * Code is licensed with BSD. See COPYING.BSD file.
29 *
30 */
31
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <string.h>
35 #include <sys/types.h>
36 #include <sys/stat.h>
37 #include <unistd.h>
38 #include <fcntl.h>
39 #include <dirent.h>
40 #include <unistd.h>
41 #include <sys/mman.h>
42 #include <assert.h>
43
44 #include "sst.h"
45 #include "debug.h"
46
47 #define BLK_MAGIC (20111225)
48 #define F_CRC (2011)
49
50 struct footer{
d8a3ac8 @BohuTANG Add: cgroup to index
authored
51 char key[NESSDB_MAX_KEY_SIZE];
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
52 __be32 count;
53 __be32 crc;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
54 };
55
7e05942 @BohuTANG Add:bloomfilter when sst load
authored
56 void _add_bloom(struct sst *sst, int fd, int count)
57 {
58 int i;
59 int blk_sizes;
60 struct sst_block *blks;
61
62 blk_sizes = count * sizeof(struct sst_block);
63
64 blks= mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
65 if (blks == MAP_FAILED) {
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
66 __PANIC("Error:Can't mmap file when add bloom");
7e05942 @BohuTANG Add:bloomfilter when sst load
authored
67 return;
68 }
69
4de0157 @BohuTANG Add:debug,and write ERROR&DEBUG level to file:ness.event
authored
70 for (i = 0; i < count; i++)
7e05942 @BohuTANG Add:bloomfilter when sst load
authored
71 bloom_add(sst->bloom, blks[i].key);
72
73 if (munmap(blks, blk_sizes) == -1)
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
74 __DEBUG(LEVEL_ERROR, "Error:un-mmapping the file");
7e05942 @BohuTANG Add:bloomfilter when sst load
authored
75 }
76
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
77 void _sst_load(struct sst *sst)
78 {
3cec7d2 @feinno-zhangyanfei Changed: change the try_lock to lock when merge-lock preemption
feinno-zhangyanfei authored
79 int fd, result, all_count = 0;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
80 DIR *dd;
81 struct dirent *de;
82
83 dd = opendir(sst->basedir);
adf98d9 @feinno-zhangyanfei Modified: extent lock area and clean code of sst
feinno-zhangyanfei authored
84 while ((de = readdir(dd))) {
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
85 if (strstr(de->d_name, ".sst")) {
3cec7d2 @feinno-zhangyanfei Changed: change the try_lock to lock when merge-lock preemption
feinno-zhangyanfei authored
86 int fcount = 0, fcrc = 0;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
87 struct meta_node mn;
88 struct footer footer;
d8a3ac8 @BohuTANG Add: cgroup to index
authored
89 char sst_file[FILE_PATH_SIZE];
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
90 int fsize = sizeof(struct footer);
91
d8a3ac8 @BohuTANG Add: cgroup to index
authored
92 memset(sst_file, 0, FILE_PATH_SIZE);
93 snprintf(sst_file, FILE_PATH_SIZE, "%s/%s", sst->basedir, de->d_name);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
94
95 fd = open(sst_file, O_RDWR, 0644);
96 lseek(fd, -fsize, SEEK_END);
97 result = read(fd, &footer, sizeof(struct footer));
98 if (result == -1)
99 abort();
100
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
101 fcount = from_be32(footer.count);
102 fcrc = from_be32(footer.crc);
103 if (fcrc != F_CRC) {
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
104 __DEBUG(LEVEL_ERROR, "Crc wrong, sst file maybe broken, crc:<%d>,index<%s>", fcrc, sst_file);
4de0157 @BohuTANG Add:debug,and write ERROR&DEBUG level to file:ness.event
authored
105 close(fd);
106 continue;
107 }
108
109 if (fcount == 0) {
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
110 close(fd);
111 continue;
112 }
113
7e05942 @BohuTANG Add:bloomfilter when sst load
authored
114 /* Add to bloom */
115 _add_bloom(sst, fd, fcount);
116
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
117 all_count += fcount;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
118
119 /* Set meta */
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
120 mn.count = fcount;
d8a3ac8 @BohuTANG Add: cgroup to index
authored
121 memset(mn.end, 0, NESSDB_MAX_KEY_SIZE);
122 memcpy(mn.end, footer.key, NESSDB_MAX_KEY_SIZE);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
123
d8a3ac8 @BohuTANG Add: cgroup to index
authored
124 memset(mn.index_name, 0, FILE_NAME_SIZE);
125 memcpy(mn.index_name, de->d_name, FILE_NAME_SIZE);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
126 meta_set(sst->meta, &mn);
127
128 close(fd);
129 }
130 }
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
131
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
132 closedir(dd);
4de0157 @BohuTANG Add:debug,and write ERROR&DEBUG level to file:ness.event
authored
133 __DEBUG(LEVEL_DEBUG, "Load sst,all entries count:<%d>", all_count);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
134 }
135
136 struct sst *sst_new(const char *basedir)
137 {
138 struct sst *s;
139
140 s = malloc(sizeof(struct sst));
141 s->lsn = 0;
142
143 s->meta = meta_new();
d8a3ac8 @BohuTANG Add: cgroup to index
authored
144 memcpy(s->basedir, basedir, FILE_PATH_SIZE);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
145
7c43287 @BohuTANG Reformat bloomfilter interfaces
authored
146 s->bloom = bloom_new();
f625b78 @BohuTANG Remove: one sst mapping one pthread_mutex
authored
147 s->mutexer.lsn = -1;
148 pthread_mutex_init(&s->mutexer.mutex, NULL);
7e05942 @BohuTANG Add:bloomfilter when sst load
authored
149
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
150 /* SST files load */
151 _sst_load(s);
152
153 return s;
154 }
27fa9ba @BohuTANG Reformat: request/reponse code style
authored
155
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
156 /*
157 * Write node to index file
158 */
159 void *_write_mmap(struct sst *sst, struct skipnode *x, size_t count, int need_new)
160 {
c8ce966 @BohuTANG Fixup: write DEL to sst
authored
161 int i, j, c_clone;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
162 int fd;
163 int sizes;
164 int result;
d8a3ac8 @BohuTANG Add: cgroup to index
authored
165 char file[FILE_PATH_SIZE];
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
166 struct skipnode *last;
167 struct sst_block *blks;
168 struct footer footer;
169
170 int fsize = sizeof(struct footer);
171
172 sizes = count * sizeof(struct sst_block);
173
d8a3ac8 @BohuTANG Add: cgroup to index
authored
174 memset(file, 0, FILE_PATH_SIZE);
175 snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
176 fd = open(file, O_RDWR | O_CREAT | O_TRUNC, 0644);
177
178 lseek(fd, sizes - 1, SEEK_SET);
179 result = write(fd, "", 1);
180 if (result == -1)
181 abort();
182
183 blks = mmap(0, sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
184
185 last = x;
c8ce966 @BohuTANG Fixup: write DEL to sst
authored
186 c_clone = count;
187 for (i = 0, j= 0; i < c_clone; i++) {
6f359fa @BohuTANG Modify: remove GET64 highest bit mark
authored
188 if (x->opt == ADD) {
d8a3ac8 @BohuTANG Add: cgroup to index
authored
189 memset(blks[j].key, 0, NESSDB_MAX_KEY_SIZE);
190 memcpy(blks[j].key, x->key,NESSDB_MAX_KEY_SIZE);
c8ce966 @BohuTANG Fixup: write DEL to sst
authored
191 blks[j].offset=to_be64(x->val);
192 j++;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
193 } else
194 count--;
195
196 last = x;
197 x = x->forward[0];
198 }
27fa9ba @BohuTANG Reformat: request/reponse code style
authored
199
200 #ifdef MSYNC
78caa3e @BohuTANG Add: msync when sst written
authored
201 if (msync(blks, sizes, MS_SYNC) == -1) {
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
202 __DEBUG(LEVEL_ERROR, "Msync error");
78caa3e @BohuTANG Add: msync when sst written
authored
203 }
27fa9ba @BohuTANG Reformat: request/reponse code style
authored
204 #endif
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
205
206 if (munmap(blks, sizes) == -1) {
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
207 __DEBUG(LEVEL_ERROR, "Un-mmapping the file");
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
208 }
209
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
210 footer.count = to_be32(count);
211 footer.crc = to_be32(F_CRC);
d8a3ac8 @BohuTANG Add: cgroup to index
authored
212 memset(footer.key, 0, NESSDB_MAX_KEY_SIZE);
213 memcpy(footer.key, last->key, NESSDB_MAX_KEY_SIZE);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
214
5abc1b6 @BohuTANG Extend testing key/value buffer,endwith NULL
authored
215 result = write(fd, &footer, fsize);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
216 if (result == -1)
217 abort();
218
219 /* Set meta */
220 struct meta_node mn;
221
222 mn.count = count;
d8a3ac8 @BohuTANG Add: cgroup to index
authored
223 memset(mn.end, 0, NESSDB_MAX_KEY_SIZE);
224 memcpy(mn.end, last->key, NESSDB_MAX_KEY_SIZE);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
225
d8a3ac8 @BohuTANG Add: cgroup to index
authored
226 memset(mn.index_name, 0, FILE_NAME_SIZE);
227 memcpy(mn.index_name, sst->name, FILE_NAME_SIZE);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
228
229 if (need_new)
230 meta_set(sst->meta, &mn);
231 else
232 meta_set_byname(sst->meta, &mn);
233
234 close(fd);
235
236 return x;
237 }
238
239 struct skiplist *_read_mmap(struct sst *sst, size_t count)
240 {
241 int i;
242 int fd;
243 int result;
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
244 int fcount;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
245 int blk_sizes;
d8a3ac8 @BohuTANG Add: cgroup to index
authored
246 char file[FILE_PATH_SIZE];
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
247 struct sst_block *blks;
248 struct skiplist *merge = NULL;
249 struct footer footer;
250 int fsize = sizeof(struct footer);
251
d8a3ac8 @BohuTANG Add: cgroup to index
authored
252 memset(file, 0, FILE_PATH_SIZE);
253 snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
254
255 fd = open(file, O_RDWR, 0644);
256 result = lseek(fd, -fsize, SEEK_END);
257 if (result == -1) {
258 abort();
259 }
260 result = read(fd, &footer, fsize);
261 if (result == -1)
262 abort();
263
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
264 fcount = from_be32(footer.count);
265
266 blk_sizes = fcount * sizeof(struct sst_block);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
267
268 /* Blocks read */
b32d7b3 @BohuTANG Add: change bg-merge-thread to single detached thread
authored
269 blks= mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
270 if (blks == MAP_FAILED) {
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
271 __DEBUG(LEVEL_ERROR, "Un-mmapping the file");
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
272 goto out;
273 }
274
275 /* Merge */
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
276 merge = skiplist_new(fcount + count + 1);
277 for (i = 0; i < fcount; i++) {
278 skiplist_insert(merge, blks[i].key, from_be64(blks[i].offset), ADD);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
279 }
280
281 if (munmap(blks, blk_sizes) == -1)
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
282 __DEBUG(LEVEL_ERROR, "Un-mmapping the file");
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
283
284 out:
285 close(fd);
286
287 return merge;
288 }
289
e8ac50e @BohuTANG Add memset to set mmap items
authored
290 uint64_t _read_offset(struct sst *sst, struct slice *sk)
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
291 {
292 int fd;
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
293 int fcount;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
294 int blk_sizes;
2f52625 @BohuTANG Fix: read off by key
authored
295 int result;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
296 uint64_t off = 0UL;
d8a3ac8 @BohuTANG Add: cgroup to index
authored
297 char file[FILE_PATH_SIZE];
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
298 struct sst_block *blks;
2f52625 @BohuTANG Fix: read off by key
authored
299 struct footer footer;
300 int fsize = sizeof(struct footer);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
301
d8a3ac8 @BohuTANG Add: cgroup to index
authored
302 memset(file, 0, FILE_PATH_SIZE);
303 snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
304
305 fd = open(file, O_RDWR, 0644);
2f52625 @BohuTANG Fix: read off by key
authored
306 result = lseek(fd, -fsize, SEEK_END);
307 if (result == -1) {
308 abort();
309 }
310 result = read(fd, &footer, fsize);
311 if (result == -1)
312 abort();
313
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
314 fcount = from_be32(footer.count);
315 blk_sizes = fcount * sizeof(struct sst_block);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
316
317 /* Blocks read */
318 blks= mmap(0, blk_sizes, PROT_READ, MAP_SHARED, fd, 0);
319 if (blks == MAP_FAILED) {
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
320 __PANIC("Map_failed when read");
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
321 goto out;
322 }
323
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
324 size_t left = 0, right = fcount, i = 0;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
325 while (left < right) {
326 i = (right -left) / 2 +left;
e8ac50e @BohuTANG Add memset to set mmap items
authored
327 int cmp = strcmp(sk->data, blks[i].key);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
328 if (cmp == 0) {
4bf1dc3 @BohuTANG Change:change all numbeal to Big-Endian
authored
329 off = from_be64(blks[i].offset);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
330 break ;
331 }
332
333 if (cmp < 0)
334 right = i;
335 else
336 left = i + 1;
337 }
338
339 if (munmap(blks, blk_sizes) == -1)
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
340 __DEBUG(LEVEL_ERROR, "un-mmapping the file");
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
341
342 out:
343 close(fd);
344
345 return off;
346 }
347
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
348 void _flush_merge_list(struct sst *sst, struct skipnode *x, size_t count, struct meta_node *meta)
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
349 {
350 int mul;
351 int rem;
32e9a40 @BohuTANG reset to forward: change meta-lsn to lsn
authored
352 int lsn;
80f5468 @BohuTANG reformat some c99 stand code style
authored
353 int i;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
354
d8a3ac8 @BohuTANG Add: cgroup to index
authored
355 /* Less than 2x SST_MAX_COUNT,compact one index file */
356 if (count <= SST_MAX_COUNT * 2) {
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
357 if (meta) {
32e9a40 @BohuTANG reset to forward: change meta-lsn to lsn
authored
358 lsn = meta->lsn;
f625b78 @BohuTANG Remove: one sst mapping one pthread_mutex
authored
359 sst->mutexer.lsn = lsn;
360 pthread_mutex_lock(&sst->mutexer.mutex);
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
361 x = _write_mmap(sst, x, count, 0);
f625b78 @BohuTANG Remove: one sst mapping one pthread_mutex
authored
362 pthread_mutex_unlock(&sst->mutexer.mutex);
363 sst->mutexer.lsn = -1;
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
364 } else
365 x = _write_mmap(sst, x, count, 0);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
366 } else {
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
367 if (meta) {
32e9a40 @BohuTANG reset to forward: change meta-lsn to lsn
authored
368 lsn = meta->lsn;
f625b78 @BohuTANG Remove: one sst mapping one pthread_mutex
authored
369 sst->mutexer.lsn = lsn;
370 pthread_mutex_lock(&sst->mutexer.mutex);
d8a3ac8 @BohuTANG Add: cgroup to index
authored
371 x = _write_mmap(sst, x, SST_MAX_COUNT, 0);
f625b78 @BohuTANG Remove: one sst mapping one pthread_mutex
authored
372 pthread_mutex_unlock(&sst->mutexer.mutex);
373 sst->mutexer.lsn = -1;
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
374 } else
d8a3ac8 @BohuTANG Add: cgroup to index
authored
375 x = _write_mmap(sst, x, SST_MAX_COUNT, 0);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
376
377 /* first+last */
d8a3ac8 @BohuTANG Add: cgroup to index
authored
378 mul = (count - SST_MAX_COUNT * 2) / SST_MAX_COUNT;
379 rem = count % SST_MAX_COUNT;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
380
80f5468 @BohuTANG reformat some c99 stand code style
authored
381 for (i = 0; i < mul; i++) {
d8a3ac8 @BohuTANG Add: cgroup to index
authored
382 memset(sst->name, 0, FILE_NAME_SIZE);
383 snprintf(sst->name, FILE_NAME_SIZE, "%d.sst", sst->meta->size);
384 x = _write_mmap(sst, x, SST_MAX_COUNT, 1);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
385 }
386
d8a3ac8 @BohuTANG Add: cgroup to index
authored
387 /* The remain part,will be larger than SST_MAX_COUNT */
388 memset(sst->name, 0, FILE_NAME_SIZE);
389 snprintf(sst->name, FILE_NAME_SIZE, "%d.sst", sst->meta->size);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
390
d8a3ac8 @BohuTANG Add: cgroup to index
authored
391 x = _write_mmap(sst, x, rem + SST_MAX_COUNT, 1);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
392 }
393 }
394
395 void _flush_new_list(struct sst *sst, struct skipnode *x, size_t count)
396 {
397 int mul ;
398 int rem;
80f5468 @BohuTANG reformat some c99 stand code style
authored
399 int i;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
400
d8a3ac8 @BohuTANG Add: cgroup to index
authored
401 if (count <= SST_MAX_COUNT * 2) {
402 memset(sst->name, 0, FILE_NAME_SIZE);
403 snprintf(sst->name, FILE_NAME_SIZE, "%d.sst", sst->meta->size);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
404 x = _write_mmap(sst, x, count, 1);
405 } else {
d8a3ac8 @BohuTANG Add: cgroup to index
authored
406 mul = count / SST_MAX_COUNT;
407 rem = count % SST_MAX_COUNT;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
408
80f5468 @BohuTANG reformat some c99 stand code style
authored
409 for (i = 0; i < (mul - 1); i++) {
d8a3ac8 @BohuTANG Add: cgroup to index
authored
410 memset(sst->name, 0, FILE_NAME_SIZE);
411 snprintf(sst->name, FILE_NAME_SIZE, "%d.sst", sst->meta->size);
412 x = _write_mmap(sst, x, SST_MAX_COUNT, 1);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
413 }
414
d8a3ac8 @BohuTANG Add: cgroup to index
authored
415 memset(sst->name, 0, FILE_NAME_SIZE);
416 snprintf(sst->name, FILE_NAME_SIZE, "%d.sst", sst->meta->size);
417 x = _write_mmap(sst, x, SST_MAX_COUNT + rem, 1);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
418 }
419 }
420
421 void _flush_list(struct sst *sst, struct skipnode *x,struct skipnode *hdr,int flush_count)
422 {
423 int pos = 0;
424 int count = flush_count;
425 struct skipnode *cur = x;
426 struct skipnode *first = hdr;
427 struct skiplist *merge = NULL;
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
428 struct meta_node *meta_info = NULL;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
429
430 while(cur != first) {
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
431 meta_info = meta_get(sst->meta, cur->key);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
432
433 /* If m is NULL, cur->key more larger than meta's largest area
434 * need to create new index-file
435 */
adf98d9 @feinno-zhangyanfei Modified: extent lock area and clean code of sst
feinno-zhangyanfei authored
436 if(!meta_info){
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
437
438 /* If merge is NULL,it has no merge*/
adf98d9 @feinno-zhangyanfei Modified: extent lock area and clean code of sst
feinno-zhangyanfei authored
439 if(merge) {
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
440 struct skipnode *h = merge->hdr->forward[0];
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
441 _flush_merge_list(sst, h, merge->count, NULL);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
442 skiplist_free(merge);
443 merge = NULL;
444 }
445
446 /* Flush the last nodes to disk */
447 _flush_new_list(sst, x, count - pos);
448
449 return;
450 } else {
451
452 /* If m is not NULL,means found the index of the cur
453 * We need:
454 * 1) compare the sst->name with meta index name
455 * a)If 0: add the cur to merge,and continue
456 * b)others:
457 * b1)Flush the merge list to disk
458 * b2)Open the meta's mmap,and load all blocks to new merge,add cur to merge
459 */
460 int cmp = strcmp(sst->name, meta_info->index_name);
461 if(cmp == 0) {
adf98d9 @feinno-zhangyanfei Modified: extent lock area and clean code of sst
feinno-zhangyanfei authored
462 if (!merge)
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
463 merge = _read_mmap(sst,count);
464
465 skiplist_insert_node(merge, cur);
466 } else {
467 if (merge) {
468 struct skipnode *h = merge->hdr->forward[0];
469
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
470 _flush_merge_list(sst, h, merge->count, meta_info);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
471 skiplist_free(merge);
472 merge = NULL;
473 }
474
d8a3ac8 @BohuTANG Add: cgroup to index
authored
475 memset(sst->name, 0, FILE_NAME_SIZE);
476 memcpy(sst->name, meta_info->index_name, FILE_NAME_SIZE);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
477 merge = _read_mmap(sst, count);
478
479 /* Add to merge list */
480 skiplist_insert_node(merge, cur);
481 }
482
483 }
484
485 pos++;
486 cur = cur->forward[0];
487 }
488
489 if (merge) {
490 struct skipnode *h = merge->hdr->forward[0];
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
491 _flush_merge_list(sst, h, merge->count, meta_info);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
492 skiplist_free(merge);
493 }
494 }
495
a400f4f @BohuTANG Fixup: log recovery bug
authored
496 void sst_merge(struct sst *sst, struct skiplist *list, int fromlog)
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
497 {
498 struct skipnode *x= list->hdr->forward[0];
499
a400f4f @BohuTANG Fixup: log recovery bug
authored
500 if (fromlog == 1) {
501 struct skipnode *cur = x;
502 struct skipnode *first = list->hdr;
503
a2a9eec @BohuTANG Add nessdb PANIC to exit when serious error
authored
504 __DEBUG(LEVEL_DEBUG, "adding log items to bloomfilter");
a400f4f @BohuTANG Fixup: log recovery bug
authored
505 while (cur != first) {
482ead3 @BohuTANG Add: 'server' info to show
authored
506 if (cur->opt == ADD)
507 bloom_add(sst->bloom, cur->key);
a400f4f @BohuTANG Fixup: log recovery bug
authored
508 cur = cur->forward[0];
509 }
510 }
511
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
512 /* First time,index is NULL,need to be created */
513 if (sst->meta->size == 0)
514 _flush_new_list(sst, x, list->count);
515 else
516 _flush_list(sst, x, list->hdr, list->count);
b32d7b3 @BohuTANG Add: change bg-merge-thread to single detached thread
authored
517 skiplist_free(list);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
518 }
519
520 uint64_t sst_getoff(struct sst *sst, struct slice *sk)
521 {
32e9a40 @BohuTANG reset to forward: change meta-lsn to lsn
authored
522 int lsn;
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
523 uint64_t off = 0UL;
524 struct meta_node *meta_info;
525
526 meta_info = meta_get(sst->meta, sk->data);
adf98d9 @feinno-zhangyanfei Modified: extent lock area and clean code of sst
feinno-zhangyanfei authored
527 if(!meta_info)
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
528 return 0UL;
529
d8a3ac8 @BohuTANG Add: cgroup to index
authored
530 memcpy(sst->name, meta_info->index_name, FILE_NAME_SIZE);
c4f16bc @feinno-zhangyanfei Add:read lock
feinno-zhangyanfei authored
531
532 /* If get one record from on-disk sst file,
533 * this file must not be operated by bg-merge thread
f625b78 @BohuTANG Remove: one sst mapping one pthread_mutex
authored
534 */
32e9a40 @BohuTANG reset to forward: change meta-lsn to lsn
authored
535 lsn = meta_info->lsn;
f625b78 @BohuTANG Remove: one sst mapping one pthread_mutex
authored
536 if (sst->mutexer.lsn == lsn) {
537 pthread_mutex_lock(&sst->mutexer.mutex);
538 off = _read_offset(sst, sk);
539 pthread_mutex_unlock(&sst->mutexer.mutex);
540 } else {
541 off = _read_offset(sst, sk);
542 }
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
543
544 return off;
545 }
546
547 void sst_free(struct sst *sst)
548 {
eadbef6 @BohuTANG Fixup: some frees
authored
549 if (sst) {
550 meta_free(sst->meta);
7e05942 @BohuTANG Add:bloomfilter when sst load
authored
551 bloom_free(sst->bloom);
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
552 free(sst);
eadbef6 @BohuTANG Fixup: some frees
authored
553 }
f98a9c9 @BohuTANG change the storage engine to LSM-Tree storage engine
authored
554 }
Something went wrong with that request. Please try again.