Permalink
Browse files

Add: stable log recovery

  • Loading branch information...
1 parent 230b9b6 commit 239b3e71ff7f7e415d3c3acf948e99575d7f005f @BohuTANG committed Jan 4, 2012
Showing with 99 additions and 73 deletions.
  1. +1 −0 CHANGELOG
  2. +1 −1 COPYING.BSD
  3. +3 −3 bench/db-bench.c
  4. +2 −2 engine/buffer.c
  5. +2 −2 engine/buffer.h
  6. +2 −2 engine/db.c
  7. +2 −2 engine/db.h
  8. +2 −2 engine/debug.h
  9. +1 −1 engine/ht.c
  10. +1 −1 engine/ht.h
  11. +32 −7 engine/index.c
  12. +2 −2 engine/index.h
  13. +1 −1 engine/level.c
  14. +1 −1 engine/level.h
  15. +1 −1 engine/llru.c
  16. +1 −1 engine/llru.h
  17. +20 −22 engine/log.c
  18. +4 −2 engine/log.h
  19. +2 −2 engine/meta.c
  20. +2 −2 engine/meta.h
  21. +2 −2 engine/platform.h
  22. +2 −2 engine/skiplist.c
  23. +2 −2 engine/skiplist.h
  24. +2 −2 engine/sst.c
  25. +2 −2 engine/sst.h
  26. +3 −3 engine/util.c
  27. +3 −3 engine/util.h
View
@@ -1,3 +1,4 @@
+2012-01-04 add stable log recovery. (BohuTANG)
2011-12-29 add slim 'read-lock' and 'write-lock',and add LLRU, this is a big deal. (BohuTANG)
2011-12-25 add 'background-merge' thread, Merry christmas. (BohuTANG)
View
@@ -1,4 +1,4 @@
-Copyright (c) 2011, BohuTANG(overred.shuttler at gmail dot com)
+Copyright (c) 2011-2012, BohuTANG(overred.shuttler at gmail dot com)
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
View
@@ -1,6 +1,6 @@
/*
- * nessDB
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
@@ -24,7 +24,7 @@
#include "../engine/util.h"
#include "../engine/debug.h"
-#define TOLOG (0)
+#define TOLOG (1)
#define KSIZE 16
#define VSIZE 80
#define BUFFERPOOL (1024*1024*1024)
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * nessDB
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * nessDB
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
* nessDB storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
* nessDB storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
@@ -50,7 +50,7 @@ void *_merge_job(void *arg)
log = idx->log;
if(list == NULL)
- goto out;
+ goto merge_out;
pthread_mutex_lock(&idx->merge_mutex);
sst_merge(sst, list);
@@ -59,7 +59,7 @@ void *_merge_job(void *arg)
/* Lock end */
log_remove(log, lsn);
-out:
+merge_out:
pthread_detach(pthread_self());
pthread_exit(NULL);
}
@@ -73,7 +73,7 @@ struct index *index_new(const char *basedir, const char *name, int max_mtbl_size
memset(dir, 0, INDEX_NSIZE);
snprintf(dir, INDEX_NSIZE, "%s/%s", basedir, DB_DIR);
- _ensure_dir_exists(dir);
+ ensure_dir_exists(dir);
idx->lsn = 0;
idx->max_mtbl = 1;
@@ -95,7 +95,29 @@ struct index *index_new(const char *basedir, const char *name, int max_mtbl_size
/* log */
idx->log = log_new(idx->basedir, idx->lsn, tolog);
- log_recovery(idx->log, idx->list);
+
+ /*
+ * Log Recovery Processes:
+ * 1) read old log file and add entries to memtable
+ * 2) read new log file and add entries to memtable
+ * 3) merge the current active log's memtable
+ * 4) remove old log file,new log file
+ * 5) create new memtable and log file
+ */
+ if (log_recovery(idx->log, idx->list)) {
+ __DEBUG("%s", "logs need to recovery....");
+ /* Merge log entries */
+ sst_merge(idx->sst, idx->list);
+
+ /* Remove old&new log files */
+ remove(idx->log->log_new);
+ remove(idx->log->log_old);
+
+ /* Create new memtable */
+ idx->list = skiplist_new(idx->max_mtbl_size);
+ }
+
+ /* Create new log:0.log */
log_next(idx->log, 0);
memset(dbfile, 0, DB_NSIZE);
@@ -106,7 +128,6 @@ struct index *index_new(const char *basedir, const char *name, int max_mtbl_size
pthread_attr_init(&idx->attr);
pthread_attr_setdetachstate(&idx->attr, PTHREAD_CREATE_DETACHED);
-
return idx;
}
@@ -151,10 +172,14 @@ int index_add(struct index *idx, struct slice *sk, struct slice *sv)
return 1;
}
+/*
+ * When db is normally closed, flush current active memtable to disk sst
+ */
void _index_flush(struct index *idx)
{
struct skiplist *list;
+ /* Waiting bg merging thread done */
pthread_mutex_lock(&idx->merge_mutex);
pthread_mutex_unlock(&idx->merge_mutex);
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
* nessDB storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
* nessDB storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
* nessDB storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
* nessDB storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
@@ -63,10 +63,6 @@ struct log *log_new(const char *basedir, int lsn, int islog)
memset(db_name, 0, LOG_NSIZE);
snprintf(db_name, LOG_NSIZE, "%s/ness.db", basedir);
- if (_file_exists(log_name))
- l->idx_wfd = open(log_name, LSM_OPEN_FLAGS, 0644);
- else
- l->idx_wfd = open(log_name, LSM_CREAT_FLAGS, 0644);
if (_file_exists(db_name)) {
l->db_wfd = open(db_name, LSM_OPEN_FLAGS, 0644);
@@ -141,8 +137,7 @@ void _log_read(char *logname, struct skiplist *list)
int log_recovery(struct log *l, struct skiplist *list)
{
DIR *dd;
- int isnew = 1;
- char logname[LOG_NSIZE];
+ int ret = 0;
char new_log[LOG_NSIZE];
char old_log[LOG_NSIZE];
struct dirent *de;
@@ -154,27 +149,30 @@ int log_recovery(struct log *l, struct skiplist *list)
while ((de = readdir(dd))) {
char *p = strstr(de->d_name, ".log");
if (p) {
- if (isnew == 1) {
+ if (ret == 0)
memcpy(new_log, de->d_name, LOG_NSIZE);
- isnew = 0;
- } else
+ else
memcpy(old_log, de->d_name, LOG_NSIZE);
+ ret = 1;
}
}
closedir(dd);
- /* TODO: stable log revoery */
- memset(logname, 0, LOG_NSIZE);
- snprintf(logname, LOG_NSIZE, "%s/%s", l->basedir, old_log);
- _log_read(logname, list);
- remove(logname);
-
- memset(logname, 0, LOG_NSIZE);
- snprintf(logname, LOG_NSIZE, "%s/%s", l->basedir, new_log);
- _log_read(logname, list);
- remove(logname);
+ /*
+ * Get the two log files:new and old
+ * Read must be sequential,read old then read new
+ */
+ if (ret) {
+ memset(l->log_old, 0, LOG_NSIZE);
+ snprintf(l->log_old, LOG_NSIZE, "%s/%s", l->basedir, old_log);
+ _log_read(l->log_old, list);
+
+ memset(l->log_new, 0, LOG_NSIZE);
+ snprintf(l->log_new, LOG_NSIZE, "%s/%s", l->basedir, new_log);
+ _log_read(l->log_new, list);
+ }
- return 0;
+ return ret;
}
uint64_t log_append(struct log *l, struct slice *sk, struct slice *sv)
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
@@ -21,6 +21,8 @@ struct log{
uint64_t db_alloc;
char name[LOG_NSIZE];
char basedir[LOG_NSIZE];
+ char log_old[LOG_NSIZE];
+ char log_new[LOG_NSIZE];
struct buffer *buf;
struct buffer *db_buf;
};
View
@@ -6,8 +6,8 @@
* +--------+--------+--------+--------+
*
*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -22,8 +22,8 @@
* | crc |
* +--------+--------+--------+--------+
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
View
@@ -1,6 +1,6 @@
/*
- * LSM-Tree storage engine
- * Copyright (c) 2011, BohuTANG <overred.shuttler at gmail dot com>
+ * nessDB storage engine
+ * Copyright (c) 2011-2012, BohuTANG <overred.shuttler at gmail dot com>
* All rights reserved.
* Code is licensed with BSD. See COPYING.BSD file.
*
@@ -50,7 +50,7 @@ static void _mkdirs(const char *path)
_portable_mkdir(tmp);
}
-void _ensure_dir_exists(const char *path)
+void ensure_dir_exists(const char *path)
{
struct stat st;
if(stat(path, &st) != 0) {
Oops, something went wrong.

0 comments on commit 239b3e7

Please sign in to comment.