Permalink
Browse files

check mmap_objs for size changes before all reads

And reload as necessary.
  • Loading branch information...
1 parent b43905f commit 3ee4d6d5060045e72381d0bbdd4b382c5a5d36e0 William Morgan committed Mar 23, 2012
Showing with 40 additions and 3 deletions.
  1. +3 −0 index.c
  2. +17 −2 mmap-obj.c
  3. +6 −1 mmap-obj.h
  4. +11 −0 segment.c
  5. +3 −0 segment.h
View
@@ -114,6 +114,7 @@ wp_error* wp_index_run_query(wp_index* index, wp_query* query, uint32_t max_num_
DEBUG("searching segment %d", query->segment_idx);
wp_segment* seg = &index->segments[query->segment_idx];
RELAY_ERROR(wp_segment_grab_readlock(seg));
+ RELAY_ERROR(wp_segment_reload(seg));
RELAY_ERROR(wp_search_run_query_on_segment(query, seg, want_num_results, &got_num_results, segment_results));
RELAY_ERROR(wp_segment_release_readlock(seg));
DEBUG("asked segment %d for %d results, got %d", query->segment_idx, want_num_results, got_num_results);
@@ -235,6 +236,7 @@ wp_error* wp_index_dumpinfo(wp_index* index, FILE* stream) {
fprintf(stream, "\nsegment %d:\n", i);
wp_segment* seg = &index->segments[i];
RELAY_ERROR(wp_segment_grab_readlock(seg));
+ RELAY_ERROR(wp_segment_reload(seg));
RELAY_ERROR(wp_segment_dumpinfo(seg, stream));
RELAY_ERROR(wp_segment_release_readlock(seg));
}
@@ -310,6 +312,7 @@ wp_error* wp_index_num_docs(wp_index* index, uint64_t* num_docs) {
for(uint32_t i = index->num_segments; i > 0; i--) {
wp_segment* seg = &index->segments[i - 1];
RELAY_ERROR(wp_segment_grab_readlock(seg));
+ RELAY_ERROR(wp_segment_reload(seg));
*num_docs += wp_segment_num_docs(seg);
RELAY_ERROR(wp_segment_release_readlock(seg));
}
View
@@ -21,7 +21,7 @@ wp_error* mmap_obj_create(mmap_obj* o, const char* magic, const char* pathname,
o->content = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
if(o->content == MAP_FAILED) RAISE_SYSERROR("mmap");
strncpy(o->content->magic, magic, MMAP_OBJ_MAGIC_SIZE);
- o->content->size = initial_size;
+ o->content->size = o->loaded_size = initial_size;
DEBUG("created new %s object with %u bytes", magic, size);
return NO_ERROR;
@@ -39,6 +39,8 @@ wp_error* mmap_obj_load(mmap_obj* o, const char* magic, const char* pathname) {
RELAY_ERROR(validate(o->content, magic));
+ o->loaded_size = o->content->size;
+
uint32_t size = o->content->size + (uint32_t)sizeof(mmap_obj_header);
DEBUG("full size is %u bytes (including %u-byte header)", size, sizeof(mmap_obj_header));
if(munmap(o->content, sizeof(mmap_obj_header)) == -1) RAISE_SYSERROR("munmap");
@@ -50,6 +52,19 @@ wp_error* mmap_obj_load(mmap_obj* o, const char* magic, const char* pathname) {
return NO_ERROR;
}
+wp_error* mmap_obj_reload(mmap_obj* o) {
+ if(o->loaded_size != o->content->size) {
+ DEBUG("need to reload %s because size of %u is now %u", o->content->magic, o->loaded_size, o->content->size);
+ uint32_t new_size = o->content->size + (uint32_t)sizeof(mmap_obj_header);
+ if(munmap(o->content, sizeof(mmap_obj_header) + o->loaded_size) == -1) RAISE_SYSERROR("munmap");
+ o->content = mmap(NULL, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
+ if(o->content == MAP_FAILED) RAISE_SYSERROR("mmap");
+ DEBUG("loaded %u bytes after reload. header is at %p", o->content->size, o->content);
+ }
+
+ return NO_ERROR;
+}
+
wp_error* mmap_obj_resize(mmap_obj* o, uint32_t data_size) {
DEBUG("going to expand from %u to %u bytes. current header is at %p", o->content->size, data_size, o->content);
@@ -62,7 +77,7 @@ wp_error* mmap_obj_resize(mmap_obj* o, uint32_t data_size) {
//lseek(fd, 0, SEEK_SET); // not necessary!
o->content = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
if(o->content == MAP_FAILED) RAISE_SYSERROR("mmap");
- o->content->size = data_size;
+ o->content->size = o->loaded_size = data_size;
DEBUG("loaded %u bytes after resize. header is at %p", o->content->size, o->content);
return NO_ERROR;
View
@@ -7,7 +7,7 @@
// wrappers around the logic of loading, unloading, and resizing
// arbitrary-sized objects using mmap.
//
-// note that aany of the mmap_obj_* functions may change the object pointer, so
+// note that any of the mmap_obj_* functions may change the object pointer, so
// use MMAP_OBJ or MAP_OBJ_PTR to dereference (again) after calling them.
#define MMAP_OBJ_MAGIC_SIZE 15
@@ -25,6 +25,7 @@ typedef struct mmap_obj_header {
// what we pass around at runtime
typedef struct mmap_obj {
int fd;
+ uint32_t loaded_size; // compare against header->sizer
mmap_obj_header* content;
} mmap_obj;
@@ -43,6 +44,10 @@ wp_error* mmap_obj_create(mmap_obj* o, const char* magic, const char* pathname,
// magic doesn't match)
wp_error* mmap_obj_load(mmap_obj* o, const char* magic, const char* pathname) RAISES_ERROR;
+// public: load an object, but only if the size has changed since the
+// first load.
+wp_error* mmap_obj_reload(mmap_obj* o) RAISES_ERROR;
+
// public: resize an object. note that the obj pointer might change after this call.
wp_error* mmap_obj_resize(mmap_obj* o, uint32_t new_size) RAISES_ERROR;
View
@@ -101,6 +101,17 @@ wp_error* wp_segment_load(wp_segment* segment, const char* pathname_base) {
return NO_ERROR;
}
+wp_error* wp_segment_reload(wp_segment* segment) {
+ RELAY_ERROR(mmap_obj_reload(&segment->seginfo));
+ RELAY_ERROR(mmap_obj_reload(&segment->stringpool));
+ RELAY_ERROR(mmap_obj_reload(&segment->stringmap));
+ RELAY_ERROR(mmap_obj_reload(&segment->termhash));
+ RELAY_ERROR(mmap_obj_reload(&segment->postings));
+ RELAY_ERROR(mmap_obj_reload(&segment->labels));
+
+ return NO_ERROR;
+}
+
wp_error* wp_segment_create(wp_segment* segment, const char* pathname_base) {
char fn[FN_SIZE];
View
@@ -101,6 +101,9 @@ wp_error* wp_segment_create(wp_segment* segment, const char* pathname_base) RAIS
// public: load a segment, raising an error unless it already exists
wp_error* wp_segment_load(wp_segment* segment, const char* pathname_base) RAISES_ERROR;
+// public: reload a segment as necessary, in case an external writer has changed the mmap object sizes
+wp_error* wp_segment_reload(wp_segment* segment) RAISES_ERROR;
+
// public: unload a segment
wp_error* wp_segment_unload(wp_segment* s) RAISES_ERROR;

0 comments on commit 3ee4d6d

Please sign in to comment.