c/include/field_index.h c/src/field_index.c @@ -1,5 +1,6 @@ -These wonderful documentation standards were taken from merb. -Every method should have documentation above the method definition that fits this basic model... +These excellent documentation standards were taken from merb. +Every Ruby method should have documentation above the method definition that +fits this basic model... # Render the specified item, with the specified options. # @@ -21,7 +22,7 @@ Every method should have documentation above the method definition that fits thi # be either the controller_name or application. If you # want to use an alternative content-type than the one # that the base template was rendered as, you will need -# to do :layout => “foo.#{content_type}” (i.e. “foo.json”) +# to do :layout => +foo.#{content_type}+ (i.e. +foo.json+) # # ==== Returns # String:: The rendered template, including layout, if appropriate. @@ -32,9 +33,8 @@ Every method should have documentation above the method definition that fits thi # # ==== Alternatives # If you pass a Hash as the first parameter, it will be moved to -# opts and “thing” will be the current action +# opts and +thing+ will be the current action # -#— # @public def render(thing = nil, opts = {}) <snip> DOCUMENTATION_STANDARDS @@ -40,7 +40,7 @@ q_filtered_query.o q_fuzzy.o q_match_all.o q_multi_term.o \ q_parser.o q_phrase.o q_prefix.o q_range.o \ q_span.o q_term.o q_wildcard.o ram_store.o \ search.o similarity.o sort.o stopwords.o \ -store.o term_vectors.o +store.o term_vectors.o field_index.o TEST_OBJS = \ test_multimapper.o test_q_const_score.o test_threading.o \ c/Makefile @@ -772,7 +772,8 @@ struct IndexReader SegmentInfos *sis; FieldInfos *fis; HashTable *cache; - HashTable *sort_cache; + HashTable *field_index_cache; + mutex_t field_index_mutex; uchar *fake_norms; mutex_t mutex; bool has_changes : 1; c/include/index.h @@ -8,6 +8,7 @@ typedef struct Scorer Scorer; #include "index.h" #include "bitvector.h" #include "similarity.h" +#include "field_index.h" /*************************************************************************** * @@ -633,6 +634,22 @@ extern bool scorer_doc_less_than(const Scorer *s1, const Scorer *s2); extern int scorer_doc_cmp(const void *p1, const void *p2); /*************************************************************************** + * Comparable + ***************************************************************************/ + +typedef struct Comparable +{ + int type; + union { + long l; + float f; + char *s; + void *p; + } val; + bool reverse : 1; +} Comparable; + +/*************************************************************************** * * Sort * @@ -650,47 +667,27 @@ enum SORT_TYPE }; /*************************************************************************** - * Comparable - ***************************************************************************/ - -typedef struct Comparable -{ - int type; - union { - int i; - float f; - char *s; - void *p; - } val; - bool reverse : 1; -} Comparable; - -/*************************************************************************** * SortField ***************************************************************************/ typedef struct SortField { - mutex_t mutex; - char *field; - enum SORT_TYPE type; - bool reverse : 1; - void *index; - int (*compare)(void *index_ptr, Hit *hit1, Hit *hit2); - void (*get_val)(void *index_ptr, Hit *hit1, Comparable *comparable); - void *(*create_index)(int size); - void (*destroy_index)(void *p); - void (*handle_term)(void *index, TermDocEnum *tde, char *text); + char *field; + enum SORT_TYPE type; + bool reverse : 1; + const FieldIndexClass *field_index_class; + int (*compare)(void *index_ptr, Hit *hit1, Hit *hit2); + void (*get_val)(void *index_ptr, Hit *hit1, Comparable *comparable); } SortField; -extern SortField *sort_field_new(char *field, enum SORT_TYPE type, bool reverse); +extern SortField *sort_field_new(const char *field, enum SORT_TYPE type, bool reverse); extern SortField *sort_field_score_new(bool reverse); extern SortField *sort_field_doc_new(bool reverse); -extern SortField *sort_field_int_new(char *field, bool reverse); -extern SortField *sort_field_byte_new(char *field, bool reverse); -extern SortField *sort_field_float_new(char *field, bool reverse); -extern SortField *sort_field_string_new(char *field, bool reverse); -extern SortField *sort_field_auto_new(char *field, bool reverse); +extern SortField *sort_field_int_new(const char *field, bool reverse); +extern SortField *sort_field_byte_new(const char *field, bool reverse); +extern SortField *sort_field_float_new(const char *field, bool reverse); +extern SortField *sort_field_string_new(const char *field, bool reverse); +extern SortField *sort_field_auto_new(const char *field, bool reverse); extern void sort_field_destroy(void *p); extern char *sort_field_to_s(SortField *self); c/include/search.h @@ -3893,6 +3893,7 @@ IndexReader *ir_setup(IndexReader *ir, Store *store, SegmentInfos *sis, FieldInfos *fis, int is_owner) { mutex_init(&ir->mutex, NULL); + mutex_init(&ir->field_index_mutex, NULL); if (store) { ir->store = store; @@ -4123,8 +4124,8 @@ void ir_close(IndexReader *ir) if (ir->cache) { h_destroy(ir->cache); } - if (ir->sort_cache) { - h_destroy(ir->sort_cache); + if (ir->field_index_cache) { + h_destroy(ir->field_index_cache); } if (ir->deleter && ir->is_owner) { deleter_destroy(ir->deleter); @@ -4132,6 +4133,7 @@ void ir_close(IndexReader *ir) free(ir->fake_norms); mutex_destroy(&ir->mutex); + mutex_destroy(&ir->field_index_mutex); free(ir); } else { c/src/index.c @@ -1,6 +1,7 @@ #include <string.h> #include "search.h" #include "index.h" +#include "field_index.h" /*************************************************************************** * @@ -8,53 +9,24 @@ * ***************************************************************************/ -unsigned long sort_field_hash(const void *p) -{ - SortField *self = (SortField *)p; - return str_hash(self->field) ^ (self->type*37); -} - -int sort_field_eq(const void *p1, const void *p2) -{ - SortField *key1 = (SortField *)p1; - SortField *key2 = (SortField *)p2; - return (strcmp(key1->field, key2->field) == 0) - && key1->type == key2->type; -} - -static int sort_field_cache_eq(const void *p1, const void *p2) -{ - SortField *key1 = (SortField *)p1; - SortField *key2 = (SortField *)p2; - int equal = (strcmp(key1->field, key2->field) == 0) - && key1->type == key2->type; - - return equal; -} - -static SortField *sort_field_clone(SortField *self) -{ - SortField *clone = ALLOC(SortField); - memcpy(clone, self, sizeof(SortField)); - mutex_init(&clone->mutex, NULL); - clone->field = estrdup(self->field); - return clone; -} - -static SortField *sort_field_alloc(char *field, int type, bool reverse) -{ - SortField *self = ALLOC(SortField); - mutex_init(&self->mutex, NULL); - self->field = field ? estrdup(field) : NULL; - self->type = type; - self->reverse = reverse; - self->index = NULL; - self->destroy_index = &free; - self->compare = NULL; +static INLINE SortField *sort_field_alloc(const char *field, + enum SORT_TYPE type, + bool reverse, + int (*compare)(void *index_ptr, Hit *hit1, Hit *hit2), + void (*get_val)(void *index_ptr, Hit *hit1, Comparable *comparable), + const FieldIndexClass *field_index_class) +{ + SortField *self = ALLOC(SortField); + self->field = field ? estrdup(field) : NULL; + self->type = type; + self->reverse = reverse; + self->field_index_class = field_index_class; + self->compare = compare; + self->get_val = get_val; return self; } -SortField *sort_field_new(char *field, enum SORT_TYPE type, bool reverse) +SortField *sort_field_new(const char *field, enum SORT_TYPE type, bool reverse) { SortField *sf = NULL; switch (type) { @@ -86,11 +58,7 @@ SortField *sort_field_new(char *field, enum SORT_TYPE type, bool reverse) void sort_field_destroy(void *p) { SortField *self = (SortField *)p; - if (self->index) { - self->destroy_index(self->index); - } free(self->field); - mutex_destroy(&self->mutex); free(p); } @@ -125,10 +93,11 @@ char *sort_field_to_s(SortField *self) break; } if (self->field) { - str = ALLOC_N(char, 10 + strlen(self->field) + strlen(type)); + str = ALLOC_N(char, 3 + strlen(self->field) + strlen(type)); sprintf(str, "%s:%s%s", self->field, type, (self->reverse ? "!" : "")); - } else { - str = ALLOC_N(char, 10 + strlen(type)); + } + else { + str = ALLOC_N(char, 2 + strlen(type)); sprintf(str, "%s%s", type, (self->reverse ? "!" : "")); } return str; @@ -138,13 +107,13 @@ char *sort_field_to_s(SortField *self) * ScoreSortField ***************************************************************************/ -void sf_score_get_val(void *index, Hit *hit, Comparable *comparable) +static void sf_score_get_val(void *index, Hit *hit, Comparable *comparable) { (void)index; comparable->val.f = hit->score; } -int sf_score_compare(void *index_ptr, Hit *hit2, Hit *hit1) +static int sf_score_compare(void *index_ptr, Hit *hit2, Hit *hit1) { float val1 = hit1->score; float val2 = hit2->score; @@ -157,49 +126,39 @@ int sf_score_compare(void *index_ptr, Hit *hit2, Hit *hit1) SortField *sort_field_score_new(bool reverse) { - SortField *self = sort_field_alloc(NULL, SORT_TYPE_SCORE, reverse); - self->compare = &sf_score_compare; - self->get_val = &sf_score_get_val; - return self; + return sort_field_alloc(NULL, SORT_TYPE_SCORE, reverse, + &sf_score_compare, &sf_score_get_val, NULL); } const SortField SORT_FIELD_SCORE = { - MUTEX_INITIALIZER, NULL, /* field */ SORT_TYPE_SCORE, /* type */ false, /* reverse */ - NULL, /* index */ + NULL, /* field_index_class */ &sf_score_compare, /* compare */ &sf_score_get_val, /* get_val */ - NULL, /* create_index */ - NULL, /* destroy_index */ - NULL, /* handle_term */ }; const SortField SORT_FIELD_SCORE_REV = { - MUTEX_INITIALIZER, NULL, /* field */ SORT_TYPE_SCORE, /* type */ true, /* reverse */ - NULL, /* index */ + NULL, /* field_index_class */ &sf_score_compare, /* compare */ &sf_score_get_val, /* get_val */ - NULL, /* create_index */ - NULL, /* destroy_index */ - NULL, /* handle_term */ }; /************************************************************************** * DocSortField ***************************************************************************/ -void sf_doc_get_val(void *index, Hit *hit, Comparable *comparable) +static void sf_doc_get_val(void *index, Hit *hit, Comparable *comparable) { (void)index; - comparable->val.i = hit->doc; + comparable->val.l = hit->doc; } -int sf_doc_compare(void *index_ptr, Hit *hit1, Hit *hit2) +static int sf_doc_compare(void *index_ptr, Hit *hit1, Hit *hit2) { int val1 = hit1->doc; int val2 = hit2->doc; @@ -212,36 +171,26 @@ int sf_doc_compare(void *index_ptr, Hit *hit1, Hit *hit2) SortField *sort_field_doc_new(bool reverse) { - SortField *self = sort_field_alloc(NULL, SORT_TYPE_DOC, reverse); - self->compare = &sf_doc_compare; - self->get_val = &sf_doc_get_val; - return self; + return sort_field_alloc(NULL, SORT_TYPE_DOC, reverse, + &sf_doc_compare, &sf_doc_get_val, NULL); } const SortField SORT_FIELD_DOC = { - MUTEX_INITIALIZER, NULL, /* field */ SORT_TYPE_DOC, /* type */ false, /* reverse */ - NULL, /* index */ + NULL, /* field_index_class */ &sf_doc_compare, /* compare */ &sf_doc_get_val, /* get_val */ - NULL, /* create_index */ - NULL, /* destroy_index */ - NULL, /* handle_term */ }; const SortField SORT_FIELD_DOC_REV = { - MUTEX_INITIALIZER, NULL, /* field */ SORT_TYPE_DOC, /* type */ true, /* reverse */ - NULL, /* index */ + NULL, /* field_index_class */ &sf_doc_compare, /* compare */ &sf_doc_get_val, /* get_val */ - NULL, /* create_index */ - NULL, /* destroy_index */ - NULL, /* handle_term */ }; /*************************************************************************** @@ -250,117 +199,60 @@ const SortField SORT_FIELD_DOC_REV = { static void sf_byte_get_val(void *index, Hit *hit, Comparable *comparable) { - comparable->val.i = ((int *)index)[hit->doc]; + comparable->val.l = ((long *)index)[hit->doc]; } static int sf_byte_compare(void *index, Hit *hit1, Hit *hit2) { - int val1 = ((int *)index)[hit1->doc]; - int val2 = ((int *)index)[hit2->doc]; + long val1 = ((long *)index)[hit1->doc]; + long val2 = ((long *)index)[hit2->doc]; if (val1 > val2) return 1; else if (val1 < val2) return -1; else return 0; } -static void *sf_byte_create_index(int size) -{ - int *index = ALLOC_AND_ZERO_N(int, size + 1); - index[0]++; - return &index[1]; -} - -static void sf_byte_destroy_index(void *p) +SortField *sort_field_byte_new(const char *field, bool reverse) { - int *index = (int *)p; - free(&index[-1]); -} - -static void sf_byte_handle_term(void *index_ptr, TermDocEnum *tde, char *text) -{ - int *index = (int *)index_ptr; - int val = index[-1]++; - (void)text; - while (tde->next(tde)) { - index[tde->doc_num(tde)] = val; - } -} - -static void sort_field_byte_methods(SortField *self) -{ - self->type = SORT_TYPE_BYTE; - self->compare = &sf_byte_compare; - self->get_val = &sf_byte_get_val; - self->create_index = &sf_byte_create_index; - self->destroy_index = &sf_byte_destroy_index; - self->handle_term = &sf_byte_handle_term; -} - -SortField *sort_field_byte_new(char *field, bool reverse) -{ - SortField *self = sort_field_alloc(field, SORT_TYPE_BYTE, reverse); - sort_field_byte_methods(self); - return self; + return sort_field_alloc(field, SORT_TYPE_BYTE, reverse, + &sf_byte_compare, &sf_byte_get_val, + &BYTE_FIELD_INDEX_CLASS); } /*************************************************************************** * IntegerSortField ***************************************************************************/ -void sf_int_get_val(void *index, Hit *hit, Comparable *comparable) +static void sf_int_get_val(void *index, Hit *hit, Comparable *comparable) { - comparable->val.i = ((int *)index)[hit->doc]; + comparable->val.l = ((long *)index)[hit->doc]; } -int sf_int_compare(void *index, Hit *hit1, Hit *hit2) +static int sf_int_compare(void *index, Hit *hit1, Hit *hit2) { - int val1 = ((int *)index)[hit1->doc]; - int val2 = ((int *)index)[hit2->doc]; + long val1 = ((long *)index)[hit1->doc]; + long val2 = ((long *)index)[hit2->doc]; if (val1 > val2) return 1; else if (val1 < val2) return -1; else return 0; } -void *sf_int_create_index(int size) +SortField *sort_field_int_new(const char *field, bool reverse) { - return ALLOC_AND_ZERO_N(int, size); -} - -void sf_int_handle_term(void *index_ptr, TermDocEnum *tde, char *text) -{ - int *index = (int *)index_ptr; - int val; - sscanf(text, "%d", &val); - while (tde->next(tde)) { - index[tde->doc_num(tde)] = val; - } -} - -void sort_field_int_methods(SortField *self) -{ - self->type = SORT_TYPE_INTEGER; - self->compare = &sf_int_compare; - self->get_val = &sf_int_get_val; - self->create_index = &sf_int_create_index; - self->handle_term = &sf_int_handle_term; -} - -SortField *sort_field_int_new(char *field, bool reverse) -{ - SortField *self = sort_field_alloc(field, SORT_TYPE_INTEGER, reverse); - sort_field_int_methods(self); - return self; + return sort_field_alloc(field, SORT_TYPE_INTEGER, reverse, + &sf_int_compare, &sf_int_get_val, + &INTEGER_FIELD_INDEX_CLASS); } /*************************************************************************** * FloatSortField ***************************************************************************/ -void sf_float_get_val(void *index, Hit *hit, Comparable *comparable) +static void sf_float_get_val(void *index, Hit *hit, Comparable *comparable) { comparable->val.f = ((float *)index)[hit->doc]; } -int sf_float_compare(void *index, Hit *hit1, Hit *hit2) +static int sf_float_compare(void *index, Hit *hit1, Hit *hit2) { float val1 = ((float *)index)[hit1->doc]; float val2 = ((float *)index)[hit2->doc]; @@ -369,58 +261,25 @@ int sf_float_compare(void *index, Hit *hit1, Hit *hit2) else return 0; } -void *sf_float_create_index(int size) +SortField *sort_field_float_new(const char *field, bool reverse) { - return ALLOC_AND_ZERO_N(float, size); -} - -void sf_float_handle_term(void *index_ptr, TermDocEnum *tde, char *text) -{ - float *index = (float *)index_ptr; - float val; - sscanf(text, "%g", &val); - while (tde->next(tde)) { - index[tde->doc_num(tde)] = val; - } -} - -void sort_field_float_methods(SortField *self) -{ - self->type = SORT_TYPE_FLOAT; - self->compare = &sf_float_compare; - self->get_val = &sf_float_get_val; - self->create_index = &sf_float_create_index; - self->handle_term = &sf_float_handle_term; -} - -SortField *sort_field_float_new(char *field, bool reverse) -{ - SortField *self = sort_field_alloc(field, SORT_TYPE_FLOAT, reverse); - sort_field_float_methods(self); - return self; + return sort_field_alloc(field, SORT_TYPE_FLOAT, reverse, + &sf_float_compare, &sf_float_get_val, + &FLOAT_FIELD_INDEX_CLASS); } /*************************************************************************** * StringSortField ***************************************************************************/ -#define VALUES_ARRAY_START_SIZE 8 -typedef struct StringIndex { - int size; - int *index; - char **values; - int v_size; - int v_capa; -} StringIndex; - -void sf_string_get_val(void *index, Hit *hit, Comparable *comparable) +static void sf_string_get_val(void *index, Hit *hit, Comparable *comparable) { comparable->val.s = ((StringIndex *)index)->values[ ((StringIndex *)index)->index[hit->doc]]; } -int sf_string_compare(void *index, Hit *hit1, Hit *hit2) +static int sf_string_compare(void *index, Hit *hit1, Hit *hit2) { char *s1 = ((StringIndex *)index)->values[ ((StringIndex *)index)->index[hit1->doc]]; @@ -447,151 +306,20 @@ int sf_string_compare(void *index, Hit *hit1, Hit *hit2) */ } -void *sf_string_create_index(int size) -{ - StringIndex *self = ALLOC_AND_ZERO(StringIndex); - self->size = size; - self->index = ALLOC_AND_ZERO_N(int, size); - self->v_capa = VALUES_ARRAY_START_SIZE; - self->v_size = 1; /* leave the first value as NULL */ - self->values = ALLOC_AND_ZERO_N(char *, VALUES_ARRAY_START_SIZE); - return self; -} - -void sf_string_destroy_index(void *p) -{ - StringIndex *self = (StringIndex *)p; - int i; - free(self->index); - for (i = 0; i < self->v_size; i++) { - free(self->values[i]); - } - free(self->values); - free(self); -} - -void sf_string_handle_term(void *index_ptr, TermDocEnum *tde, char *text) -{ - StringIndex *index = (StringIndex *)index_ptr; - if (index->v_size >= index->v_capa) { - index->v_capa *= 2; - index->values = REALLOC_N(index->values, char *, index->v_capa); - } - index->values[index->v_size] = estrdup(text); - while (tde->next(tde)) { - index->index[tde->doc_num(tde)] = index->v_size; - } - index->v_size++; -} - -void sort_field_string_methods(SortField *self) -{ - self->type = SORT_TYPE_STRING; - self->compare = &sf_string_compare; - self->get_val = &sf_string_get_val; - self->create_index = &sf_string_create_index; - self->destroy_index = &sf_string_destroy_index; - self->handle_term = &sf_string_handle_term; -} - -SortField *sort_field_string_new(char *field, bool reverse) +SortField *sort_field_string_new(const char *field, bool reverse) { - SortField *self = sort_field_alloc(field, SORT_TYPE_STRING, reverse); - sort_field_string_methods(self); - return self; + return sort_field_alloc(field, SORT_TYPE_STRING, reverse, + &sf_string_compare, &sf_string_get_val, + &STRING_FIELD_INDEX_CLASS); } /*************************************************************************** * AutoSortField ***************************************************************************/ -void sort_field_auto_evaluate(SortField *sf, char *text) -{ - int int_val; - float float_val; - int text_len = 0, scan_len = 0; - - text_len = (int)strlen(text); - sscanf(text, "%d%n", &int_val, &scan_len); - if (scan_len == text_len) { - sort_field_int_methods(sf); - } else { - sscanf(text, "%f%n", &float_val, &scan_len); - if (scan_len == text_len) { - sort_field_float_methods(sf); - } else { - sort_field_string_methods(sf); - } - } -} - -SortField *sort_field_auto_new(char *field, bool reverse) -{ - return sort_field_alloc(field, SORT_TYPE_AUTO, reverse); -} - -/*************************************************************************** - * - * FieldCache - * - ***************************************************************************/ - -void *field_cache_get_index(IndexReader *ir, SortField *sf) +SortField *sort_field_auto_new(const char *field, bool reverse) { - void *volatile index = NULL; - int length = 0; - TermEnum *volatile te = NULL; - TermDocEnum *volatile tde = NULL; - SortField *sf_clone; - const int field_num = fis_get_field_num(ir->fis, sf->field); - - if (field_num < 0) { - RAISE(ARG_ERROR, - "Cannot sort by field \"%s\". It doesn't exist in the index.", - sf->field); - } - - mutex_lock(&sf->mutex); - if (!ir->sort_cache) { - ir->sort_cache = h_new(&sort_field_hash, &sort_field_cache_eq, - &sort_field_destroy, NULL); - } - - if (sf->type == SORT_TYPE_AUTO) { - te = ir->terms(ir, field_num); - if (!te->next(te) && (ir->num_docs(ir) > 0)) { - RAISE(ARG_ERROR, - "Cannot sort by field \"%s\" as there are no terms " - "in that field in the index.", sf->field); - } - sort_field_auto_evaluate(sf, te->curr_term); - te->close(te); - } - - index = h_get(ir->sort_cache, sf); - - if (index == NULL) { - length = ir->max_doc(ir); - if (length > 0) { - TRY - tde = ir->term_docs(ir); - te = ir->terms(ir, field_num); - index = sf->create_index(length); - while (te->next(te)) { - tde->seek_te(tde, te); - sf->handle_term(index, tde, te->curr_term); - } - XFINALLY - tde->close(tde); - te->close(te); - XENDTRY - } - sf_clone = sort_field_clone(sf); - sf_clone->index = index; - h_set(ir->sort_cache, sf_clone, index); - } - mutex_unlock(&sf->mutex); - return index; + return sort_field_alloc(field, SORT_TYPE_AUTO, reverse, NULL, NULL, NULL); } /*************************************************************************** @@ -630,12 +358,88 @@ typedef struct Sorter { Sort *sort; } Sorter; +#define SET_AUTO(upper_type, lower_type) \ + sf->type = SORT_TYPE_ ## upper_type;\ + sf->field_index_class = &upper_type ## _FIELD_INDEX_CLASS;\ + sf->compare = sf_ ## lower_type ## _compare;\ + sf->get_val = sf_ ## lower_type ## _get_val + +static void sort_field_auto_evaluate(SortField *sf, char *text) +{ + int int_val; + float float_val; + int text_len = 0, scan_len = 0; + + text_len = (int)strlen(text); + sscanf(text, "%d%n", &int_val, &scan_len); + if (scan_len == text_len) { + SET_AUTO(INTEGER, int); + } else { + sscanf(text, "%f%n", &float_val, &scan_len); + if (scan_len == text_len) { + SET_AUTO(FLOAT, float); + } else { + SET_AUTO(STRING, string); + } + } +} +/* +static INLINE void set_auto(SortField *sf, + enum SORT_TYPE type, + const FieldIndexClass *field_index_class, + int (*compare)(void *index_ptr, Hit *hit1, Hit *hit2), + void (*get_val)(void *index_ptr, Hit *hit1, Comparable *comparable)) +{ + sf->type = type; + sf->field_index_class = field_index_class; + sf->compare = compare; + sf->get_val = get_val; +} + +static void sort_field_auto_evaluate(SortField *sf, char *text) +{ + int int_val; + float float_val; + int text_len = 0, scan_len = 0; + + text_len = (int)strlen(text); + sscanf(text, "%d%n", &int_val, &scan_len); + if (scan_len == text_len) { + set_auto(sf, SORT_TYPE_INTEGER, &INTEGER_FIELD_INDEX_CLASS, + sf_int_compare, sf_int_get_val); + } else { + sscanf(text, "%f%n", &float_val, &scan_len); + if (scan_len == text_len) { + set_auto(sf, SORT_TYPE_FLOAT, &FLOAT_FIELD_INDEX_CLASS, + sf_float_compare, sf_float_get_val); + } else { + set_auto(sf, SORT_TYPE_STRING, &STRING_FIELD_INDEX_CLASS, + sf_string_compare, sf_string_get_val); + } + } +} +*/ + Comparator *sorter_get_comparator(SortField *sf, IndexReader *ir) { void *index = NULL; if (sf->type > SORT_TYPE_DOC) { - index = field_cache_get_index(ir, sf); + FieldIndex *field_index = NULL; + if (sf->type == SORT_TYPE_AUTO) { + TermEnum *te = ir_terms(ir, sf->field); + if (!te->next(te) && (ir->num_docs(ir) > 0)) { + RAISE(ARG_ERROR, + "Cannot sort by field \"%s\" as there are no terms " + "in that field in the index.", sf->field); + } + sort_field_auto_evaluate(sf, te->curr_term); + te->close(te); + } + mutex_lock(&ir->field_index_mutex); + field_index = field_index_new(ir, sf->field, sf->field_index_class); + mutex_unlock(&ir->field_index_mutex); + index = field_index->index; } return comparator_new(index, sf->reverse, sf->compare); } @@ -864,12 +668,12 @@ bool fdshq_lt(FieldDoc *fd1, FieldDoc *fd2) if (fd1->hit.doc < fd2->hit.doc) c = -1; break; case SORT_TYPE_INTEGER: - if (cmps1[i].val.i > cmps2[i].val.i) c = 1; - if (cmps1[i].val.i < cmps2[i].val.i) c = -1; + if (cmps1[i].val.l > cmps2[i].val.l) c = 1; + if (cmps1[i].val.l < cmps2[i].val.l) c = -1; break; case SORT_TYPE_BYTE: - if (cmps1[i].val.i > cmps2[i].val.i) c = 1; - if (cmps1[i].val.i < cmps2[i].val.i) c = -1; + if (cmps1[i].val.l > cmps2[i].val.l) c = 1; + if (cmps1[i].val.l < cmps2[i].val.l) c = -1; break; case SORT_TYPE_STRING: do { c/src/sort.c @@ -122,9 +122,9 @@ static void do_test_top_docs(tst_case *tc, Searcher *searcher, Query *query, case SORT_TYPE_FLOAT: printf("f:%f ", fd->comparables[j].val.f); break; case SORT_TYPE_DOC: - printf("d:%d ", fd->comparables[j].val.i); break; + printf("d:%ld ", fd->comparables[j].val.l); break; case SORT_TYPE_INTEGER: - printf("i:%d ", fd->comparables[j].val.i); break; + printf("i:%ld ", fd->comparables[j].val.l); break; case SORT_TYPE_STRING: printf("s:%s ", fd->comparables[j].val.s); break; default: c/test/test_sort.c @@ -16,66 +16,22 @@ end objs = Dir["src/*.c"].map {|full_path| File.basename(full_path).gsub(/c$/, "o")} objs << "q_parser.o" unless objs.index("q_parser.o") -test_objs = Dir["test/*.c"].map {|full_path| File.basename(full_path).gsub(/c$/, "o")} +OBJS = objs.join(' ') +TEST_OBJS = Dir["test/*.c"].map {|full_path| File.basename(full_path).gsub(/c$/, "o")}.join(' ') -print """ -CFLAGS = -std=c99 -pedantic -fno-stack-protector -Wall -Wextra -Iinclude -Ilib/libstemmer_c/include -fno-common -g -DDEBUG -D_FILE_OFFSET_BITS=64 - -LFLAGS = -lm -lpthread - -include lib/libstemmer_c/mkinc.mak - -STEMMER_OBJS = $(patsubst %.c,src/libstemmer_c/%.o, $(snowball_sources)) - -TEST_OBJS = #{test_objs.join(" ")} - -OBJS = #{objs.join(" ")} libstemmer.o - -vpath %.c test src - -vpath %.h test include lib/libstemmer_c/include - -runtests: testall - ./testall -v -f -q - -testall: $(OBJS) $(TEST_OBJS) - $(CC) $(CFLAGS) $(LFLAGS) $(OBJS) $(TEST_OBJS) -o testall - -valgrind: testall - valgrind --leak-check=yes --show-reachable=yes --workaround-gcc296-bugs=yes -v ./testall -q - -bench: bench.c $(OBJS) - $(CC) $(CFLAGS) -lpthread bench.c $(OBJS) $(LFLAGS) -o bench - -search_bench: search_bench.c $(OBJS) - $(CC) $(CFLAGS) search_bench.c $(OBJS) $(LFLAGS) -o search_bench - -sort_bench: sort_bench.c $(OBJS) - $(CC) $(CFLAGS) sort_bench.c $(OBJS) $(LFLAGS) -o sort_bench - -libstemmer.o: $(snowball_sources:%.c=lib/libstemmer_c/%.o) - $(AR) -cru $@ $^ - -q_parser.o: src/q_parser.c - $(CC) $(CFLAGS) src/q_parser.c -c - -src/q_parser.c: src/q_parser.y - -.PHONY: clean -clean: - rm -f *.o testall gmon.out bench search_bench -""" - -def get_deps(src) +def get_deps(src_file) deps = Set.new - direct_deps = $dep_tree[src] + direct_deps = $dep_tree[src_file] return deps unless direct_deps deps.merge(direct_deps) direct_deps.each {|dep| deps.merge(get_deps(dep))} deps end -$dep_tree.each_key do |src| - next unless src =~ /\.c$/ - puts src.gsub(/c$/, "o") + ": " + get_deps(src).to_a.join(" ") + "\n\n" -end +OBJECT_DEPENDENCIES = $dep_tree.keys.map {|src_file| + next unless src_file =~ /\.c$/ + src_file.gsub(/c$/, "o") + ": " + get_deps(src_file).to_a.join(" ") +}.compact.join("\n\n") + +MAKEFILE_TEMPLATE = File.read(File.join(File.dirname(__FILE__), '../Makefile.template')) +puts MAKEFILE_TEMPLATE.gsub(/%([A-Z_]+)%/) {eval($1)} c/tools/make_makefile.rb c/tools/mkmf.pl 8bc2898052585f89aa8a9838fe95fd4b42edd65d dbalmain dbalmain@06fd6eb0-0002-0410-a719-e5602cce40bc http://github.com/dbalmain/ferret/commit/a98ed8eff04bfb75b9e9bef649018bb2ffc652b6 a98ed8eff04bfb75b9e9bef649018bb2ffc652b6 2008-03-21T19:59:19-07:00 2008-03-21T19:59:19-07:00 * removed obsolete Perl makefile generator * extracted field-index functionality from sort. For example; FieldIndex *field_index = field_index_new(index_reader, "integer-field", INTEGER_FIELD_INDEX_CLASS); long val_in_doc_10 = ((long *)field_index->index)[10]; git-svn-id: svn+ssh://davebalmain.com/home/dave/repos/ferret/trunk@873 06fd6eb0-0002-0410-a719-e5602cce40bc 101ae45f6585a5bd136059ff2e27e781dc8eab34 dbalmain dbalmain@06fd6eb0-0002-0410-a719-e5602cce40bc