From cc968905cb22542e7152ee8a92e5cc652bcbe82f Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Wed, 6 Apr 2016 10:49:43 +0900 Subject: [PATCH] Split cache related code --- lib/cache.c | 271 +++++++++++++++++++++++++++++++++++++++++ lib/ctx.c | 249 +------------------------------------ lib/db.c | 1 + lib/grn_cache.h | 50 ++++++++ lib/grn_ctx.h | 23 ---- lib/mrb/mrb_cache.c | 1 + lib/proc.c | 1 + lib/proc/proc_select.c | 1 + lib/sources.am | 2 + 9 files changed, 328 insertions(+), 271 deletions(-) create mode 100644 lib/cache.c create mode 100644 lib/grn_cache.h diff --git a/lib/cache.c b/lib/cache.c new file mode 100644 index 0000000000..a4d9c15357 --- /dev/null +++ b/lib/cache.c @@ -0,0 +1,271 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2009-2016 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "grn_cache.h" +#include "grn_ctx.h" +#include "grn_ctx_impl.h" +#include "grn_hash.h" +#include "grn_db.h" + +typedef struct _grn_cache_entry grn_cache_entry; + +struct _grn_cache { + grn_cache_entry *next; + grn_cache_entry *prev; + grn_hash *hash; + grn_mutex mutex; + uint32_t max_nentries; + uint32_t nfetches; + uint32_t nhits; +}; + +struct _grn_cache_entry { + grn_cache_entry *next; + grn_cache_entry *prev; + grn_obj *value; + grn_timeval tv; + grn_id id; + uint32_t nref; +}; + +static grn_cache *grn_cache_current = NULL; +static grn_cache *grn_cache_default = NULL; + +grn_cache * +grn_cache_open(grn_ctx *ctx) +{ + grn_cache *cache = NULL; + + GRN_API_ENTER; + cache = GRN_MALLOC(sizeof(grn_cache)); + if (!cache) { + ERR(GRN_NO_MEMORY_AVAILABLE, "[cache] failed to allocate grn_cache"); + goto exit; + } + + cache->next = (grn_cache_entry *)cache; + cache->prev = (grn_cache_entry *)cache; + cache->hash = grn_hash_create(&grn_gctx, NULL, GRN_CACHE_MAX_KEY_SIZE, + sizeof(grn_cache_entry), GRN_OBJ_KEY_VAR_SIZE); + MUTEX_INIT(cache->mutex); + cache->max_nentries = GRN_CACHE_DEFAULT_MAX_N_ENTRIES; + cache->nfetches = 0; + cache->nhits = 0; + +exit : + GRN_API_RETURN(cache); +} + +grn_rc +grn_cache_close(grn_ctx *ctx, grn_cache *cache) +{ + grn_ctx *ctx_original = ctx; + grn_cache_entry *vp; + + GRN_API_ENTER; + + ctx = &grn_gctx; + GRN_HASH_EACH(ctx, cache->hash, id, NULL, NULL, &vp, { + grn_obj_close(ctx, vp->value); + }); + grn_hash_close(ctx, cache->hash); + MUTEX_FIN(cache->mutex); + ctx = ctx_original; + GRN_FREE(cache); + + GRN_API_RETURN(ctx->rc); +} + +grn_rc +grn_cache_current_set(grn_ctx *ctx, grn_cache *cache) +{ + grn_cache_current = cache; + return GRN_SUCCESS; +} + +grn_cache * +grn_cache_current_get(grn_ctx *ctx) +{ + return grn_cache_current; +} + +void +grn_cache_init(void) +{ + grn_cache_default = grn_cache_open(&grn_gctx); + grn_cache_current_set(&grn_gctx, grn_cache_default); +} + +grn_rc +grn_cache_set_max_n_entries(grn_ctx *ctx, grn_cache *cache, unsigned int n) +{ + uint32_t current_max_n_entries; + + if (!cache) { + return GRN_INVALID_ARGUMENT; + } + + current_max_n_entries = cache->max_nentries; + cache->max_nentries = n; + if (n < current_max_n_entries) { + grn_cache_expire(cache, current_max_n_entries - n); + } + + return GRN_SUCCESS; +} + +uint32_t +grn_cache_get_max_n_entries(grn_ctx *ctx, grn_cache *cache) +{ + if (!cache) { + return 0; + } + return cache->max_nentries; +} + +void +grn_cache_get_statistics(grn_ctx *ctx, grn_cache *cache, + grn_cache_statistics *statistics) +{ + MUTEX_LOCK(cache->mutex); + statistics->nentries = GRN_HASH_SIZE(cache->hash); + statistics->max_nentries = cache->max_nentries; + statistics->nfetches = cache->nfetches; + statistics->nhits = cache->nhits; + MUTEX_UNLOCK(cache->mutex); +} + +static void +grn_cache_expire_entry(grn_cache *cache, grn_cache_entry *ce) +{ + if (!ce->nref) { + ce->prev->next = ce->next; + ce->next->prev = ce->prev; + grn_obj_close(&grn_gctx, ce->value); + grn_hash_delete_by_id(&grn_gctx, cache->hash, ce->id, NULL); + } +} + +grn_obj * +grn_cache_fetch(grn_ctx *ctx, grn_cache *cache, + const char *str, uint32_t str_len) +{ + grn_cache_entry *ce; + grn_obj *obj = NULL; + if (!ctx->impl || !ctx->impl->db) { return obj; } + MUTEX_LOCK(cache->mutex); + cache->nfetches++; + if (grn_hash_get(&grn_gctx, cache->hash, str, str_len, (void **)&ce)) { + if (ce->tv.tv_sec <= grn_db_lastmod(ctx->impl->db)) { + grn_cache_expire_entry(cache, ce); + goto exit; + } + ce->nref++; + obj = ce->value; + ce->prev->next = ce->next; + ce->next->prev = ce->prev; + { + grn_cache_entry *ce0 = (grn_cache_entry *)cache; + ce->next = ce0->next; + ce->prev = ce0; + ce0->next->prev = ce; + ce0->next = ce; + } + cache->nhits++; + } +exit : + MUTEX_UNLOCK(cache->mutex); + return obj; +} + +void +grn_cache_unref(grn_ctx *ctx, grn_cache *cache, + const char *str, uint32_t str_len) +{ + grn_cache_entry *ce; + ctx = &grn_gctx; + MUTEX_LOCK(cache->mutex); + if (grn_hash_get(ctx, cache->hash, str, str_len, (void **)&ce)) { + if (ce->nref) { ce->nref--; } + } + MUTEX_UNLOCK(cache->mutex); +} + +void +grn_cache_update(grn_ctx *ctx, grn_cache *cache, + const char *str, uint32_t str_len, grn_obj *value) +{ + grn_id id; + int added = 0; + grn_cache_entry *ce; + grn_rc rc = GRN_SUCCESS; + grn_obj *old = NULL, *obj; + if (!ctx->impl || !cache->max_nentries) { return; } + if (!(obj = grn_obj_open(&grn_gctx, GRN_BULK, 0, GRN_DB_TEXT))) { return; } + GRN_TEXT_PUT(&grn_gctx, obj, GRN_TEXT_VALUE(value), GRN_TEXT_LEN(value)); + MUTEX_LOCK(cache->mutex); + if ((id = grn_hash_add(&grn_gctx, cache->hash, str, str_len, (void **)&ce, &added))) { + if (!added) { + if (ce->nref) { + rc = GRN_RESOURCE_BUSY; + goto exit; + } + old = ce->value; + ce->prev->next = ce->next; + ce->next->prev = ce->prev; + } + ce->id = id; + ce->value = obj; + ce->tv = ctx->impl->tv; + ce->nref = 0; + { + grn_cache_entry *ce0 = (grn_cache_entry *)cache; + ce->next = ce0->next; + ce->prev = ce0; + ce0->next->prev = ce; + ce0->next = ce; + } + if (GRN_HASH_SIZE(cache->hash) > cache->max_nentries) { + grn_cache_expire_entry(cache, cache->prev); + } + } else { + rc = GRN_NO_MEMORY_AVAILABLE; + } +exit : + MUTEX_UNLOCK(cache->mutex); + if (rc) { grn_obj_close(&grn_gctx, obj); } + if (old) { grn_obj_close(&grn_gctx, old); } +} + +void +grn_cache_expire(grn_cache *cache, int32_t size) +{ + grn_cache_entry *ce0 = (grn_cache_entry *)cache; + MUTEX_LOCK(cache->mutex); + while (ce0 != ce0->prev && size--) { + grn_cache_expire_entry(cache, ce0->prev); + } + MUTEX_UNLOCK(cache->mutex); +} + +void +grn_cache_fin(void) +{ + grn_cache_current_set(&grn_gctx, NULL); + grn_cache_close(&grn_gctx, grn_cache_default); +} diff --git a/lib/ctx.c b/lib/ctx.c index 4ddd25b2ce..11fc74f705 100644 --- a/lib/ctx.c +++ b/lib/ctx.c @@ -33,6 +33,7 @@ #include "grn_mrb.h" #include "grn_ctx_impl_mrb.h" #include "grn_logger.h" +#include "grn_cache.h" #include #include #include @@ -1905,254 +1906,6 @@ grn_ctx_info_get(grn_ctx *ctx, grn_ctx_info *info) } -typedef struct _grn_cache_entry grn_cache_entry; - -struct _grn_cache { - grn_cache_entry *next; - grn_cache_entry *prev; - grn_hash *hash; - grn_mutex mutex; - uint32_t max_nentries; - uint32_t nfetches; - uint32_t nhits; -}; - -struct _grn_cache_entry { - grn_cache_entry *next; - grn_cache_entry *prev; - grn_obj *value; - grn_timeval tv; - grn_id id; - uint32_t nref; -}; - -static grn_cache *grn_cache_current = NULL; -static grn_cache *grn_cache_default = NULL; - -grn_cache * -grn_cache_open(grn_ctx *ctx) -{ - grn_cache *cache = NULL; - - GRN_API_ENTER; - cache = GRN_MALLOC(sizeof(grn_cache)); - if (!cache) { - ERR(GRN_NO_MEMORY_AVAILABLE, "[cache] failed to allocate grn_cache"); - goto exit; - } - - cache->next = (grn_cache_entry *)cache; - cache->prev = (grn_cache_entry *)cache; - cache->hash = grn_hash_create(&grn_gctx, NULL, GRN_CACHE_MAX_KEY_SIZE, - sizeof(grn_cache_entry), GRN_OBJ_KEY_VAR_SIZE); - MUTEX_INIT(cache->mutex); - cache->max_nentries = GRN_CACHE_DEFAULT_MAX_N_ENTRIES; - cache->nfetches = 0; - cache->nhits = 0; - -exit : - GRN_API_RETURN(cache); -} - -grn_rc -grn_cache_close(grn_ctx *ctx, grn_cache *cache) -{ - grn_ctx *ctx_original = ctx; - grn_cache_entry *vp; - - GRN_API_ENTER; - - ctx = &grn_gctx; - GRN_HASH_EACH(ctx, cache->hash, id, NULL, NULL, &vp, { - grn_obj_close(ctx, vp->value); - }); - grn_hash_close(ctx, cache->hash); - MUTEX_FIN(cache->mutex); - ctx = ctx_original; - GRN_FREE(cache); - - GRN_API_RETURN(ctx->rc); -} - -grn_rc -grn_cache_current_set(grn_ctx *ctx, grn_cache *cache) -{ - grn_cache_current = cache; - return GRN_SUCCESS; -} - -grn_cache * -grn_cache_current_get(grn_ctx *ctx) -{ - return grn_cache_current; -} - -void -grn_cache_init(void) -{ - grn_cache_default = grn_cache_open(&grn_gctx); - grn_cache_current_set(&grn_gctx, grn_cache_default); -} - -grn_rc -grn_cache_set_max_n_entries(grn_ctx *ctx, grn_cache *cache, unsigned int n) -{ - uint32_t current_max_n_entries; - - if (!cache) { - return GRN_INVALID_ARGUMENT; - } - - current_max_n_entries = cache->max_nentries; - cache->max_nentries = n; - if (n < current_max_n_entries) { - grn_cache_expire(cache, current_max_n_entries - n); - } - - return GRN_SUCCESS; -} - -uint32_t -grn_cache_get_max_n_entries(grn_ctx *ctx, grn_cache *cache) -{ - if (!cache) { - return 0; - } - return cache->max_nentries; -} - -void -grn_cache_get_statistics(grn_ctx *ctx, grn_cache *cache, - grn_cache_statistics *statistics) -{ - MUTEX_LOCK(cache->mutex); - statistics->nentries = GRN_HASH_SIZE(cache->hash); - statistics->max_nentries = cache->max_nentries; - statistics->nfetches = cache->nfetches; - statistics->nhits = cache->nhits; - MUTEX_UNLOCK(cache->mutex); -} - -static void -grn_cache_expire_entry(grn_cache *cache, grn_cache_entry *ce) -{ - if (!ce->nref) { - ce->prev->next = ce->next; - ce->next->prev = ce->prev; - grn_obj_close(&grn_gctx, ce->value); - grn_hash_delete_by_id(&grn_gctx, cache->hash, ce->id, NULL); - } -} - -grn_obj * -grn_cache_fetch(grn_ctx *ctx, grn_cache *cache, - const char *str, uint32_t str_len) -{ - grn_cache_entry *ce; - grn_obj *obj = NULL; - if (!ctx->impl || !ctx->impl->db) { return obj; } - MUTEX_LOCK(cache->mutex); - cache->nfetches++; - if (grn_hash_get(&grn_gctx, cache->hash, str, str_len, (void **)&ce)) { - if (ce->tv.tv_sec <= grn_db_lastmod(ctx->impl->db)) { - grn_cache_expire_entry(cache, ce); - goto exit; - } - ce->nref++; - obj = ce->value; - ce->prev->next = ce->next; - ce->next->prev = ce->prev; - { - grn_cache_entry *ce0 = (grn_cache_entry *)cache; - ce->next = ce0->next; - ce->prev = ce0; - ce0->next->prev = ce; - ce0->next = ce; - } - cache->nhits++; - } -exit : - MUTEX_UNLOCK(cache->mutex); - return obj; -} - -void -grn_cache_unref(grn_ctx *ctx, grn_cache *cache, - const char *str, uint32_t str_len) -{ - grn_cache_entry *ce; - ctx = &grn_gctx; - MUTEX_LOCK(cache->mutex); - if (grn_hash_get(ctx, cache->hash, str, str_len, (void **)&ce)) { - if (ce->nref) { ce->nref--; } - } - MUTEX_UNLOCK(cache->mutex); -} - -void -grn_cache_update(grn_ctx *ctx, grn_cache *cache, - const char *str, uint32_t str_len, grn_obj *value) -{ - grn_id id; - int added = 0; - grn_cache_entry *ce; - grn_rc rc = GRN_SUCCESS; - grn_obj *old = NULL, *obj; - if (!ctx->impl || !cache->max_nentries) { return; } - if (!(obj = grn_obj_open(&grn_gctx, GRN_BULK, 0, GRN_DB_TEXT))) { return; } - GRN_TEXT_PUT(&grn_gctx, obj, GRN_TEXT_VALUE(value), GRN_TEXT_LEN(value)); - MUTEX_LOCK(cache->mutex); - if ((id = grn_hash_add(&grn_gctx, cache->hash, str, str_len, (void **)&ce, &added))) { - if (!added) { - if (ce->nref) { - rc = GRN_RESOURCE_BUSY; - goto exit; - } - old = ce->value; - ce->prev->next = ce->next; - ce->next->prev = ce->prev; - } - ce->id = id; - ce->value = obj; - ce->tv = ctx->impl->tv; - ce->nref = 0; - { - grn_cache_entry *ce0 = (grn_cache_entry *)cache; - ce->next = ce0->next; - ce->prev = ce0; - ce0->next->prev = ce; - ce0->next = ce; - } - if (GRN_HASH_SIZE(cache->hash) > cache->max_nentries) { - grn_cache_expire_entry(cache, cache->prev); - } - } else { - rc = GRN_NO_MEMORY_AVAILABLE; - } -exit : - MUTEX_UNLOCK(cache->mutex); - if (rc) { grn_obj_close(&grn_gctx, obj); } - if (old) { grn_obj_close(&grn_gctx, old); } -} - -void -grn_cache_expire(grn_cache *cache, int32_t size) -{ - grn_cache_entry *ce0 = (grn_cache_entry *)cache; - MUTEX_LOCK(cache->mutex); - while (ce0 != ce0->prev && size--) { - grn_cache_expire_entry(cache, ce0->prev); - } - MUTEX_UNLOCK(cache->mutex); -} - -void -grn_cache_fin(void) -{ - grn_cache_current_set(&grn_gctx, NULL); - grn_cache_close(&grn_gctx, grn_cache_default); -} - /**** memory allocation ****/ #define ALIGN_SIZE (1<<3) diff --git a/lib/db.c b/lib/db.c index 0ddbbe66ac..eb7c618633 100644 --- a/lib/db.c +++ b/lib/db.c @@ -35,6 +35,7 @@ #include "grn_normalizer.h" #include "grn_report.h" #include "grn_util.h" +#include "grn_cache.h" #include typedef struct { diff --git a/lib/grn_cache.h b/lib/grn_cache.h new file mode 100644 index 0000000000..d2709d7917 --- /dev/null +++ b/lib/grn_cache.h @@ -0,0 +1,50 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2009-2016 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#pragma once + +#include "grn.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GRN_CACHE_MAX_KEY_SIZE GRN_HASH_MAX_KEY_SIZE_LARGE + +typedef struct { + uint32_t nentries; + uint32_t max_nentries; + uint32_t nfetches; + uint32_t nhits; +} grn_cache_statistics; + +void grn_cache_init(void); +grn_obj *grn_cache_fetch(grn_ctx *ctx, grn_cache *cache, + const char *str, uint32_t str_size); +void grn_cache_unref(grn_ctx *ctx, grn_cache *cache, + const char *str, uint32_t str_size); +void grn_cache_update(grn_ctx *ctx, grn_cache *cache, + const char *str, uint32_t str_size, grn_obj *value); +void grn_cache_expire(grn_cache *cache, int32_t size); +void grn_cache_fin(void); +void grn_cache_get_statistics(grn_ctx *ctx, grn_cache *cache, + grn_cache_statistics *statistics); + +#ifdef __cplusplus +} +#endif diff --git a/lib/grn_ctx.h b/lib/grn_ctx.h index e33a9c08c8..47d83bfef1 100644 --- a/lib/grn_ctx.h +++ b/lib/grn_ctx.h @@ -598,29 +598,6 @@ typedef struct { (db_obj)->obj.source_size = 0;\ } while (0) -/**** cache ****/ - -#define GRN_CACHE_MAX_KEY_SIZE GRN_HASH_MAX_KEY_SIZE_LARGE - -typedef struct { - uint32_t nentries; - uint32_t max_nentries; - uint32_t nfetches; - uint32_t nhits; -} grn_cache_statistics; - -void grn_cache_init(void); -grn_obj *grn_cache_fetch(grn_ctx *ctx, grn_cache *cache, - const char *str, uint32_t str_size); -void grn_cache_unref(grn_ctx *ctx, grn_cache *cache, - const char *str, uint32_t str_size); -void grn_cache_update(grn_ctx *ctx, grn_cache *cache, - const char *str, uint32_t str_size, grn_obj *value); -void grn_cache_expire(grn_cache *cache, int32_t size); -void grn_cache_fin(void); -void grn_cache_get_statistics(grn_ctx *ctx, grn_cache *cache, - grn_cache_statistics *statistics); - /**** receive handler ****/ GRN_API void grn_ctx_stream_out_func(grn_ctx *c, int flags, void *stream); diff --git a/lib/mrb/mrb_cache.c b/lib/mrb/mrb_cache.c index 76873f2ea6..3bf71fc04b 100644 --- a/lib/mrb/mrb_cache.c +++ b/lib/mrb/mrb_cache.c @@ -24,6 +24,7 @@ #include #include "../grn_db.h" +#include "../grn_cache.h" #include "mrb_bulk.h" #include "mrb_cache.h" diff --git a/lib/proc.c b/lib/proc.c index 2b16518efe..ba8ee3c6f4 100644 --- a/lib/proc.c +++ b/lib/proc.c @@ -25,6 +25,7 @@ #include "grn_pat.h" #include "grn_geo.h" #include "grn_expr.h" +#include "grn_cache.h" #include #include diff --git a/lib/proc/proc_select.c b/lib/proc/proc_select.c index 559e1e3040..94431f1f07 100644 --- a/lib/proc/proc_select.c +++ b/lib/proc/proc_select.c @@ -21,6 +21,7 @@ #include "../grn_str.h" #include "../grn_output.h" #include "../grn_util.h" +#include "../grn_cache.h" #include "../grn_ts.h" diff --git a/lib/sources.am b/lib/sources.am index 6ed5afcd29..a1d1266c95 100644 --- a/lib/sources.am +++ b/lib/sources.am @@ -1,4 +1,6 @@ libgroonga_la_SOURCES = \ + cache.c \ + grn_cache.h \ com.c \ grn_com.h \ command.c \