Navigation Menu

Skip to content

Commit

Permalink
Support token metadata
Browse files Browse the repository at this point in the history
New API:

  * grn_token_get_metadata()
  * grn_token_metadata_get_size()
  * grn_token_metadata_at()
  * grn_token_metadata_get()
  * grn_token_metadata_add()
  • Loading branch information
kou committed Sep 10, 2018
1 parent 5858594 commit 6baff61
Show file tree
Hide file tree
Showing 9 changed files with 328 additions and 1 deletion.
2 changes: 2 additions & 0 deletions include/groonga.h
@@ -1,5 +1,6 @@
/*
Copyright(C) 2014-2018 Brazil
Copyright(C) 2018 Kouhei Sutou <kou@clear-code.com>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
Expand Down Expand Up @@ -51,6 +52,7 @@
#include "groonga/thread.h"
#include "groonga/time.h"
#include "groonga/token.h"
#include "groonga/token_metadata.h"
#include "groonga/type.h"
#include "groonga/util.h"
#include "groonga/window_function.h"
Expand Down
3 changes: 2 additions & 1 deletion include/groonga/Makefile.am
Expand Up @@ -37,8 +37,9 @@ groonga_include_HEADERS = \
thread.h \
time.h \
token.h \
tokenizer.h \
token_filter.h \
token_metadata.h \
tokenizer.h \
tokenizer_query_deprecated.h \
type.h \
nfkc.h \
Expand Down
5 changes: 5 additions & 0 deletions include/groonga/token.h
@@ -1,6 +1,7 @@
/* -*- c-basic-offset: 2 -*- */
/*
Copyright(C) 2014-2018 Brazil
Copyright(C) 2018 Kouhei Sutou <kou@clear-code.com>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
Expand Down Expand Up @@ -161,6 +162,10 @@ grn_token_set_overlap(grn_ctx *ctx,
grn_token *token,
grn_bool have_overlap);

GRN_API grn_obj *
grn_token_get_metadata(grn_ctx *ctx,
grn_token *token);

#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
49 changes: 49 additions & 0 deletions include/groonga/token_metadata.h
@@ -0,0 +1,49 @@
/* -*- c-basic-offset: 2 -*- */
/*
Copyright(C) 2018 Kouhei Sutou <kou@clear-code.com>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 2.1 as published by the Free Software Foundation.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#pragma once

#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */

GRN_API size_t
grn_token_metadata_get_size(grn_ctx *ctx,
grn_obj *metadata);
GRN_API grn_rc
grn_token_metadata_at(grn_ctx *ctx,
grn_obj *metadata,
size_t i,
grn_obj *name,
grn_obj *value);
GRN_API grn_rc
grn_token_metadata_get(grn_ctx *ctx,
grn_obj *metadata,
const char *name,
int name_length,
grn_obj *value);
GRN_API grn_rc
grn_token_metadata_add(grn_ctx *ctx,
grn_obj *metadata,
const char *name,
int name_length,
grn_obj *value);

#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
2 changes: 2 additions & 0 deletions lib/c_sources.am
Expand Up @@ -107,6 +107,8 @@ libgroonga_c_sources = \
grn_token.h \
token_cursor.c \
grn_token_cursor.h \
token_metadata.c \
grn_token_metadata.h \
tokenizer.c \
grn_tokenizer.h \
tokenizers.c \
Expand Down
2 changes: 2 additions & 0 deletions lib/grn_token.h
@@ -1,6 +1,7 @@
/* -*- c-basic-offset: 2 -*- */
/*
Copyright(C) 2009-2018 Brazil
Copyright(C) 2018 Kouhei Sutou <kou@clear-code.com>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
Expand Down Expand Up @@ -31,6 +32,7 @@ struct _grn_token {
uint32_t source_length;
uint32_t source_first_character_length;
grn_bool have_overlap;
grn_obj metadata;
};

grn_rc grn_token_init(grn_ctx *ctx, grn_token *token);
Expand Down
34 changes: 34 additions & 0 deletions lib/grn_token_metadata.h
@@ -0,0 +1,34 @@
/* -*- c-basic-offset: 2 -*- */
/*
Copyright(C) 2018 Kouhei Sutou <kou@clear-code.com>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 2.1 as published by the Free Software Foundation.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#pragma once

#include "grn_ctx.h"

#ifdef __cplusplus
extern "C" {
#endif

void grn_token_metadata_init(grn_ctx *ctx, grn_obj *metadata);
void grn_token_metadata_fin(grn_ctx *ctx, grn_obj *metadata);
void grn_token_metadata_reset(grn_ctx *ctx, grn_obj *metadata);
void grn_token_metadata_copy(grn_ctx *ctx, grn_obj *metadata, grn_obj *source);

#ifdef __cplusplus
}
#endif
19 changes: 19 additions & 0 deletions lib/token.c
@@ -1,6 +1,7 @@
/* -*- c-basic-offset: 2 -*- */
/*
Copyright(C) 2012-2018 Brazil
Copyright(C) 2018 Kouhei Sutou <kou@clear-code.com>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
Expand All @@ -17,6 +18,7 @@
*/

#include "grn_token.h"
#include "grn_token_metadata.h"

grn_rc
grn_token_init(grn_ctx *ctx, grn_token *token)
Expand All @@ -28,13 +30,15 @@ grn_token_init(grn_ctx *ctx, grn_token *token)
token->source_length = 0;
token->source_first_character_length = 0;
token->have_overlap = GRN_FALSE;
grn_token_metadata_init(ctx, &(token->metadata));
GRN_API_RETURN(ctx->rc);
}

grn_rc
grn_token_fin(grn_ctx *ctx, grn_token *token)
{
GRN_API_ENTER;
grn_token_metadata_fin(ctx, &(token->metadata));
GRN_OBJ_FIN(ctx, &(token->data));
GRN_API_RETURN(ctx->rc);
}
Expand Down Expand Up @@ -229,6 +233,18 @@ grn_token_set_overlap(grn_ctx *ctx,
GRN_API_RETURN(ctx->rc);
}

grn_obj *
grn_token_get_metadata(grn_ctx *ctx, grn_token *token)
{
GRN_API_ENTER;
if (!token) {
ERR(GRN_INVALID_ARGUMENT,
"[token][data][get][metadata] token must not be NULL");
GRN_API_RETURN(NULL);
}
GRN_API_RETURN(&(token->metadata));
}

grn_rc
grn_token_reset(grn_ctx *ctx, grn_token *token)
{
Expand All @@ -243,6 +259,7 @@ grn_token_reset(grn_ctx *ctx, grn_token *token)
token->source_length = 0;
token->source_first_character_length = 0;
token->have_overlap = GRN_FALSE;
grn_token_metadata_reset(ctx, &(token->metadata));
exit:
GRN_API_RETURN(ctx->rc);
}
Expand All @@ -266,6 +283,8 @@ grn_token_copy(grn_ctx *ctx,
token->source_length = source->source_length;
token->source_first_character_length = source->source_first_character_length;
token->have_overlap = source->have_overlap;
grn_token_metadata_reset(ctx, &(token->metadata));
grn_token_metadata_copy(ctx, &(token->metadata), &(source->metadata));
exit:
GRN_API_RETURN(ctx->rc);
}

0 comments on commit 6baff61

Please sign in to comment.