From 98675340e429b752bdb8a4d7f7d6c1c5f60227d5 Mon Sep 17 00:00:00 2001 From: lazedo Date: Tue, 26 Jan 2016 19:40:51 +0000 Subject: [PATCH] db_text: add buffer size for reading text files the static value of 4096 may not be enough when dealing with presence tables that have xml content --- modules/db_text/README | 84 ++++++++++++++++----------- modules/db_text/dbt_file.c | 15 ++++- modules/db_text/dbt_lib.h | 1 + modules/db_text/dbtext.c | 4 ++ modules/db_text/doc/db_text_admin.xml | 21 +++++++ 5 files changed, 89 insertions(+), 36 deletions(-) diff --git a/modules/db_text/README b/modules/db_text/README index df3e46b21ca..68a8911f498 100644 --- a/modules/db_text/README +++ b/modules/db_text/README @@ -22,7 +22,7 @@ Olle E. Johansson - Copyright © 2003, 2004 FhG FOKUS + Copyright © 2003, 2004 FhG FOKUS __________________________________________________________________ Table of Contents @@ -44,6 +44,7 @@ Olle E. Johansson 3.1. db_mode (integer) 3.2. emptystring (integer) + 3.3. file_buffer_size (integer) 4. Exported RPC Functions @@ -62,11 +63,12 @@ Olle E. Johansson 1.3. Minimal Kamailio subscriber dbtext table example 1.4. Set db_mode parameter 1.5. Set emptystring parameter - 1.6. Load the dbtext module - 1.7. Definition of 'subscriber' table (one line) - 1.8. Definition of 'location' and 'aliases' tables (one line) - 1.9. Definition of 'version' table and sample records - 1.10. Configuration file + 1.6. Set file_buffer_size parameter + 1.7. Load the dbtext module + 1.8. Definition of 'subscriber' table (one line) + 1.9. Definition of 'location' and 'aliases' tables (one line) + 1.10. Definition of 'version' table and sample records + 1.11. Configuration file Chapter 1. Admin Guide @@ -87,6 +89,7 @@ Chapter 1. Admin Guide 3.1. db_mode (integer) 3.2. emptystring (integer) + 3.3. file_buffer_size (integer) 4. Exported RPC Functions @@ -129,12 +132,12 @@ Chapter 1. Admin Guide * A database is represented by a directory in the local file system. NOTE: when you use dbtext in Kamailio, the database URL for modules must be the path to the directory where the table-files are - located, prefixed by "text://", e.g., "text:///var/dbtext/ser". If - there is no "/" after "text://" then "CFG_DIR/" is inserted at the + located, prefixed by “text://”, e.g., “text:///var/dbtext/ser”. If + there is no “/” after “text://” then “CFG_DIR/” is inserted at the beginning of the database path. So, either you provide an absolute - path to database directory or a relative one to "CFG_DIR" + path to database directory or a relative one to “CFG_DIR” directory. - Do not forget that all databases in Kamailio needs the "version" + Do not forget that all databases in Kamailio needs the “version” table. * A table is represented by a text file inside database directory. @@ -145,7 +148,7 @@ Chapter 1. Admin Guide * the name of column must not include white spaces. * the format of a column definition is: name(type,attr). * between two column definitions must be a white space, e.g., - "first_name(str) last_name(str)". + “first_name(str) last_name(str)”. * the type of a column can be: + int - integer numbers. + double - real numbers with two decimals. @@ -157,12 +160,12 @@ Chapter 1. Admin Guide + null - accept null values in column fields. + if no attribute is set, the fields of the column cannot have null value. - * each other line is a row with data. The line ends with "\n". - * the fields are separated by ":". + * each other line is a row with data. The line ends with “\n”. + * the fields are separated by “:”. * no value between two ':' (or between ':' and start/end of a row) - means "null" value. If the parameter "emptystring" is enabled, + means “null” value. If the parameter "emptystring" is enabled, db_text sets a NULL string to an empty string value. - * next characters must be escaped in strings: "\n", "\r", "\t", ":". + * next characters must be escaped in strings: “\n”, “\r”, “\t”, “:”. * 0 -- the zero value must be escaped too. Example 1.1. Sample of a dbtext table @@ -211,6 +214,7 @@ suser:supasswd:xxx:alpha.org:xxx 3.1. db_mode (integer) 3.2. emptystring (integer) + 3.3. file_buffer_size (integer) 3.1. db_mode (integer) @@ -219,7 +223,7 @@ suser:supasswd:xxx:alpha.org:xxx table is requested whether the corresponding file on disk has changed, and if yes, will re-load the table from file. - Default value is "0". + Default value is “0”. Example 1.4. Set db_mode parameter ... @@ -233,18 +237,30 @@ modparam("db_text", "db_mode", 1) enable emptystring an empty string will not be NULL, but an empty string. - Default value is "0" (off). + Default value is “0” (off). Example 1.5. Set emptystring parameter ... modparam("db_text", "emptystring", 1) ... +3.3. file_buffer_size (integer) + + size of the buffer used to read the text file. Some presence tables + have columns with large content. + + Default value is “16384” (off). + + Example 1.6. Set file_buffer_size parameter +... +modparam("db_text", "file_buffer_size", 8192) +... + 4. Exported RPC Functions 4.1. db_text.dump -4.1. db_text.dump +4.1. db_text.dump Write back to hard drive all modified tables. @@ -263,12 +279,12 @@ modparam("db_text", "emptystring", 1) REMINDER: when you use text in Kamailio, the database URL for modules must be the path to the directory where the table-files are located, - prefixed by "text://", e.g., "text:///var/dbtext/ser". If there is no - "/" after "text://" then "CFG_DIR/" is inserted at the beginning of the + prefixed by “text://”, e.g., “text:///var/dbtext/ser”. If there is no + “/” after “text://” then “CFG_DIR/” is inserted at the beginning of the database path. So, either you provide an absolute path to database - directory or a relative one to "CFG_DIR" directory. + directory or a relative one to “CFG_DIR” directory. - Example 1.6. Load the dbtext module + Example 1.7. Load the dbtext module ... loadmodule "/path/to/kamailio/modules_k/db_text.so" ... @@ -280,29 +296,29 @@ modparam("module_name", "db_url", "text:///path/to/dbtext/database") Here are definitions for the most important tables as well as a basic configuration file to use db_text with Kamailio. The table structures may change in time and you will have to adjust these examples. Check - the source code directory "utils/kamctl/dbtxt/kamailio" for current + the source code directory “utils/kamctl/dbtxt/kamailio” for current definitions. You have to populate the table 'subscriber' by hand with user profiles in order to have authentication. To use with the given configuration file, the table files must be placed in the '/tmp/serdb' directory. - Example 1.7. Definition of 'subscriber' table (one line) + Example 1.8. Definition of 'subscriber' table (one line) ... -username(str) domain(str) password(str) first_name(str) last_name(str) phone(str -) email_address(str) datetime_created(int) datetime_modified(int) confirmation(s -tr) flag(str) sendnotification(str) greeting(str) ha1(str) ha1b(str) perms(str) -allow_find(str) timezone(str,null) rpid(str,null) +username(str) domain(str) password(str) first_name(str) last_name(str) phone(st +r) email_address(str) datetime_created(int) datetime_modified(int) confirmation +(str) flag(str) sendnotification(str) greeting(str) ha1(str) ha1b(str) perms(st +r) allow_find(str) timezone(str,null) rpid(str,null) ... - Example 1.8. Definition of 'location' and 'aliases' tables (one line) + Example 1.9. Definition of 'location' and 'aliases' tables (one line) ... -username(str) domain(str,null) contact(str,null) received(str) expires(int,null) - q(double,null) callid(str,null) cseq(int,null) last_modified(str) flags(int) us -er_agent(str) socket(str) +username(str) domain(str,null) contact(str,null) received(str) expires(int,null +) q(double,null) callid(str,null) cseq(int,null) last_modified(str) flags(int) +user_agent(str) socket(str) ... - Example 1.9. Definition of 'version' table and sample records + Example 1.10. Definition of 'version' table and sample records ... table_name(str) table_version(int) subscriber:3 @@ -310,7 +326,7 @@ location:6 aliases:6 ... - Example 1.10. Configuration file + Example 1.11. Configuration file ... # # $Id$ diff --git a/modules/db_text/dbt_file.c b/modules/db_text/dbt_file.c index 5d9ca849e84..eac2b736726 100644 --- a/modules/db_text/dbt_file.c +++ b/modules/db_text/dbt_file.c @@ -84,7 +84,8 @@ int dbt_check_mtime(const str *tbn, const str *dbn, time_t *mt) dbt_table_p dbt_load_file(const str *tbn, const str *dbn) { FILE *fin=NULL; - char path[512], buf[4096]; + char path[512]; + char *buf; int c, crow, ccol, bp, sign, max_auto; dbt_val_t dtval; dbt_table_p dtp = NULL; @@ -120,10 +121,16 @@ dbt_table_p dbt_load_file(const str *tbn, const str *dbn) if(!fin) return NULL; + buf = pkg_malloc(_db_text_read_buffer_size); + if(!buf) { + LM_ERR("error allocating read buffer, %i\n", _db_text_read_buffer_size); + goto done; + } + dtp = dbt_table_new(tbn, dbn, path); if(!dtp) goto done; - + state = DBT_FLINE_ST; crow = ccol = -1; colp = colp0 = NULL; @@ -473,6 +480,8 @@ dbt_table_p dbt_load_file(const str *tbn, const str *dbn) done: if(fin) fclose(fin); + if(buf) + pkg_free(buf); return dtp; clean: // ????? FILL IT IN - incomplete row/column @@ -480,6 +489,8 @@ dbt_table_p dbt_load_file(const str *tbn, const str *dbn) LM_DBG("error at row=%d col=%d c=%c\n", crow+1, ccol+1, c); if(dtp) dbt_table_free(dtp); + if(buf) + pkg_free(buf); return NULL; } diff --git a/modules/db_text/dbt_lib.h b/modules/db_text/dbt_lib.h index 2505a58afb6..7578a9573bf 100644 --- a/modules/db_text/dbt_lib.h +++ b/modules/db_text/dbt_lib.h @@ -49,6 +49,7 @@ * */ extern int db_mode; /* Database usage mode: 0 = no cache, 1 = cache */ extern int empty_string; /* If TRUE, an empty string is an empty string, otherwise NULL */ +extern int _db_text_read_buffer_size; /* size of the buffer to allocate when reading file */ typedef db_val_t dbt_val_t, *dbt_val_p; diff --git a/modules/db_text/dbtext.c b/modules/db_text/dbtext.c index f601398172b..5f65e6e53ea 100644 --- a/modules/db_text/dbtext.c +++ b/modules/db_text/dbtext.c @@ -37,11 +37,14 @@ MODULE_VERSION static int mod_init(void); static void destroy(void); +#define DEFAULT_DB_TEXT_READ_BUFFER_SIZE 16384 + /* * Module parameter variables */ int db_mode = 0; /* Database usage mode: 0 = cache, 1 = no cache */ int empty_string = 0; /* Treat empty string as "" = 0, 1 = NULL */ +int _db_text_read_buffer_size = DEFAULT_DB_TEXT_READ_BUFFER_SIZE; int dbt_bind_api(db_func_t *dbb); @@ -60,6 +63,7 @@ static cmd_export_t cmds[] = { static param_export_t params[] = { {"db_mode", INT_PARAM, &db_mode}, {"emptystring", INT_PARAM, &empty_string}, + {"file_buffer_size", INT_PARAM, &_db_text_read_buffer_size}, {0, 0, 0} }; diff --git a/modules/db_text/doc/db_text_admin.xml b/modules/db_text/doc/db_text_admin.xml index 6d6078aa1d6..344713dd2cc 100644 --- a/modules/db_text/doc/db_text_admin.xml +++ b/modules/db_text/doc/db_text_admin.xml @@ -290,6 +290,27 @@ modparam("db_text", "db_mode", 1) ... modparam("db_text", "emptystring", 1) ... + + + +
+ <varname>file_buffer_size</varname> (integer) + + size of the buffer used to read the text file. + Some presence tables have columns with large content. + + + + + Default value is 16384 (off). + + + + Set <varname>file_buffer_size</varname> parameter + +... +modparam("db_text", "file_buffer_size", 8192) +...