ext/senatus.lua ext/usherette.lua @@ -1,3 +1,13 @@ +2009-02-05 Mikio Hirabayashi <mikio@users.sourceforge.net> + + * ttutil.c (ttservstart, ttservaddtimedhandler): multiple tasks are now supported. + + * ttutil.c (ttsockrecv, ttsockgetint32, ttsockgetint64): performance was improved. + + * ttserver.c (do_extpc): new function. + + - Release: 1.1.14 + 2009-02-04 Mikio Hirabayashi <mikio@users.sourceforge.net> * tcrdb.c (tcrdbqrysearchout): a bug related to the protocol format was fixed. ChangeLog @@ -19,6 +19,7 @@ Contents of the directory tree is below. ./ - sources of Tokyo Tyrant ./doc/ - manuals and specifications ./man/ - manuals for nroff + ./ext/ - sample script for the script extension ./example/ - sample code of tutorial ./lab/ - for test and experiment README @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.61 for tokyotyrant 1.1.13. +# Generated by GNU Autoconf 2.61 for tokyotyrant 1.1.14. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. @@ -572,8 +572,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='tokyotyrant' PACKAGE_TARNAME='tokyotyrant' -PACKAGE_VERSION='1.1.13' -PACKAGE_STRING='tokyotyrant 1.1.13' +PACKAGE_VERSION='1.1.14' +PACKAGE_STRING='tokyotyrant 1.1.14' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1194,7 +1194,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures tokyotyrant 1.1.13 to adapt to many kinds of systems. +\`configure' configures tokyotyrant 1.1.14 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1255,7 +1255,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of tokyotyrant 1.1.13:";; + short | recursive ) echo "Configuration of tokyotyrant 1.1.14:";; esac cat <<\_ACEOF @@ -1349,7 +1349,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -tokyotyrant configure 1.1.13 +tokyotyrant configure 1.1.14 generated by GNU Autoconf 2.61 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1363,7 +1363,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by tokyotyrant $as_me 1.1.13, which was +It was created by tokyotyrant $as_me 1.1.14, which was generated by GNU Autoconf 2.61. Invocation command line was $ $0 $@ @@ -1719,7 +1719,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu # Package information MYLIBVER=2 -MYLIBREV=4 +MYLIBREV=5 MYPROTVER="0.9" # Targets @@ -1731,7 +1731,7 @@ MYINITFILES="ttservctl" MYMAN1FILES="ttserver.1 ttulmgr.1 ttultest.1 tcrtest.1 tcrmttest.1 tcrmgr.1" MYMAN3FILES="ttutil.3 tculog.3 tcrdb.3" MYMAN8FILES="ttservctl.8" -MYSCREXTFILES="senatus.lua usherette.lua" +MYSCREXTFILES="ext" MYDOCUMENTFILES="COPYING ChangeLog THANKS doc" MYPCFILES="tokyotyrant.pc" @@ -5906,7 +5906,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by tokyotyrant $as_me 1.1.13, which was +This file was extended by tokyotyrant $as_me 1.1.14, which was generated by GNU Autoconf 2.61. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5949,7 +5949,7 @@ Report bugs to <bug-autoconf@gnu.org>." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF ac_cs_version="\\ -tokyotyrant config.status 1.1.13 +tokyotyrant config.status 1.1.14 configured by $0, generated by GNU Autoconf 2.61, with options \\"`echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" configure @@ -7,11 +7,11 @@ #================================================================ # Package name -AC_INIT(tokyotyrant, 1.1.13) +AC_INIT(tokyotyrant, 1.1.14) # Package information MYLIBVER=2 -MYLIBREV=4 +MYLIBREV=5 MYPROTVER="0.9" # Targets @@ -23,7 +23,7 @@ MYINITFILES="ttservctl" MYMAN1FILES="ttserver.1 ttulmgr.1 ttultest.1 tcrtest.1 tcrmttest.1 tcrmgr.1" MYMAN3FILES="ttutil.3 tculog.3 tcrdb.3" MYMAN8FILES="ttservctl.8" -MYSCREXTFILES="senatus.lua usherette.lua" +MYSCREXTFILES="ext" MYDOCUMENTFILES="COPYING ChangeLog THANKS doc" MYPCFILES="tokyotyrant.pc" configure.in @@ -189,9 +189,12 @@ var { <h2 id="introduction">Introduction</h2> -<p>Tokyo Tyrant is a package of network interface to the DBM called Tokyo Cabinet. Though the DBM has high performance, you might bother in case that multiple processes share the same database, or remote processes access the database. Thus, Tokyo Tyrant is provided for concurrent and remote connections to Tokyo Cabinet. It is composed of the server process managing a database and its access library for client applications. The server can embed Lua, a lightweight script language so that you can define arbitrary operations of the database.</p> +<p>Tokyo Tyrant is a package of network interface to the DBM called Tokyo Cabinet. Though the DBM has high performance, you might bother in case that multiple processes share the same database, or remote processes access the database. Thus, Tokyo Tyrant is provided for concurrent and remote connections to Tokyo Cabinet. It is composed of the server process managing a database and its access library for client applications.</p> + +<p>The server features high concurrency due to thread-pool modeled implementation and the epoll/kqueue mechanism of the modern Linux/*BSD kernel. The server and its clients communicate with each other by simple binary protocol on TCP/IP. Protocols compatible with memcached and HTTP/1.1 are also supported so that almost all principal platforms and programming languages can use Tokyo Tyrant. High availability and high integrity are also featured due to such mechanisms as hot backup, update logging, and replication. The server can embed Lua, a lightweight script language so that you can define arbitrary operations of the database.</p> + +<p>Because the server uses the abstract API of Tokyo Cabinet, all of the six APIs: the on-memory hash database API, the on-memory tree database API, the hash API, the B+ tree database API, the fixed-length database API, and the table database API, are available from the client with the common interface. Moreover, the table extension is provided to use specifidc features of the table database.</p> -<p>The server features high concurrency due to thread-pool modeled implementation and the epoll/kqueue mechanism of the modern Linux/*BSD kernel. The server and its clients communicate with each other by simple binary protocol on TCP/IP. Protocols compatible with memcached and HTTP/1.1 are also supported so that almost all principal platforms and programming languages can use Tokyo Tyrant. High availability and high integrity are also featured due to such mechanisms as hot backup, update logging, and replication.</p> <p><em>As for now, the server works on Linux, FreeBSD, Mac OS X only.</em></p> @@ -254,12 +257,12 @@ var { <h2 id="serverprog">Server Programs</h2> -<h3>ttserver</h3> +<h3 id="serverprog_ttserver">ttserver</h3> <p>The command `<code>ttserver</code>' runs the server managing a database instance. Because the database is treated by the abstract API of Tokyo Cabinet, you can choose the scheme on start-up of the server. Supported schema are on-memory hash database, on-memory tree database, hash database, and B+ tree database. This command is used in the following format. `<var>dbname</var>' specifies the database name. If it is omitted, on-memory hash database is specified.</p> <dl class="api"> -<dt><code>ttserver [-host <var>name</var>] [-port <var>num</var>] [-th<var>num</var> <var>num</var>] [-tout <var>num</var>] [-dmn] [-pid <var>path</var>] [-log <var>path</var>] [-ld|-le] [-ulog <var>path</var>] [-ulim <var>num</var>] [-uas] [-sid <var>num</var>] [-mhost <var>name</var>] [-mport <var>num</var>] [-rts <var>path</var>] [-ext <var>path</var>] [-mask <var>expr</var>] [<var>dbname</var>]</code></dt> +<dt><code>ttserver [-host <var>name</var>] [-port <var>num</var>] [-th<var>num</var> <var>num</var>] [-tout <var>num</var>] [-dmn] [-pid <var>path</var>] [-log <var>path</var>] [-ld|-le] [-ulog <var>path</var>] [-ulim <var>num</var>] [-uas] [-sid <var>num</var>] [-mhost <var>name</var>] [-mport <var>num</var>] [-rts <var>path</var>] [-ext <var>path</var>] [-extpc <var>name</var> <var>period</var>] [-mask <var>expr</var>] [<var>dbname</var>]</code></dt> </dl> <p>Options feature the following.</p> @@ -282,6 +285,7 @@ var { <li><code>-mport <var>num</var></code> : specify the port number of the replication master server.</li> <li><code>-rts <var>path</var></code> : specify the replication time stamp file.</li> <li><code>-ext <var>path</var></code> : specify the script language extension file.</li> +<li><code>-extpc <var>name</var> <var>period</var></code> : specify the function name and the calling period of a periodic command.</li> <li><code>-mask <var>expr</var></code> : specify the names of forbidden commands.</li> <li><code>-unmask <var>expr</var></code> : specify the names of allowed commands.</li> </ul> @@ -292,7 +296,7 @@ var { <p>The command mask expression is a list of command names separated by ",". For example, "out,vanish,copy" means a set of "out", "vanish", and "copy". Commands of the memcached compatible protocol and the HTTP compatible protocol are also forbidden or allowed, related by the mask of each original command. Moreover, there are meta expressions. "all" means all commands. "allorg" means all commands of the original binary protocol. "allmc" means all commands of the memcached compatible protocol. "allhttp" means all commands of the HTTP compatible protocol. "allread" is the abbreviation of `get', `mget', `vsiz', `iterinit', `iternext', `fwmkeys', `rnum', `size', and `stat'. "allwrite" is the abbreviation of `put', `putkeep', `putcat', `putshl', `putnr', `out', `addint', `adddouble', `vanish', and `misc'. "allmanage" is the abbreviation of `sync', `copy', `restore', and `setmst'. "repl" means replication as master. "slave" means replication as slave.</p> -<h3>ttservctl</h3> +<h3 id="serverprog_ttservctl">ttservctl</h3> <p>The command `<code>ttservctl</code>' is the startup script of the server. It can be called by the RC script of the bootstrap process of the operating system. This command is used in the following format.</p> @@ -309,7 +313,7 @@ var { <p>The database is placed as "/var/ttserver/casket.tch". The log and related files are also placed in "/var/ttserver". This command returns 0 on success, another on failure.</p> -<h3>ttulmgr</h3> +<h3 id="serverprog_ttulmgr">ttulmgr</h3> <p>The command `<code>ttulmgr</code>' is the utility to export and import the update log. It is useful to filter the update log with such text utilities as `<code>grep</code>' and `<code>sed</code>'. This command is used in the following format. `<var>upath</var>' specifies the update log directory.</p> @@ -333,7 +337,7 @@ var { <h2 id="clientprog">Client Programs</h2> -<h3>tcrtest</h3> +<h3 id="clientprog_tcrtest">tcrtest</h3> <p>The command `<code>tcrtest</code>' is a utility for facility test and performance test. This command is used in the following format. `<var>host</var>' specifies the host name of the server. `<var>rnum</var>' specifies the number of iterations.</p> @@ -372,7 +376,7 @@ var { <p>If the port number is not more than 0, UNIX domain socket is used and the path of the socket file is specified by the host parameter. This command returns 0 on success, another on failure.</p> -<h3>tcrmttest</h3> +<h3 id="clientprog_tcrmttest">tcrmttest</h3> <p>The command `<code>tcrmttest</code>' is a utility for facility test under multi-thread situation. This command is used in the following format. `<var>host</var>' specifies the host name of the server. `<var>rnum</var>' specifies the number of iterations.</p> @@ -399,7 +403,7 @@ var { <p>If the port number is not more than 0, UNIX domain socket is used and the path of the socket file is specified by the host parameter. This command returns 0 on success, another on failure.</p> -<h3>tcrmgr</h3> +<h3 id="clientprog_tcrmgr">tcrmgr</h3> <p>The command `<code>tcrmgr</code>' is a utility for test and debugging of the remote database API and its applications. `<var>host</var>' specifies the host name of the server. `<var>key</var>' specifies the key of a record. `<var>value</var>' specifies the value of a record. `<var>func</var> specifies the name of the function. `<var>arg</var>' specifies the arguments of the function. `<var>dpath</var>' specifies the destination file. `<var>file</var>' specifies the input file. `<var>upath</var>' specifies the update log directory. `<var>mhost</var>' specifies the host name of the replication master. `<var>url</var>' specifies the target URL.</p> @@ -475,7 +479,7 @@ var { <p>Remote database is a set of interfaces to use an abstract database of Tokyo Cabinet, mediated by a server of Tokyo Tyrant. See `<code>tcrdb.h</code>' for entire specification.</p> -<h3>Description</h3> +<h3 id="clientprog_description">Description</h3> <p>To use the remote database API, include `<code>tcrdb.h</code>' and related standard header files. Usually, write the following description near the front of a source file.</p> @@ -490,7 +494,7 @@ var { <p>Before operations to store or retrieve records, it is necessary to connect the remote database object to the server. The function `<code>tcrdbopen</code>' is used to open a database connection and the function `<code>tcrdbclose</code>' is used to close the connection.</p> -<h3>API</h3> +<h3 id="tcrdbapi_api">API</h3> <p>The function `tcrdberrmsg' is used in order to get the message string corresponding to an error code.</p> @@ -919,7 +923,7 @@ var { <dd>Because the object of the return value is created with the function `tclistnew', it should be deleted with the function `tclistdel' when it is no longer in use.</dd> </dl> -<h3>API of the Table Extension</h3> +<h3 id="tcrdbapi_apitbl">API of the Table Extension</h3> <p>The function `tcrdbtblput' is used in order to store a record into a remote database object.</p> @@ -1075,7 +1079,7 @@ var { <dd>Because the object of the return value is created with the function `tcmapnew', it should be deleted with the function `tcmapdel' when it is no longer in use.</dd> </dl> -<h3>Example Code</h3> +<h3 id="tcrdbapi_example">Example Code</h3> <p>The following code is an example to use a remote database.</p> @@ -1130,7 +1134,90 @@ int main(int argc, char **argv){ } </pre> -<h3>How to Use the Library</h3> +<p>The following code is an example to use a remote database with the table extension.</p> + +<pre>#include &lt;tcrdb.h&gt; +#include &lt;stdlib.h&gt; +#include &lt;stdbool.h&gt; +#include &lt;stdint.h&gt; + +int main(int argc, char **argv){ + + TCRDB *rdb; + int ecode, pksiz, i, rsiz; + char pkbuf[256]; + const char *rbuf, *name; + TCMAP *cols; + RDBQRY *qry; + TCLIST *res; + + /* create the object */ + rdb = tcrdbnew(); + + /* connect to the server */ + if(!tcrdbopen(rdb, "localhost", 1978)){ + ecode = tcrdbecode(rdb); + fprintf(stderr, "open error: %s\n", tcrdberrmsg(ecode)); + } + + /* store a record */ + pksiz = sprintf(pkbuf, "%ld", (long)tcrdbtblgenuid(rdb)); + cols = tcmapnew3("name", "mikio", "age", "30", "lang", "ja,en,c", NULL); + if(!tcrdbtblput(rdb, pkbuf, pksiz, cols)){ + ecode = tcrdbecode(rdb); + fprintf(stderr, "put error: %s\n", tcrdberrmsg(ecode)); + } + tcmapdel(cols); + + /* store a record in a naive way */ + pksiz = sprintf(pkbuf, "12345"); + cols = tcmapnew(); + tcmapput2(cols, "name", "falcon"); + tcmapput2(cols, "age", "31"); + tcmapput2(cols, "lang", "ja"); + if(!tcrdbtblput(rdb, pkbuf, pksiz, cols)){ + ecode = tcrdbecode(rdb); + fprintf(stderr, "put error: %s\n", tcrdberrmsg(ecode)); + } + tcmapdel(cols); + + /* search for records */ + qry = tcrdbqrynew(rdb); + tcrdbqryaddcond(qry, "age", RDBQCNUMGE, "20"); + tcrdbqryaddcond(qry, "lang", RDBQCSTROR, "ja,en"); + tcrdbqrysetorder(qry, "name", RDBQOSTRASC); + tcrdbqrysetmax(qry, 10); + res = tcrdbqrysearch(qry); + for(i = 0; i &lt; tclistnum(res); i++){ + rbuf = tclistval(res, i, &amp;rsiz); + cols = tcrdbtblget(rdb, rbuf, rsiz); + if(cols){ + printf("%s", rbuf); + tcmapiterinit(cols); + while((name = tcmapiternext2(cols)) != NULL){ + printf("\t%s\t%s", name, tcmapget2(cols, name)); + } + printf("\n"); + tcmapdel(cols); + } + } + tclistdel(res); + tcrdbqrydel(qry); + + /* close the connection */ + if(!tcrdbclose(rdb)){ + ecode = tcrdbecode(rdb); + fprintf(stderr, "close error: %s\n", tcrdberrmsg(ecode)); + } + + /* delete the object */ + tcrdbdel(rdb); + + return 0; +} +</pre> + +<h3 id="tcrdbapi_uselib">How to Use the Library</h3> <p>The API of C is available by programs conforming to the C89 (ANSI C) standard or the C99 standard. As the header files of Tokyo Tyrant are provided as `<code>tcrdb.h</code>', applications should include it to use the API. As the library is provided as `<code>libtokyotyrant.a</code>' and `<code>libtokyotyrant.so</code>' and they depends `<code>libtokyocabinet.so</code>', `<code>libz.so</code>', `<code>libbz2.so</code>', `<code>libresolv.so</code>', `<code>libnsl.so</code>', `<code>libdl.so</code>', `<code>librt.so</code>', `<code>libpthread.so</code>', `<code>libm.so</code>', and `<code>libc.so</code>', linker options `<code>-ltokyotyrant</code>', `<code>-ltokyocabinet</code>', `<code>-lz</code>', `<code>-lbz2</code>', `<code>-lresolv</code>', `<code>-lnsl</code>', `<code>-ldl</code>', `<code>-lrt</code>', `<code>-lpthread</code>', `<code>-lm</code>', and `<code>-lc</code>' are required for build command. A typical build command is the following.</p> @@ -1147,7 +1234,7 @@ int main(int argc, char **argv){ <p>The database server can starts reading a Lua script file by the `<code>-ext</code>' option. Clients can call functions defined in the script file by the `<code>tcrdbext</code>' function of the remote database API.</p> -<h3>User-defined Functions</h3> +<h3 id="luaext_userfunc">User-defined Functions</h3> <p>You can define some arbitrary functions in the script file. Each function receives two string parameters of the key and the value. The return value is sent back to the client. If the function returns `nil', the server send the error code to the client.</p> @@ -1155,7 +1242,7 @@ int main(int argc, char **argv){ <p>Note that instances of Lua interpreter are handled separately by each native thread. Because global variables of Lua are not useful to share some data among native threads or sessions, shared data should be handled in the database or by the stash functions.</p> -<h3>Built-in Functions</h3> +<h3 id="luaext_builtinfunc">Built-in Functions</h3> <p>The following build-in functions for database operations are available in user defined functions. The type of `key' and `value' parameters should be string or number. If number is given, it is converted as decimal string.</p> @@ -1318,7 +1405,7 @@ int main(int argc, char **argv){ <p>The global variable `<code>_version</code>' contains the version information of the server. The global variable `<code>_pid</code>' contains the process ID. The global variable `<code>_sid</code>' contains the server ID. The global variable `<code>_thnum</code>' contains the number of native threads. The global variable `<code>_thid</code>' contains the ID number of each native thread.</p> -<h3>Example Code</h3> +<h3 id="luaext_example">Example Code</h3> <p>The following code is an example to increment the record value and store as a decimal number string. This function should be called with the record locking option to ensure atomicity.</p> @@ -1678,7 +1765,7 @@ end <h2 id="tutorial">Tutorial</h2> -<h3>Basic Use</h3> +<h3 id="tutorial_basic">Basic Use</h3> <p>After installation of Tokyo Tyrant, you can start the server immediately by executing the command `<code>ttserver</code>' in the terminal. By default, the server listens to the port 1978 and serves as the accessor of an on-memory hash database, which is useful to store cache data.</p> @@ -1733,7 +1820,7 @@ end <pre>[terminal-1]$ rm casket.tch </pre> -<h3>Daemon</h3> +<h3 id="tutorial_daemon">Daemon</h3> <p>To run the server as a daemon process, specify the option `<code>-dmn</code>'. Moreover, the option `<code>-pid</code>' should be specified to record the process ID into a file. Note that the current directory of the daemon process is changed to the root directory. So, the file path parameter should be expressed as the absolute path.</p> @@ -1752,7 +1839,7 @@ end <p>By default, the database file and the related files are placed under `<code>/var/ttserver</code>'. Because `<code>ttservctl</code>' is a tiny shell script, copy and edit it for your purpose. Also, it is suitable to install the modified script into `<code>/etc/init.d</code>' and set symbolic links from `<code>/etc/rc3.d/S98ttserver</code>' and `<code>/etc/rc5.d/S98ttserver</code>'.</p> -<h3>Backup and Recovery</h3> +<h3 id="tutorial_backup">Backup and Recovery</h3> <p>Let's run the server again to continue this tutorial.</p> @@ -1792,7 +1879,7 @@ end <pre>[terminal-1]$ rm casket.tch backup.tch </pre> -<h3>Update Log</h3> +<h3 id="tutorial_ulog">Update Log</h3> <p>Let's run the server with update logging enabled. The option `<code>-ulog</code>' specifies the directory to contain the update log files.</p> @@ -1834,7 +1921,7 @@ end <pre>[terminal-1]$ rm -rf casket.tch ulog ulog-back </pre> -<h3>Replication</h3> +<h3 id="tutorial_replication">Replication</h3> <p>Replication is a mechanism to synchronize two or more database servers for high availability and high integrity. The replication source server is called "master" and each destination server is called "slave". Replication requires the following preconditions.</p> @@ -1905,7 +1992,7 @@ end <p>Tokyo Tyrant supports "dual master" replication which realizes higher availability. To do it, run two servers which replicate each other.</p> -<h3>Setting Replication on Demand</h3> +<h3 id="tutorial_repondemand">Setting Replication on Demand</h3> <p>You can set replication of the running database service without any downtime. First, prepare the following script for backup operation and save it as "ttbackup.sh" with executable permission (0755).</p> @@ -1975,7 +2062,7 @@ cp -f "$srcpath" "$destpath" [terminal-2]$ rm -rf casket-2.tch ulog-2 2.rts </pre> -<h3>Tuning</h3> +<h3 id="tutorial_tuning">Tuning</h3> <p>If you use a hash database, set the tuning parameter "#bnum=<var>xxx</var>" to improve performance. It specifies the bucket number and should be more than the number of record to be stored.</p> @@ -1983,7 +2070,7 @@ cp -f "$srcpath" "$destpath" <p>If huge number of clients access the server, make sure the limit number of file descriptors per process is cleared. By default on most systems, it is set as 1024. If so, use `<code>ulimit</code>' to clear it.</p> -<h3>Lua Extension</h3> +<h3 id="tutorial_luaext">Lua Extension</h3> <p>If you want more complex database operations than existing ones, use the Lua extension. For example, prepare the following script and save it as "test.lua". There is a function "fibonacci" which returns the Fibonacci number of a number of the key.</p> @@ -2037,9 +2124,9 @@ end [terminal-2]$ tcrmgr ext localhost fibnext </pre> -<p>As you see, the called function receives two string parameters of the key and the value. The return value is sent back to the client. You can use such built-in functions for database operations as "_put", "_out", "_get", and so on. There is a sample file `<code>/usr/local/share/tokyotyrant/senatus.lua</code>'.</p> +<p>As you see, the called function receives two string parameters of the key and the value. The return value is sent back to the client. You can use such built-in functions for database operations as "_put", "_out", "_get", and so on. There is a sample file `<code>ext/senatus.lua</code>'.</p> -<h3>Using memcached Client</h3> +<h3 id="tutorial_memcached">Using memcached Client</h3> <p>This section describes how to use a memcached client library of Perl (Cache::Memcached) with Tokyo Tyrant. Run the server of Tokyo Tyrant as usual. And, the following script is a typical example.</p> @@ -2063,7 +2150,7 @@ printf("three: %s\n", $val-&gt;{three}); $memd-&gt;delete('one'); </pre> -<h3>Using HTTP Client</h3> +<h3 id="tutorial_http">Using HTTP Client</h3> <p>This section describes how to use an HTTP client library of Perl (LWP::UserAgent) with Tokyo Tyrant. Run the server of Tokyo Tyrant as usual. And, the following script is a typical example.</p> @@ -2097,6 +2184,49 @@ if($res-&gt;is_success()){ } </pre> +<h3 id="tutorial_expcache">Persistent but Expirable Cache</h3> + +<p>If you want to cache data like session information for your Web application but want to avoid data loss because of the server crash, using Tokyo Tyrant can be the solution, that is to say, "persistent" expirable cache. It requires the following preconditions.</p> + +<ul> +<li>The server must open a table database.</li> +<li>Clients should store each record with an expiration date column.</li> +<li>The database should have an index for the expiration date column.</li> +<li>The server must periodically call the user defined function provided through the Lua extension.</li> +</ul> + +<p>First, prepare the following script for expiration and save it as "ttexpire.lua". It will expire records where the value of the "x" column exceeds the current date.</p> + +<pre>function expire() + local args = {}; + local cdate = string.format("%d", _time()) + table.insert(args, "addcond\0x\0NUMLE\0" .. cdate) + table.insert(args, "out") + local res = _misc("search", args) + if not res then + _log("expiration was failed") + end +end +</pre> + +<p>Start the server by opening a table database which has the index for the "x" column, and by scheduling it to call the expiration function per second.</p> + +<pre>[terminal-1]$ ttserver -ext ttexpire.lua -extpc expire 1.0 "casket.tct#idx=x:dec" +</pre> + +<p>Store test records from another terminal.</p> + +<pre>[terminal-2]$ now=`date +%s` +for((i=1;i&lt;=60;i++)); do + tcrmgr put -sep '|' localhost "$i" "x|$((now+i))" +done +</pre> + +<p>You can confirm that the records are being removed by expiration.</p> + +<pre>[terminal-2]$ tcrmgr list -pv -sep '|' localhost +</pre> + <hr /> <h2 id="license">License</h2> doc/index.html @@ -5,11 +5,11 @@ ttserver \- the server of Tokyo Tyrant .SH DESCRIPTION .PP -The command `\fBttserver\fR' runs the server managing a database instance. Because the database is treated by the abstract API of Tokyo Cabinet, you can choose the scheme on start\-up of the server. Supported schema are on\-memory database, hash database, and B+ tree database. This command is used in the following format. `\fIdbname\fR' specifies the database name. If it is omitted, on\-memory database is specified. +The command `\fBttserver\fR' runs the server managing a database instance. Because the database is treated by the abstract API of Tokyo Cabinet, you can choose the scheme on start\-up of the server. Supported schema are on\-memory hash database, on\-memory tree database, hash database, and B+ tree database. This command is used in the following format. `\fIdbname\fR' specifies the database name. If it is omitted, on\-memory hash database is specified. .PP .RS .br -\fBttserver \fR[\fB\-host \fIname\fB\fR]\fB \fR[\fB\-port \fInum\fB\fR]\fB \fR[\fB\-th\fInum\fB \fInum\fB\fR]\fB \fR[\fB\-tout \fInum\fB\fR]\fB \fR[\fB\-dmn\fR]\fB \fR[\fB\-pid \fIpath\fB\fR]\fB \fR[\fB\-log \fIpath\fB\fR]\fB \fR[\fB\-ld\fR|\fB\-le\fR]\fB \fR[\fB\-ulog \fIpath\fB\fR]\fB \fR[\fB\-ulim \fInum\fB\fR]\fB \fR[\fB\-uas\fR]\fB \fR[\fB\-sid \fInum\fB\fR]\fB \fR[\fB\-mhost \fIname\fB\fR]\fB \fR[\fB\-mport \fInum\fB\fR]\fB \fR[\fB\-rts \fIpath\fB\fR]\fB \fR[\fB\-ext \fIpath\fB\fR]\fB \fR[\fB\fIdbname\fB\fR]\fB\fR +\fBttserver \fR[\fB\-host \fIname\fB\fR]\fB \fR[\fB\-port \fInum\fB\fR]\fB \fR[\fB\-th\fInum\fB \fInum\fB\fR]\fB \fR[\fB\-tout \fInum\fB\fR]\fB \fR[\fB\-dmn\fR]\fB \fR[\fB\-pid \fIpath\fB\fR]\fB \fR[\fB\-log \fIpath\fB\fR]\fB \fR[\fB\-ld\fR|\fB\-le\fR]\fB \fR[\fB\-ulog \fIpath\fB\fR]\fB \fR[\fB\-ulim \fInum\fB\fR]\fB \fR[\fB\-uas\fR]\fB \fR[\fB\-sid \fInum\fB\fR]\fB \fR[\fB\-mhost \fIname\fB\fR]\fB \fR[\fB\-mport \fInum\fB\fR]\fB \fR[\fB\-rts \fIpath\fB\fR]\fB \fR[\fB\-ext \fIpath\fB\fR]\fB \fR[\fB\-extpc \fIname\fB \fIperiod\fB\fR]\fB \fR[\fB\-mask \fIexpr\fB\fR]\fB \fR[\fB\fIdbname\fB\fR]\fB\fR .RE .PP Options feature the following. @@ -49,6 +49,12 @@ Options feature the following. .br \fB\-ext \fIpath\fR\fR : specify the script language extension file. .br +\fB\-extpc \fIname\fR \fIperiod\fR\fR : specify the function name and the calling period of a periodic command. +.br +\fB\-mask \fIexpr\fR\fR : specify the names of forbidden commands. +.br +\fB\-unmask \fIexpr\fR\fR : specify the names of allowed commands. +.br .RE .PP To terminate the server normally, send SIGINT or SIGTERM to the process. It is okay to press Ctrl\-C on the controlling terminal. To restart the server, send SIGHUP to the process. If the port number is not more than 0, UNIX domain socket is used and the path of the socket file is specified by the host parameter. This command returns 0 on success, another on failure. man/ttserver.1 @@ -1317,7 +1317,7 @@ static int proctable(const char *host, int port, int cnum, int rnum){ int type = types[myrand(sizeof(types) / sizeof(*types))]; tcrdbqrysetorder(qry, name, type); } - if(myrand(3) != 0) tcrdbqrysetmax(qry, myrand(i)); + tcrdbqrysetmax(qry, myrand(10)); TCLIST *res = tcrdbqrysearch(qry); tclistdel(res); tcrdbqrydel(qry); tcrtest.c @@ -79,6 +79,15 @@ typedef struct { // type of structure of master synchron bool recon; } REPLARG; +typedef struct { // type of structure of periodic command + const char *name; + TCADB *adb; + TCULOG *ulog; + uint32_t sid; + REPLARG *sarg; + void *scrext; +} EXTPCARG; + typedef struct { // type of structure of task opaque object uint64_t mask; TCADB *adb; @@ -108,9 +117,10 @@ static int proc(const char *dbname, const char *host, int port, int thnum, int t bool dmn, const char *pidpath, const char *logpath, const char *ulogpath, uint64_t ulim, bool uas, uint32_t sid, const char *mhost, int mport, const char *rtspath, const char *extpath, - uint64_t mask); + const TCLIST *extpcs, uint64_t mask); static void do_log(int level, const char *msg, void *opq); static void do_slave(void *opq); +static void do_extpc(void *opq); static void do_task(TTSOCK *sock, void *opq, TTREQ *req); static int tokenize(char *str, char **tokens, int max); static uint32_t recmtxidx(const char *kbuf, int ksiz); @@ -169,6 +179,7 @@ int main(int argc, char **argv){ char *mhost = NULL; char *rtspath = NULL; char *extpath = NULL; + TCLIST *extpcs = NULL; int port = DEFPORT; int thnum = DEFTHNUM; int tout = 0; @@ -227,6 +238,12 @@ int main(int argc, char **argv){ } else if(!strcmp(argv[i], "-ext")){ if(++i >= argc) usage(); extpath = argv[i]; + } else if(!strcmp(argv[i], "-extpc")){ + if(!extpcs) extpcs = tclistnew2(1); + if(++i >= argc) usage(); + tclistpush2(extpcs, argv[i]); + if(++i >= argc) usage(); + tclistpush2(extpcs, argv[i]); } else if(!strcmp(argv[i], "-mask")){ if(++i >= argc) usage(); mask |= getcmdmask(argv[i]); @@ -263,8 +280,9 @@ int main(int argc, char **argv){ if(!rtspath) rtspath = DEFRTSPATH; g_serv = ttservnew(); int rv = proc(dbname, host, port, thnum, tout, dmn, pidpath, logpath, - ulogpath, ulim, uas, sid, mhost, mport, rtspath, extpath, mask); + ulogpath, ulim, uas, sid, mhost, mport, rtspath, extpath, extpcs, mask); ttservdel(g_serv); + if(extpcs) tclistdel(extpcs); return rv; } @@ -276,8 +294,8 @@ static void usage(void){ fprintf(stderr, "usage:\n"); fprintf(stderr, " %s [-host name] [-port num] [-thnum num] [-tout num]" " [-dmn] [-pid path] [-log path] [-ld|-le] [-ulog path] [-ulim num] [-uas] [-sid num]" - " [-mhost name] [-mport num] [-rts path] [-ext path] [-mask expr] [-unmask expr]" - " [dbname]\n", g_progname); + " [-mhost name] [-mport num] [-rts path] [-ext path] [-extpc name period]" + " [-mask expr] [-unmask expr] [dbname]\n", g_progname); fprintf(stderr, "\n"); exit(1); } @@ -382,7 +400,7 @@ static int proc(const char *dbname, const char *host, int port, int thnum, int t bool dmn, const char *pidpath, const char *logpath, const char *ulogpath, uint64_t ulim, bool uas, uint32_t sid, const char *mhost, int mport, const char *rtspath, const char *extpath, - uint64_t mask){ + const TCLIST *extpcs, uint64_t mask){ LOGARG larg; larg.fd = 1; ttservsetloghandler(g_serv, do_log, &larg); @@ -516,7 +534,31 @@ static int proc(const char *dbname, const char *host, int port, int thnum, int t sarg.sid = sid; sarg.fail = false; sarg.recon = false; - if(!(mask & TTMSKSLAVE)) ttservsettimedhandler(g_serv, 1.0, do_slave, &sarg); + if(!(mask & TTMSKSLAVE)) ttservaddtimedhandler(g_serv, 1.0, do_slave, &sarg); + EXTPCARG *pcargs = NULL; + int pcnum = 0; + if(extpath && extpcs){ + pcnum = tclistnum(extpcs) / 2; + pcargs = tcmalloc(sizeof(*pcargs) * pcnum); + for(int i = 0; i < pcnum; i++){ + const char *name = tclistval2(extpcs, i * 2); + double period = tcatof(tclistval2(extpcs, i * 2 + 1)); + EXTPCARG *pcarg = pcargs + i; + pcarg->name = name; + pcarg->adb = adb; + pcarg->ulog = ulog; + pcarg->sid = sid; + pcarg->sarg = &sarg; + pcarg->scrext = scrextnew(thnum, thnum + i, extpath, adb, ulog, sid, scrstash, + scrlcks, RECMTXNUM, do_log); + if(pcarg->scrext){ + if(*name && period > 0) ttservaddtimedhandler(g_serv, period, do_extpc, pcarg); + } else { + err = true; + ttservlog(g_serv, TTLOGERROR, "scrextnew failed"); + } + } + } TASKARG targ; targ.mask = mask; targ.adb = adb; @@ -552,6 +594,16 @@ static int proc(const char *dbname, const char *host, int port, int thnum, int t } if(!ttservstart(g_serv)) err = true; } while(g_restart); + if(pcargs){ + for(int i = 0; i < pcnum; i++){ + EXTPCARG *pcarg = pcargs + i; + if(pcarg->scrext && !scrextdel(pcarg->scrext)){ + err = true; + ttservlog(g_serv, TTLOGERROR, "scrextdel failed"); + } + } + tcfree(pcargs); + } for(int i = 0; i < RECMTXNUM; i++){ if(pthread_mutex_destroy(targ.rmtxs + i) != 0) ttservlog(g_serv, TTLOGERROR, "pthread_mutex_destroy failed"); @@ -669,6 +721,17 @@ static void do_slave(void *opq){ } +/* perform an extension command */ +static void do_extpc(void *opq){ + EXTPCARG *arg = (EXTPCARG *)opq; + const char *name = arg->name; + void *scr = arg->scrext; + int xsiz; + char *xbuf = scrextcallmethod(scr, name, "", 0, "", 0, &xsiz); + tcfree(xbuf); +} + + /* handle a task and dispatch it */ static void do_task(TTSOCK *sock, void *opq, TTREQ *req){ TASKARG *arg = (TASKARG *)opq; ttserver.c @@ -387,6 +387,11 @@ bool ttsockprintf(TTSOCK *sock, const char *format, ...){ /* Receive data by a socket. */ bool ttsockrecv(TTSOCK *sock, char *buf, int size){ assert(sock && buf && size >= 0); + if(sock->rp + size <= sock->ep){ + memcpy(buf, sock->rp, size); + sock->rp += size; + return true; + } bool err = false; char *wp = buf; while(size > 0){ @@ -460,26 +465,18 @@ bool ttsockgets(TTSOCK *sock, char *buf, int size){ /* Receive an 32-bit integer by a socket. */ uint32_t ttsockgetint32(TTSOCK *sock){ assert(sock); - uint32_t num = ttsockgetc(sock) << 24; - num |= ttsockgetc(sock) << 16; - num |= ttsockgetc(sock) << 8; - num |= ttsockgetc(sock); - return num; + uint32_t num; + ttsockrecv(sock, (char *)&num, sizeof(num)); + return TTNTOHL(num); } /* Receive an 64-bit integer by a socket. */ uint64_t ttsockgetint64(TTSOCK *sock){ assert(sock); - uint64_t num = (uint64_t)ttsockgetc(sock) << 56; - num |= (uint64_t)ttsockgetc(sock) << 48; - num |= (uint64_t)ttsockgetc(sock) << 40; - num |= (uint64_t)ttsockgetc(sock) << 32; - num |= (uint64_t)ttsockgetc(sock) << 24; - num |= (uint64_t)ttsockgetc(sock) << 16; - num |= (uint64_t)ttsockgetc(sock) << 8; - num |= (uint64_t)ttsockgetc(sock); - return num; + uint64_t num; + ttsockrecv(sock, (char *)&num, sizeof(num)); + return TTNTOHLL(num); } @@ -743,14 +740,14 @@ TTSERV *ttservnew(void){ serv->queue = tclistnew(); if(pthread_mutex_init(&serv->qmtx, NULL) != 0) tcmyfatal("pthread_mutex_init failed"); if(pthread_cond_init(&serv->qcnd, NULL) != 0) tcmyfatal("pthread_cond_init failed"); + if(pthread_mutex_init(&serv->tmtx, NULL) != 0) tcmyfatal("pthread_mutex_init failed"); + if(pthread_cond_init(&serv->tcnd, NULL) != 0) tcmyfatal("pthread_cond_init failed"); serv->thnum = TTDEFTHNUM; serv->timeout = 0; serv->term = false; serv->do_log = NULL; serv->opq_log = NULL; - serv->freq_timed = 0.0; - serv->do_timed = NULL; - serv->opq_timed = NULL; + serv->timernum = 0; serv->do_task = NULL; serv->opq_task = NULL; return serv; @@ -760,8 +757,10 @@ TTSERV *ttservnew(void){ /* Delete a server object. */ void ttservdel(TTSERV *serv){ assert(serv); - pthread_mutex_destroy(&serv->qmtx); + pthread_cond_destroy(&serv->tcnd); + pthread_mutex_destroy(&serv->tmtx); pthread_cond_destroy(&serv->qcnd); + pthread_mutex_destroy(&serv->qmtx); tclistdel(serv->queue); tcfree(serv); } @@ -806,12 +805,15 @@ void ttservsetloghandler(TTSERV *serv, void (*do_log)(int, const char *, void *) } -/* Set the timed handler of a server object. */ -void ttservsettimedhandler(TTSERV *serv, double freq, void (*do_timed)(void *), void *opq){ +/* Add a timed handler to a server object. */ +void ttservaddtimedhandler(TTSERV *serv, double freq, void (*do_timed)(void *), void *opq){ assert(serv && freq >= 0.0 && do_timed); - serv->freq_timed = freq; - serv->do_timed = do_timed; - serv->opq_timed = opq; + if(serv->timernum >= TTTIMERMAX - 1) return; + TTTIMER *timer = serv->timers + serv->timernum; + timer->freq_timed = freq; + timer->do_timed = do_timed; + timer->opq_timed = opq; + serv->timernum++; } @@ -848,13 +850,13 @@ bool ttservstart(TTSERV *serv){ } ttservlog(serv, TTLOGSYSTEM, "service started: %d", getpid()); bool err = false; - TTTIMER timer; - timer.alive = false; - timer.serv = serv; - if(serv->do_timed){ - if(pthread_create(&timer.thid, NULL, ttservtimer, &timer) == 0){ - ttservlog(serv, TTLOGINFO, "timer thread started"); - timer.alive = true; + for(int i = 0; i < serv->timernum; i++){ + TTTIMER *timer = serv->timers + i; + timer->alive = false; + timer->serv = serv; + if(pthread_create(&(timer->thid), NULL, ttservtimer, timer) == 0){ + ttservlog(serv, TTLOGINFO, "timer thread %d started", i + 1); + timer->alive = true; } else { ttservlog(serv, TTLOGERROR, "pthread_create (ttservtimer) failed"); err = true; @@ -975,6 +977,10 @@ bool ttservstart(TTSERV *serv){ err = true; ttservlog(serv, TTLOGERROR, "pthread_cond_broadcast failed"); } + if(pthread_cond_broadcast(&serv->tcnd) != 0){ + err = true; + ttservlog(serv, TTLOGERROR, "pthread_cond_broadcast failed"); + } usleep(TTWAITWORKER * 1000); for(int i = 0; i < thnum; i++){ if(!reqs[i].alive) continue; @@ -992,12 +998,14 @@ bool ttservstart(TTSERV *serv){ if(tclistnum(serv->queue) > 0) ttservlog(serv, TTLOGINFO, "%d requests discarded", tclistnum(serv->queue)); tclistclear(serv->queue); - if(timer.alive){ + for(int i = 0; i < serv->timernum; i++){ + TTTIMER *timer = serv->timers + i; + if(!timer->alive) continue; void *rv; - if(pthread_cancel(timer.thid) == 0) - ttservlog(serv, TTLOGINFO, "timer thread was canceled"); - if(pthread_join(timer.thid, &rv) == 0){ - ttservlog(serv, TTLOGINFO, "timer thread finished"); + if(pthread_cancel(timer->thid) == 0) + ttservlog(serv, TTLOGINFO, "timer thread %d was canceled", i + 1); + if(pthread_join(timer->thid, &rv) == 0){ + ttservlog(serv, TTLOGINFO, "timer thread %d finished", i + 1); if(rv && rv != PTHREAD_CANCELED) err = true; } else { err = true; @@ -1062,9 +1070,40 @@ static void *ttservtimer(void *argp){ ttservlog(serv, TTLOGERROR, "pthread_setcancelstate failed"); } usleep(100000); + double freqi; + double freqd = modf(timer->freq_timed, &freqi); while(!serv->term){ - serv->do_timed(serv->opq_timed); - usleep(serv->freq_timed * 1000000); + if(pthread_mutex_lock(&serv->tmtx) == 0){ + struct timeval tv; + struct timespec ts; + if(gettimeofday(&tv, NULL) == 0){ + ts.tv_sec = tv.tv_sec + (int)freqi; + ts.tv_nsec = tv.tv_usec * 1000 + freqd * 1000000000; + if(ts.tv_nsec >= 1000000000){ + ts.tv_nsec -= 1000000000; + ts.tv_sec++; + } + } else { + ts.tv_sec = (1ULL << (sizeof(time_t) * 8 - 1)) - 1; + ts.tv_nsec = 0; + } + int code = pthread_cond_timedwait(&serv->tcnd, &serv->tmtx, &ts); + if(code == 0 || code == ETIMEDOUT || code == EINTR){ + if(pthread_mutex_unlock(&serv->tmtx) != 0){ + err = true; + ttservlog(serv, TTLOGERROR, "pthread_mutex_unlock failed"); + break; + } + if(code != 0) timer->do_timed(timer->opq_timed); + } else { + pthread_mutex_unlock(&serv->tmtx); + err = true; + ttservlog(serv, TTLOGERROR, "pthread_cond_timedwait failed"); + } + } else { + err = true; + ttservlog(serv, TTLOGERROR, "pthread_mutex_lock failed"); + } } return err ? "error" : NULL; } ttutil.c @@ -274,10 +274,15 @@ double ttunpackdouble(const char *buf); #define TTCMDMISC 0x90 /* ID of misc command */ #define TTCMDREPL 0xa0 /* ID of repl command */ +#define TTTIMERMAX 8 /* maximum number of timers */ + typedef struct _TTTIMER { /* type of structure for a timer */ pthread_t thid; /* thread ID */ bool alive; /* alive flag */ struct _TTSERV *serv; /* server object */ + double freq_timed; /* frequency of timed handler */ + void (*do_timed)(void *); /* call back function for timed handler */ + void *opq_timed; /* opaque pointer for timed handler */ } TTTIMER; typedef struct _TTREQ { /* type of structure for a server */ @@ -297,14 +302,15 @@ typedef struct _TTSERV { /* type of structure for a server */ TCLIST *queue; /* queue of requests */ pthread_mutex_t qmtx; /* mutex for the queue */ pthread_cond_t qcnd; /* condition variable for the queue */ + pthread_mutex_t tmtx; /* mutex for the timer */ + pthread_cond_t tcnd; /* condition variable for the timer */ int thnum; /* number of threads */ double timeout; /* timeout milliseconds of each task */ bool term; /* terminate flag */ void (*do_log)(int, const char *, void *); /* call back function for logging */ void *opq_log; /* opaque pointer for logging */ - double freq_timed; /* frequency of timed handler */ - void (*do_timed)(void *); /* call back function for timed handler */ - void *opq_timed; /* opaque pointer for timed handler */ + TTTIMER timers[TTTIMERMAX]; /* timer objects */ + int timernum; /* number of timer objects */ void (*do_task)(TTSOCK *, void *, TTREQ *req); /* call back function for task */ void *opq_task; /* opaque pointer for task */ } TTSERV; @@ -354,13 +360,13 @@ void ttservtune(TTSERV *serv, int thnum, double timeout); void ttservsetloghandler(TTSERV *serv, void (*do_log)(int, const char *, void *), void *opq); -/* Set the timed handler of a server object. +/* Add a timed handler to a server object. `serv' specifies the server object. `freq' specifies the frequency of execution in seconds. `do_timed' specifies the pointer to a function to do with a event. Its parameter is the opaque pointer. `opq' specifies the opaque pointer to be passed to the handler. It can be `NULL'. */ -void ttservsettimedhandler(TTSERV *serv, double freq, void (*do_timed)(void *), void *opq); +void ttservaddtimedhandler(TTSERV *serv, double freq, void (*do_timed)(void *), void *opq); /* Set the response handler of a server object. @@ -404,8 +410,8 @@ bool ttserviskilled(TTSERV *serv); *************************************************************************************************/ -#define _TT_VERSION "1.1.13" -#define _TT_LIBVER 204 +#define _TT_VERSION "1.1.14" +#define _TT_LIBVER 205 #define _TT_PROTVER "0.9" ttutil.h cc296d2e4f68ceb40626252ac265d296a2d3b19c Bob Ippolito bob@redivi.com http://github.com/etrepum/tokyo-tyrant/commit/9d36d5d6e027e4b054e180873155a590519e587d 9d36d5d6e027e4b054e180873155a590519e587d 2009-02-08T20:18:50-08:00 2009-02-08T20:18:50-08:00 sync with tokyotyrant-1.1.14.tar.gz e4a695a157e28a3a3d13fd737f864f670ba6ed61 Bob Ippolito bob@redivi.com