@@ -1,3 +1,25 @@ +2009-04-08 Mikio Hirabayashi <mikio@users.sourceforge.net> + + * tcutil.c (tcsleep): new function. + + * tchdb.c (tchdbtranbegin): locking algorithm was modified. + + * tchdb.c (tchdblocktran, tchdbunlocktran): abolished. + + * tcbdb.c (tcbdbtranbegin): locking algorithm was modified. + + * tcbdb.c (tcbdbcurjumpimpl): a bug related to cursor initialization was fixed. + + * tcfdb.c (tcbdbtranbegin): locking algorithm was modified. + + * tcfdb.c (tcfdblocktran, tcfdbunlocktran): abolished. + + * tctdb.c (tctdbqryallcondmatch): a bug related to handling null value was fixed. + + * tcadb.c (tcadboptimize, tcadbpath): new functions. + + - Release: 1.4.15 + 2009-04-07 Mikio Hirabayashi <mikio@users.sourceforge.net> * tcbdb.c (tcbdbputimpl, tcbdbcurputimpl): page size limitation was added. ChangeLog @@ -89,7 +89,7 @@ all : $(LIBRARYFILES) $(COMMANDFILES) $(CGIFILES) clean : rm -rf $(LIBRARYFILES) $(LIBOBJFILES) $(COMMANDFILES) $(CGIFILES) \ *.o a.out tokyocabinet_all.c check.in check.out gmon.out leak.log words.tsv \ - casket casket-* casket.* *.wal *~ hoge moge + casket casket-* casket.* *.tch *.tcb *.tcf *.tct *.idx.* *.wal *~ hoge moge tako ika version : @@ -518,6 +518,7 @@ check-adb : $(RUNENV) $(RUNCMD) ./tcamgr get casket.tch four > check.out $(RUNENV) $(RUNCMD) ./tcamgr get casket.tch five > check.out $(RUNENV) $(RUNCMD) ./tcamgr list -pv -fm f casket.tch > check.out + $(RUNENV) $(RUNCMD) ./tcamgr optimize casket.tch $(RUNENV) $(RUNCMD) ./tcamgr put -dc casket.tch three third $(RUNENV) $(RUNCMD) ./tcamgr get casket.tch three > check.out $(RUNENV) $(RUNCMD) ./tcamgr get casket.tch four > check.out Makefile.in @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.61 for tokyocabinet 1.4.14. +# Generated by GNU Autoconf 2.61 for tokyocabinet 1.4.15. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. @@ -572,8 +572,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='tokyocabinet' PACKAGE_TARNAME='tokyocabinet' -PACKAGE_VERSION='1.4.14' -PACKAGE_STRING='tokyocabinet 1.4.14' +PACKAGE_VERSION='1.4.15' +PACKAGE_STRING='tokyocabinet 1.4.15' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1192,7 +1192,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures tokyocabinet 1.4.14 to adapt to many kinds of systems. +\`configure' configures tokyocabinet 1.4.15 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1253,7 +1253,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of tokyocabinet 1.4.14:";; + short | recursive ) echo "Configuration of tokyocabinet 1.4.15:";; esac cat <<\_ACEOF @@ -1354,7 +1354,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -tokyocabinet configure 1.4.14 +tokyocabinet configure 1.4.15 generated by GNU Autoconf 2.61 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1368,7 +1368,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by tokyocabinet $as_me 1.4.14, which was +It was created by tokyocabinet $as_me 1.4.15, which was generated by GNU Autoconf 2.61. Invocation command line was $ $0 $@ @@ -1724,7 +1724,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu # Package information MYLIBVER=8 -MYLIBREV=5 +MYLIBREV=6 MYFORMATVER="1.0" # Targets @@ -6351,7 +6351,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by tokyocabinet $as_me 1.4.14, which was +This file was extended by tokyocabinet $as_me 1.4.15, which was generated by GNU Autoconf 2.61. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -6394,7 +6394,7 @@ Report bugs to <bug-autoconf@gnu.org>." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF ac_cs_version="\\ -tokyocabinet config.status 1.4.14 +tokyocabinet config.status 1.4.15 configured by $0, generated by GNU Autoconf 2.61, with options \\"`echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" configure @@ -7,11 +7,11 @@ #================================================================ # Package name -AC_INIT(tokyocabinet, 1.4.14) +AC_INIT(tokyocabinet, 1.4.15) # Package information MYLIBVER=8 -MYLIBREV=5 +MYLIBREV=6 MYFORMATVER="1.0" # Targets configure.in @@ -126,7 +126,7 @@ <p>The following are the source packages of Tokyo Cabinet. As for binary packages, see the site of each distributor.</p> <ul> -<li><a href="tokyocabinet-1.4.14.tar.gz">Latest Source Package (version 1.4.14)</a></li> +<li><a href="tokyocabinet-1.4.15.tar.gz">Latest Source Package (version 1.4.15)</a></li> <li><a href="pastpkg/">Past Versions</a></li> </ul> doc/index.html @@ -167,7 +167,7 @@ <p>以下のソースパッケージをダウンロードしてください。バイナリパッケージについては、各ディストリビュータのサイトをご覧ください。</p> <ul> -<li><a href="tokyocabinet-1.4.14.tar.gz">最新のソースパッケージ（バージョン1.4.14）</a></li> +<li><a href="tokyocabinet-1.4.15.tar.gz">最新のソースパッケージ（バージョン1.4.15）</a></li> <li><a href="pastpkg/">過去のバージョン</a></li> </ul> doc/index.ja.html @@ -41,6 +41,7 @@ <li><a href="#tcfdbapi">The Fixed-length Database API</a></li> <li><a href="#tctdbapi">The Table Database API</a></li> <li><a href="#tcadbapi">The Abstract Database API</a></li> +<li><a href="#fileformat">File Format</a></li> <li><a href="#license">License</a></li> </ol> @@ -56,7 +57,7 @@ <p>As for database of fixed-length array, records are stored with unique natural numbers. It is impossible to store two or more records with a key overlaps. Moreover, the length of each record is limited by the specified length. Provided operations are the same as ones of hash database.</p> -<p>Table database is also provided as a variant of hash database. Each record is identified by the primary key and has a set of named columns. Although there is no concept of data schema, it is possible to search for records with complex conditions efficiently by using indexes of arbitrary columns.</p> +<p>Table database is also provided as a variant of hash database. Each record is identified by the primary key and has a set of named columns. Although there is no concept of data schema, it is possible to search for records with complex conditions efficiently by using indices of arbitrary columns.</p> <p>Tokyo Cabinet is written in the C language, and provided as API of C, Perl, Ruby, Java, and Lua. Tokyo Cabinet is available on platforms which have API conforming to C99 and POSIX. Tokyo Cabinet is a free software licensed under the GNU Lesser General Public License.</p> @@ -117,7 +118,7 @@ <p>Table database supports query functions with not only the primary key but also with conditions about arbitrary columns. Each column condition is composed of the name of a column and a condition expression. Operators of full matching, forward matching, regular expression matching, and so on are provided for the string type. Operators of full matching, range matching and so on are provided for the number type. A query can contain multiple conditions. The order of the result set can be specified as the ascending or descending order of strings or numbers.</p> -<p>You can create indexes for arbitrary columns to improve performance of search and sorting. Although columns do not have data types, indexes have types for strings or numbers. The query optimizer uses indexes in suitable way according to each query. Indexes are implemented as different files of B+ tree database.</p> +<p>You can create indices for arbitrary columns to improve performance of search and sorting. Although columns do not have data types, indices have types for strings or numbers. The query optimizer uses indices in suitable way according to each query. Indices are implemented as different files of B+ tree database.</p> <h3 id="features_practical">Practical Functionality</h3> @@ -5307,7 +5308,7 @@ int main(int argc, char **argv){ <dd>`<var>lcnum</var>' specifies the maximum number of leaf nodes to be cached. If it is not more than 0, the default value is specified. The default value is 4096.</dd> <dd>`<var>ncnum</var>' specifies the maximum number of non-leaf nodes to be cached. If it is not more than 0, the default value is specified. The default value is 512.</dd> <dd>If successful, the return value is true, else, it is false.</dd> -<dd>Note that the caching parameters should be set before the database is opened. Leaf nodes and non-leaf nodes are used in column indexes.</dd> +<dd>Note that the caching parameters should be set before the database is opened. Leaf nodes and non-leaf nodes are used in column indices.</dd> </dl> <p>The function `tctdbsetxmsiz' is used in order to set the size of the extra mapped memory of a table database object.</p> @@ -5692,7 +5693,7 @@ int main(int argc, char **argv){ <dd>`<var>name</var>' specifies the name of a column. If the name of an existing index is specified, the index is rebuilt. An empty string means the primary key.</dd> <dd>`<var>type</var>' specifies the index type: `TDBITLEXICAL' for lexical string, `TDBITDECIMAL' for decimal string. If it is `TDBITOPT', the index is optimized. If it is `TDBITVOID', the index is removed. If `TDBITKEEP' is added by bitwise-or and the index exists, this function merely returns failure.</dd> <dd>If successful, the return value is true, else, it is false.</dd> -<dd>Note that the setting indexes should be set after the database is opened.</dd> +<dd>Note that the setting indices should be set after the database is opened.</dd> </dl> @@ -6045,7 +6046,7 @@ int main(int argc, char **argv){ <p>Objects whose type is pointer to `<code>TCADB</code>' are used to handle abstract databases. An abstract database object is created with the function `<code>tcadbnew</code>' and is deleted with the function `<code>tcadbdel</code>'. To avoid memory leak, it is important to delete every object when it is no longer in use.</p> -<p>Before operations to store or retrieve records, it is necessary to connect the abstract database object to the concrete one. The function `<code>tcadbopen</code>' is used to open a concrete database and the function `<code>tcadbclose</code>' is used to close the database. To avoid data missing or corruption, it is important to close every database file when it is no longer in use.</p> +<p>Before operations to store or retrieve records, it is necessary to connect the abstract database object to the concrete one. The function `<code>tcadbopen</code>' is used to open a concrete database and the function `<code>tcadbclose</code>' is used to close the database. To avoid data missing or corruption, it is important to close every database instance when it is no longer in use.</p> <h3 id="tcadbapi_api">API</h3> @@ -6297,6 +6298,16 @@ int main(int argc, char **argv){ <dd>If successful, the return value is true, else, it is false.</dd> </dl> +<p>The function `tcadboptimize' is used in order to optimize the storage of an abstract database object.</p> + +<dl class="api"> +<dt><code>bool tcadboptimize(TCADB *<var>adb</var>, const char *<var>params</var>);</code></dt> +<dd>`<var>adb</var>' specifies the abstract database object.</dd> +<dd>`<var>params</var>' specifies the string of the tuning parameters, which works as with the tuning of parameters the function `tcadbopen'. If it is `NULL', it is not used.</dd> +<dd>If successful, the return value is true, else, it is false.</dd> +<dd>This function is useful to reduce the size of the database storage with data fragmentation by successive updating.</dd> +</dl> + <p>The function `tcadbvanish' is used in order to remove all records of an abstract database object.</p> <dl class="api"> @@ -6342,6 +6353,14 @@ int main(int argc, char **argv){ <dd>Update in the transaction is discarded when it is aborted. The state of the database is rollbacked to before transaction.</dd> </dl> +<p>The function `tcadbpath' is used in order to get the file path of an abstract database object.</p> + +<dl class="api"> +<dt><code>const char *tcadbpath(TCADB *<var>adb</var>);</code></dt> +<dd>`<var>adb</var>' specifies the abstract database object.</dd> +<dd>The return value is the path of the database file or `NULL' if the object does not connect to any database. "*" stands for on-memory hash database. "+" stands for on-memory tree database.</dd> +</dl> + <p>The function `tcadbrnum' is used in order to get the number of records of an abstract database object.</p> <dl class="api"> @@ -6455,7 +6474,7 @@ int main(int argc, char **argv){ <p>This command returns 0 on success, another on failure.</p> -<p>The command `<code>tcamgr</code>' is a utility for test and debugging of the abstract database API and its applications. `<var>name</var>' specifies the name of a database. `<var>key</var>' specifies the key of a record. `<var>value</var>' specifies the value of a record. `<var>func</var>' specifies the name of a function. `<var>arg</var>' specifies the arguments of the function. `<var>dest</var>' specifies the path of the destination file.</p> +<p>The command `<code>tcamgr</code>' is a utility for test and debugging of the abstract database API and its applications. `<var>name</var>' specifies the name of a database. `<var>key</var>' specifies the key of a record. `<var>value</var>' specifies the value of a record. `<var>params</var>' specifies the tuning parameters. `<var>func</var>' specifies the name of a function. `<var>arg</var>' specifies the arguments of the function. `<var>dest</var>' specifies the path of the destination file.</p> <dl class="api"> <dt><code>tcamgr create <var>name</var></code></dt> @@ -6470,6 +6489,8 @@ int main(int argc, char **argv){ <dd>Print the value of a record.</dd> <dt><code>tcamgr list [-sep <var>chr</var>] [-m <var>num</var>] [-pv] [-px] [-fm <var>str</var>] <var>name</var></code></dt> <dd>Print keys of all records, separated by line feeds.</dd> +<dt><code>tcamgr optimize <var>name</var> <var>params</var></code></dt> +<dd>Optimize a database file.</dd> <dt><code>tcamgr misc [-sx] [-sep <var>chr</var>] [-px] <var>name</var> <var>func</var> [<var>arg</var>...]</code></dt> <dd>Call a versatile function for miscellaneous operations.</dd> <dt><code>tcamgr map [-fm <var>str</var>] <var>name</var> <var>dest</var></code></dt> @@ -6504,6 +6525,563 @@ int main(int argc, char **argv){ <hr /> +<h2 id="fileformat">File Format</h2> + +<p>This section describes the format of the database files of Tokyo Cabinet.</p> + +<h3 id="fileformat_tchdb">File Format of Hash Database</h3> + +<p>There are four sections in the file managed by the hash database; the header section, the bucket section, the free block pool section, and the record section. Numeric values in the file are serialized in the little endian order or in the variable length format. The latter format is delta encoding based on the 128-radix numbering.</p> + +<p>The header section is from the top of the file and its length is 256 bytes. There are the following information.</p> + +<table summary="database header format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>magic number</td> +<td class="number">0</td> +<td class="number">32</td> +<td>identification of the database. Begins with "ToKyO CaBiNeT"</td> +</tr> +<tr> +<td>database type</td> +<td class="number">32</td> +<td class="number">1</td> +<td>hash (0x01) / B+ tree (0x02) / fixed-length (0x03) / table (0x04)</td> +</tr> +<tr> +<td>additional flags</td> +<td class="number">33</td> +<td class="number">1</td> +<td>logical union of open (1<<0) and fatal (1<<1)</td> +</tr> +<tr> +<td>alignment power</td> +<td class="number">34</td> +<td class="number">1</td> +<td>the alignment size, by power of 2</td> +</tr> +<tr> +<td>free block pool power</td> +<td class="number">35</td> +<td class="number">1</td> +<td>the number of elements in the free block pool, by power of 2</td> +</tr> +<tr> +<td>options</td> +<td class="number">36</td> +<td class="number">1</td> +<td>logical union of large (1<<0), Deflate (1<<1), BZIP2 (1<<2), TCBS (1<<3), extra codec (1<<4)</td> +</tr> +<tr> +<td>bucket number</td> +<td class="number">40</td> +<td class="number">8</td> +<td>the number of elements of the bucket array</td> +</tr> +<tr> +<td>record number</td> +<td class="number">48</td> +<td class="number">8</td> +<td>the number of records in the database</td> +</tr> +<tr> +<td>file size</td> +<td class="number">56</td> +<td class="number">8</td> +<td>the file size of the database</td> +</tr> +<tr> +<td>first record</td> +<td class="number">64</td> +<td class="number">8</td> +<td>the offset of the first record</td> +</tr> +<tr> +<td>opaque region</td> +<td class="number">128</td> +<td class="number">128</td> +<td>users can use this region arbitrarily</td> +</tr> +</table> + +<p>The bucket section trails the header section and its size is defined by the bucket number. Each element of the bucket array indicates the offset of the first record of the hash chain. The format of each element is the fixed length number and its size is 4 bytes in the normal mode or 8 bytes in the large mode. The offset is recorded as the quotient by the alignment.</p> + +<p>The free block pool section trails the bucket section and its size is defined by the free block pool number. Each element of the free block pool indicates the offset and the size of each free block. The offset is recorded as the difference of the former free block and as the quotient by the alignment. The offset and the size are serialized in the variable length format.</p> + +<p>The record section trails the free block pool section and occupies the rest region to the end of the file. Each element has the following information. The region of each record begins at the offset of the multiple of the alignment.</p> + +<table summary="record format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>magic number</td> +<td class="number">0</td> +<td class="number">1</td> +<td>identification of record block. always 0xC8</td> +</tr> +<tr> +<td>hash value</td> +<td class="number">1</td> +<td class="number">1</td> +<td>the hash value to decide the path of the hash chain</td> +</tr> +<tr> +<td>left chain</td> +<td class="number">2</td> +<td class="number">4</td> +<td>the alignment quotient of the destination of the left chain</td> +</tr> +<tr> +<td>right chain</td> +<td class="number">6</td> +<td class="number">4</td> +<td>the alignment quotient of the destination of the right chain</td> +</tr> +<tr> +<td>padding size</td> +<td class="number">10</td> +<td class="number">2</td> +<td>the size of the padding</td> +</tr> +<tr> +<td>key size</td> +<td class="number">12</td> +<td class="number">vary</td> +<td>the size of the key</td> +</tr> +<tr> +<td>value size</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the size of the value</td> +</tr> +<tr> +<td>key</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the data of the key</td> +</tr> +<tr> +<td>value</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the data of the value</td> +</tr> +<tr> +<td>padding</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>useless data</td> +</tr> +</table> + +<p>However, regions of free blocks contain the following information.</p> + +<table summary="free block format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>magic number</td> +<td class="number">0</td> +<td class="number">1</td> +<td>identification of record block. always 0xB0</td> +</tr> +<tr> +<td>block size</td> +<td class="number">1</td> +<td class="number">4</td> +<td>size of the block</td> +</tr> +</table> + +<p>The transaction log is recorded in the file whose name is composed of the database name and the suffix ".wal". The top eight bytes indicate the file size of the beginning of the transaction. After that, there are the following information.</p> + +<table summary="transaction log format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>offset</td> +<td class="number">0</td> +<td class="number">8</td> +<td>the offset of the updated region</td> +</tr> +<tr> +<td>size</td> +<td class="number">8</td> +<td class="number">4</td> +<td>the size of the updated region</td> +</tr> +<tr> +<td>data</td> +<td class="number">12</td> +<td class="number">vary</td> +<td>the data before update</td> +</tr> +</table> + +<h3 id="fileformat_tcbdb">File Format of B+ Tree Database</h3> + +<p>All data managed by the B+ tree database are recorded in the hash database. Recorded data are classified into meta data and logical pages. Logical pages are classified into leaf nodes and non-leaf nodes. The formats of the fixed length number and the variable length number are the same as with the hash database.</p> + +<p>Meta data are recorded in the opaque region in the header of the hash database and have the following information.</p> + +<table summary="database header format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> + +<tr> +<td>comparison function</td> +<td class="number">0</td> +<td class="number">1</td> +<td>tccmplexical (0x00), tccmpdecimal (0x01), tccmpint32 (0x02), tccmpint64 (0x03), other (0xff)</td> +</tr> +<tr> +<td>reserved region</td> +<td class="number">1</td> +<td class="number">7</td> +<td>not used</td> +</tr> +<tr> +<td>record number of leaf node</td> +<td class="number">8</td> +<td class="number">4</td> +<td>the maximum number of records in a leaf node</td> +</tr> +<tr> +<td>index number of non-leaf node</td> +<td class="number">12</td> +<td class="number">4</td> +<td>the maximum number of indices in a leaf node</td> +</tr> +<tr> +<td>root node ID</td> +<td class="number">16</td> +<td class="number">8</td> +<td>the page ID of the root node of B+ tree</td> +</tr> +<tr> +<td>first leaf ID</td> +<td class="number">24</td> +<td class="number">8</td> +<td>the page ID of the first leaf node</td> +</tr> +<tr> +<td>last leaf ID</td> +<td class="number">32</td> +<td class="number">8</td> +<td>the page ID of the last leaf node</td> +</tr> +<tr> +<td>leaf number</td> +<td class="number">40</td> +<td class="number">8</td> +<td>the number of the leaf nodes</td> +</tr> +<tr> +<td>non-leaf number</td> +<td class="number">48</td> +<td class="number">8</td> +<td>the number of the non-leaf nodes</td> +</tr> +<tr> +<td>record number</td> +<td class="number">56</td> +<td class="number">8</td> +<td>the number of records in the database</td> +</tr> +</table> + +<p>Each leaf node contains a list of records. Each non-leaf node contains a list of indices to child nodes. Though each record is a logical unit of user data, records with the same key are integrated into one record physically. Each physical record has the following information.</p> + +<table summary="record format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>key size</td> +<td class="number">0</td> +<td class="number">vary</td> +<td>the size of the key</td> +</tr> +<tr> +<td>value size</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the size of the value</td> +</tr> +<tr> +<td>duplication number</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the number of values with the same key</td> +</tr> +<tr> +<td>key</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the data of the key</td> +</tr> +<tr> +<td>value</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the data of the value</td> +</tr> +<tr> +<td>duplicated records</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>a list of value sizes and value data</td> +</tr> +</table> + +<p>Each leaf node is a physical unit of a set of records. Each leaf node is identified by the sequential ID number from 1. Each leaf node is recorded in the hash database. The key is a string in the hexadecimal numbering. The value has the following information. Records are kept in the ascending order of keys.</p> + +<table summary="leaf node format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>previous leaf</td> +<td class="number">0</td> +<td class="number">vary</td> +<td>the ID number of the previous leaf node</td> +</tr> +<tr> +<td>next leaf</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the ID number of the next leaf node</td> +</tr> +<tr> +<td>record list</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the serialized data of all records in the node</td> +</tr> +</table> + +<p>Each index is a logical unit of pointer to the child node. Each index has the following information.</p> + +<table summary="index format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>page ID</td> +<td class="number">0</td> +<td class="number">vary</td> +<td>the ID number of the referred page</td> +</tr> +<tr> +<td>key size</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the size of the key</td> +</tr> +<tr> +<td>key</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the data of the key</td> +</tr> +</table> + +<p>Each non-leaf node is a physical unit of a set of indices. Each non-leaf node is identified by the sequential number from 281474976710657. Each non-leaf node is recorded in the hash database. The key is a string begins with "#" and is trailed by the hexadecimal number of the ID number subtracted by 281474976710657. The value has the following information. Indices are kept in the ascending order of keys.</p> + +<table summary="non-leaf format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>accession ID</td> +<td class="number">0</td> +<td class="number">vary</td> +<td>the ID number of the first child node</td> +</tr> +<tr> +<td>index list</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the serialized data of all indices in the node</td> +</tr> +</table> + +<h3 id="fileformat_tcfdb">File Format of Fixed-length Database</h3> + +<p>There are two sections in the file managed by the fixed-length database; the header section, and the record section. Numeric values in the file are serialized in the little endian order.</p> + +<p>The header section is from the top of the file and its length is 256 bytes. There are the following information.</p> + +<table summary="database header format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>magic number</td> +<td class="number">0</td> +<td class="number">32</td> +<td>identification of the database. Begins with "ToKyO CaBiNeT"</td> +</tr> +<tr> +<td>database type</td> +<td class="number">32</td> +<td class="number">1</td> +<td>always 0x03</td> +</tr> +<tr> +<td>additional flags</td> +<td class="number">33</td> +<td class="number">1</td> +<td>logical union of open (1<<0) and fatal (1<<1)</td> +</tr> +<tr> +<td>record number</td> +<td class="number">48</td> +<td class="number">8</td> +<td>the number of records in the database</td> +</tr> +<tr> +<td>file size</td> +<td class="number">56</td> +<td class="number">8</td> +<td>the file size of the database</td> +</tr> +<tr> +<td>record width</td> +<td class="number">64</td> +<td class="number">8</td> +<td>the width of each record</td> +</tr> +<tr> +<td>limit size</td> +<td class="number">72</td> +<td class="number">8</td> +<td>the limit size of the database</td> +</tr> +<tr> +<td>least ID</td> +<td class="number">80</td> +<td class="number">8</td> +<td>the least ID number of records</td> +</tr> +<tr> +<td>greatest ID</td> +<td class="number">88</td> +<td class="number">8</td> +<td>the greatest ID number of records</td> +</tr> +<tr> +<td>opaque region</td> +<td class="number">128</td> +<td class="number">128</td> +<td>users can use this region arbitrarily</td> +</tr> +</table> + +<p>The record section trails the header section and occupies the rest region to the end of the file. Each element has the following information. The size region takes 1 byte if the record width is less than 256 bytes, or takes 2 bytes if the record width is less than 65536, else takes 4 bytes. The size of each record is the summation of the size of the width region and the record width. So, the region of each record begins at the offset generated by the ID number subtracted by 1 and multiplied by the record width and the added by 256.</p> + +<table summary="record format"> +<tr> +<td class="label">name</td> +<td class="label">offset</td> +<td class="label">length</td> +<td class="label">feature</td> +</tr> +<tr> +<td>value size</td> +<td class="number">0</td> +<td class="number">vary</td> +<td>the size of the value</td> +</tr> +<tr> +<td>value</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>the data of the value</td> +</tr> +<tr> +<td>padding</td> +<td class="number">vary</td> +<td class="number">vary</td> +<td>padding. If the size of the value is 0, the first byte indicates whether the record exists or not</td> +</tr> +</table> + +<p>The naming convention and the file format of the transaction log file is the same as the one of the hash database.</p> + +<h3 id="fileformat_note">Note</h3> + +<p>Because database files are not sparse, you can copy them as with normal files. Moreover, the database formats don't depend on the byte order of the running environment, you can migrate the database files between environments with different byte orders.</p> + +<p>If possible, set the MIME type `<code>application/x-tokyocabinet-hash</code>' when sending files of the hash database. The suffix of the file name should be `<code>.tch</code>'. As for the B+ tree database, `<code>application/x-tokyocabinet-btree</code>' and `<code>.tcb</code>'. As for the fixed-length database, `<code>application/x-tokyocabinet-fixed</code>' and `<code>.tcf</code>'. As for the table database, `<code>application/x-tokyocabinet-btree</code>' and `<code>.tct</code>'.</p> + +<p>To make the `<code>file</code>' command identify the database formats, append the following lines to the `<code>magic</code>' file.</p> + +<pre># Tokyo Cabinet magic data +0 string ToKyO\ CaBiNeT\n Tokyo Cabinet +>14 string x \b (%s) +>32 byte 0 \b, Hash +!:mime application/x-tokyocabinet-hash +>32 byte 1 \b, B+ tree +!:mime application/x-tokyocabinet-btree +>32 byte 2 \b, Fixed-length +!:mime application/x-tokyocabinet-fixed +>32 byte 3 \b, Table +!:mime application/x-tokyocabinet-table +>33 byte &1 \b, [open] +>33 byte &2 \b, [fatal] +>34 byte x \b, apow=%d +>35 byte x \b, fpow=%d +>36 byte &1 \b, [large] +>36 byte &2 \b, [deflate] +>36 byte &4 \b, [bzip] +>36 byte &8 \b, [tcbs] +>36 byte &16 \b, [excodec] +>40 lequad x \b, bnum=%lld +>48 lequad x \b, rnum=%lld +>56 lequad x \b, fsiz=%lld +</pre> + +<hr /> + <h2 id="license">License</h2> <p>Tokyo Cabinet is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License or any later version.</p> doc/spex-en.html @@ -5310,7 +5310,7 @@ int main(int argc, char **argv){ <dd>`<var>lcnum</var>' specifies the maximum number of leaf nodes to be cached. If it is not more than 0, the default value is specified. The default value is 2048.</dd> <dd>`<var>ncnum</var>' specifies the maximum number of non-leaf nodes to be cached. If it is not more than 0, the default value is specified. The default value is 512.</dd> <dd>If successful, the return value is true, else, it is false.</dd> -<dd>Note that the caching parameters should be set before the database is opened. Leaf nodes and non-leaf nodes are used in column indexes.</dd> +<dd>Note that the caching parameters should be set before the database is opened. Leaf nodes and non-leaf nodes are used in column indices.</dd> </dl> <p>The function `tctdbsetxmsiz' is used in order to set the size of the extra mapped memory of a table database object.</p> @@ -5695,7 +5695,7 @@ int main(int argc, char **argv){ <dd>`<var>name</var>' specifies the name of a column. If the name of an existing index is specified, the index is rebuilt. An empty string means the primary key.</dd> <dd>`<var>type</var>' specifies the index type: `TDBITLEXICAL' for lexical string, `TDBITDECIMAL' for decimal string. If it is `TDBITOPT', the index is optimized. If it is `TDBITVOID', the index is removed. If `TDBITKEEP' is added by bitwise-or and the index exists, this function merely returns failure.</dd> <dd>If successful, the return value is true, else, it is false.</dd> -<dd>Note that the setting indexes should be set after the database is opened.</dd> +<dd>Note that the setting indices should be set after the database is opened.</dd> </dl> @@ -6298,6 +6298,16 @@ int main(int argc, char **argv){ <dd>If successful, the return value is true, else, it is false.</dd> </dl> +<p>The function `tcadboptimize' is used in order to optimize the storage of an abstract database object.</p> + +<dl class="api"> +<dt><code>bool tcadboptimize(TCADB *<var>adb</var>, const char *<var>params</var>);</code></dt> +<dd>`<var>adb</var>' specifies the abstract database object.</dd> +<dd>`<var>params</var>' specifies the string of the tuning parameters, which works as with the tuning of parameters the function `tcadbopen'. If it is `NULL', it is not used.</dd> +<dd>If successful, the return value is true, else, it is false.</dd> +<dd>This function is useful to reduce the size of the database storage with data fragmentation by successive updating.</dd> +</dl> + <p>The function `tcadbvanish' is used in order to remove all records of an abstract database object.</p> <dl class="api"> @@ -6343,6 +6353,14 @@ int main(int argc, char **argv){ <dd>Update in the transaction is discarded when it is aborted. The state of the database is rollbacked to before transaction.</dd> </dl> +<p>The function `tcadbpath' is used in order to get the file path of an abstract database object.</p> + +<dl class="api"> +<dt><code>const char *tcadbpath(TCADB *<var>adb</var>);</code></dt> +<dd>`<var>adb</var>' specifies the abstract database object.</dd> +<dd>The return value is the path of the database file or `NULL' if the object does not connect to any database. "*" stands for on-memory hash database. "+" stands for on-memory tree database.</dd> +</dl> + <p>The function `tcadbrnum' is used in order to get the number of records of an abstract database object.</p> <dl class="api"> @@ -6456,7 +6474,7 @@ int main(int argc, char **argv){ <p>このコマンドは処理が正常に終了すれば 0 を返し、エラーがあればそれ以外の値を返して終了します。</p> -<p>コマンド `<code>tcamgr</code>' は、抽象データベースAPIやそのアプリケーションのテストやデバッグに役立つツールです。以下の書式で用います。`<var>name</var>' はデータベースの名前を指定し、`<var>key</var>' はレコードのキーを指定し、`<var>value</var>' はレコードの値を指定し、`<var>func</var>' は関数の名前を指定し、`<var>arg</var>' は関数の引数を指定し、`<var>dest</var>' は格納先のファイルを指定します。</p> +<p>コマンド `<code>tcamgr</code>' は、抽象データベースAPIやそのアプリケーションのテストやデバッグに役立つツールです。以下の書式で用います。`<var>name</var>' はデータベースの名前を指定し、`<var>key</var>' はレコードのキーを指定し、`<var>value</var>' はレコードの値を指定し、`<var>params</var>' はチューニングパラメータを指定し、`<var>func</var>' は関数の名前を指定し、`<var>arg</var>' は関数の引数を指定し、`<var>dest</var>' は格納先のファイルを指定します。</p> <dl class="api"> <dt><code>tcamgr create <var>name</var></code></dt> @@ -6471,6 +6489,8 @@ int main(int argc, char **argv){ <dd>レコードの値を取得して標準出力する。</dd> <dt><code>tcamgr list [-sep <var>chr</var>] [-m <var>num</var>] [-pv] [-px] [-fm <var>str</var>] <var>name</var></code></dt> <dd>全てのレコードのキーを改行で区切って標準出力する。</dd> +<dt><code>tcamgr optimize <var>name</var> <var>params</var></code></dt> +<dd>データベースを最適化する。</dd> <dt><code>tcamgr misc [-sx] [-sep <var>chr</var>] [-px] <var>name</var> <var>func</var> [<var>arg</var>...]</code></dt> <dd>雑多な操作の多目的関数を呼び出す。</dd> <dt><code>tcamgr map [-fm <var>str</var>] <var>name</var> <var>dest</var></code></dt> @@ -6771,7 +6791,7 @@ cp -f "$srcpath" "$destpath" <td>データベースタイプ</td> <td class="number">32</td> <td class="number">1</td> -<td>ハッシュ表（0x01）かB+木（0x02）</td> +<td>ハッシュ表（0x01）かB+木（0x02）か固定長（0x03）かテーブル（0x04）</td> </tr> <tr> <td>追加フラグ</td> @@ -6900,7 +6920,7 @@ cp -f "$srcpath" "$destpath" <td>パディング</td> <td class="number">可変</td> <td class="number">可変</td> -<td>パディング</td> +<td>意味を持たないデータ</td> </tr> </table> @@ -6971,10 +6991,10 @@ cp -f "$srcpath" "$destpath" </tr> <tr> -<td>比較関数フラグ</td> +<td>比較関数</td> <td class="number">0</td> <td class="number">1</td> -<td>比較関数がtccmplexical（デフォルト）なら0x0、tccmpdecimalなら0x1、tccmpint32なら0x2、tccmpint64なら0x3、それ以外なら0xff</td> +<td>比較関数がtccmplexical（デフォルト）なら0x00、tccmpdecimalなら0x01、tccmpint32なら0x02、tccmpint64なら0x03、それ以外なら0xff</td> </tr> <tr> <td>予約領域</td> @@ -7108,7 +7128,7 @@ cp -f "$srcpath" "$destpath" </tr> </table> -<p>インデックスはページを探索するためのポインタの論理的な単位です。インデックスは以下の形式で直列化されます。</p> +<p>インデックスは子ページを探索するためのポインタの論理的な単位です。インデックスは以下の形式で直列化されます。</p> <table summary="index format"> <tr> @@ -7183,7 +7203,7 @@ cp -f "$srcpath" "$destpath" <td>データベースタイプ</td> <td class="number">32</td> <td class="number">1</td> -<td>0x03</td> +<td>0x03固定</td> </tr> <tr> <td>追加フラグ</td> @@ -7260,15 +7280,17 @@ cp -f "$srcpath" "$destpath" <td>パディング</td> <td class="number">可変</td> <td class="number">可変</td> -<td>パディング。値サイズが0の時は、先頭バイトの真偽値でレコードの有無を示す</td> +<td>値サイズが0の時は、先頭バイトの真偽値でレコードの有無を示す</td> </tr> </table> +<p>トランザクションログの命名規則やフォーマットはハッシュデータベースのものと同じです。</p> + <h3 id="fileformat_note">注記</h3> -<p>データベースファイルはスパースではないので、通常のファイルと同様に複製等の操作を行うことができます。ハッシュデータベースのファイルもB+木データベースのファイルも実行環境のバイトオーダに依存しない形式なので、バイトオーダの異なる環境にデータベースファイルを移設してもそのままで利用できます。</p> +<p>データベースファイルはスパースではないので、通常のファイルと同様に複製等の操作を行うことができます。またフォーマットも実行環境のバイトオーダに依存しないので、バイトオーダの異なる環境にデータベースファイルを移設してもそのままで利用できます。</p> -<p>なるべくなら、ハッシュデータベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-hash</code>' にしてください。ファイル名の接尾辞は `<code>.tch</code>' にしてください。B+木データベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-btree</code>' にしてください。ファイル名の接尾辞は `<code>.tcb</code>' にしてください。固定長データベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-fixlen</code>' にしてください。ファイル名の接尾辞は `<code>.tcf</code>' にしてください。テーブルデータベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-table</code>' にしてください。ファイル名の接尾辞は `<code>.tct</code>' にしてください。</p> +<p>なるべくなら、ハッシュデータベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-hash</code>' にしてください。ファイル名の接尾辞は `<code>.tch</code>' にしてください。B+木データベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-btree</code>' にしてください。ファイル名の接尾辞は `<code>.tcb</code>' にしてください。固定長データベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-fixed</code>' にしてください。ファイル名の接尾辞は `<code>.tcf</code>' にしてください。テーブルデータベースのファイルをネットワークで配布する際には、MIMEタイプを `<code>application/x-tokyocabinet-table</code>' にしてください。ファイル名の接尾辞は `<code>.tct</code>' にしてください。</p> <p>データベースファイルのマジックデータを `<code>file</code>' コマンドに識別させたい場合は、`<code>magic</code>' ファイルに以下の行を追記してください。</p> @@ -7276,9 +7298,13 @@ cp -f "$srcpath" "$destpath" 0 string ToKyO\ CaBiNeT\n Tokyo Cabinet >14 string x \b (%s) >32 byte 0 \b, Hash +!:mime application/x-tokyocabinet-hash >32 byte 1 \b, B+ tree +!:mime application/x-tokyocabinet-btree >32 byte 2 \b, Fixed-length +!:mime application/x-tokyocabinet-fixed >32 byte 3 \b, Table +!:mime application/x-tokyocabinet-table >33 byte &1 \b, [open] >33 byte &2 \b, [fatal] >34 byte x \b, apow=%d @@ -7288,9 +7314,9 @@ cp -f "$srcpath" "$destpath" >36 byte &4 \b, [bzip] >36 byte &8 \b, [tcbs] >36 byte &16 \b, [excodec] ->40 lelong x \b, bnum=%d ->48 lelong x \b, rnum=%d ->56 lelong x \b, fsiz=%d +>40 lequad x \b, bnum=%lld +>48 lequad x \b, rnum=%lld +>56 lequad x \b, fsiz=%lld </pre> <hr /> doc/spex-ja.html doc/tokyoproducts.pdf doc/tokyoproducts.ppt @@ -24,7 +24,7 @@ To use the abstract database API, include `\fBtcutil.h\fR', `\fBtcadb.h\fR', and .PP Objects whose type is pointer to `\fBTCADB\fR' are used to handle abstract databases. An abstract database object is created with the function `\fBtcadbnew\fR' and is deleted with the function `\fBtcadbdel\fR'. To avoid memory leak, it is important to delete every object when it is no longer in use. .PP -Before operations to store or retrieve records, it is necessary to connect the abstract database object to the concrete one. The function `\fBtcadbopen\fR' is used to open a concrete database and the function `\fBtcadbclose\fR' is used to close the database. To avoid data missing or corruption, it is important to close every database file when it is no longer in use. +Before operations to store or retrieve records, it is necessary to connect the abstract database object to the concrete one. The function `\fBtcadbopen\fR' is used to open a concrete database and the function `\fBtcadbclose\fR' is used to close the database. To avoid data missing or corruption, it is important to close every database instance when it is no longer in use. .SH API .PP @@ -508,6 +508,25 @@ If successful, the return value is true, else, it is false. .RE .RE .PP +The function `tcadboptimize' is used in order to optimize the storage of an abstract database object. +.PP +.RS +.br +\fBbool tcadboptimize(TCADB *\fIadb\fB, const char *\fIparams\fB);\fR +.RS +`\fIadb\fR' specifies the abstract database object. +.RE +.RS +`\fIparams\fR' specifies the string of the tuning parameters, which works as with the tuning of parameters the function `tcadbopen'. If it is `NULL', it is not used. +.RE +.RS +If successful, the return value is true, else, it is false. +.RE +.RS +This function is useful to reduce the size of the database storage with data fragmentation by successive updating. +.RE +.RE +.PP The function `tcadbvanish' is used in order to remove all records of an abstract database object. .PP .RS @@ -588,6 +607,19 @@ Update in the transaction is discarded when it is aborted. The state of the dat .RE .RE .PP +The function `tcadbpath' is used in order to get the file path of an abstract database object. +.PP +.RS +.br +\fBconst char *tcadbpath(TCADB *\fIadb\fB);\fR +.RS +`\fIadb\fR' specifies the abstract database object. +.RE +.RS +The return value is the path of the database file or `NULL' if the object does not connect to any database. "*" stands for on\-memory hash database. "+" stands for on\-memory tree database. +.RE +.RE +.PP The function `tcadbrnum' is used in order to get the number of records of an abstract database object. .PP .RS man/tcadb.3 @@ -5,7 +5,7 @@ tcamgr \- the command line utility of the abstract database API .SH DESCRIPTION .PP -The command `\fBtcamgr\fR' is a utility for test and debugging of the abstract database API and its applications. `\fIname\fR' specifies the name of a database. `\fIkey\fR' specifies the key of a record. `\fIvalue\fR' specifies the value of a record. `\fIfunc\fR' specifies the name of a function. `\fIarg\fR' specifies the arguments of the function. `\fIdest\fR' specifies the path of the destination file. +The command `\fBtcamgr\fR' is a utility for test and debugging of the abstract database API and its applications. `\fIname\fR' specifies the name of a database. `\fIkey\fR' specifies the key of a record. `\fIvalue\fR' specifies the value of a record. `\fIparams\fR' specifies the tuning parameters. `\fIfunc\fR' specifies the name of a function. `\fIarg\fR' specifies the arguments of the function. `\fIdest\fR' specifies the path of the destination file. .PP .RS .br @@ -39,6 +39,11 @@ Print the value of a record. Print keys of all records, separated by line feeds. .RE .br +\fBtcamgr optimize \fIname\fB \fIparams\fB\fR +.RS +Optimize a database file. +.RE +.br \fBtcamgr misc \fR[\fB\-sx\fR]\fB \fR[\fB\-sep \fIchr\fB\fR]\fB \fR[\fB\-px\fR]\fB \fIname\fB \fIfunc\fB \fR[\fB\fIarg\fB...\fR]\fB\fR .RS Call a versatile function for miscellaneous operations. man/tcamgr.1 @@ -145,7 +145,7 @@ The function `tctdbsetcache' is set the caching parameters of a table database o If successful, the return value is true, else, it is false. .RE .RS -Note that the caching parameters should be set before the database is opened. Leaf nodes and non\-leaf nodes are used in column indexes. +Note that the caching parameters should be set before the database is opened. Leaf nodes and non\-leaf nodes are used in column indices. .RE .RE .PP @@ -893,7 +893,7 @@ The function `tctdbsetindex' is used in order to set a column index to a table d If successful, the return value is true, else, it is false. .RE .RS -Note that the setting indexes should be set after the database is opened. +Note that the setting indices should be set after the database is opened. .RE .RE .PP man/tctdb.3 @@ -13,7 +13,7 @@ As for database of B+ tree, records whose keys are duplicated can be stored. Ac .PP As for database of fixed\-length array, records are stored with unique natural numbers. It is impossible to store two or more records with a key overlaps. Moreover, the length of each record is limited by the specified length. Provided operations are the same as ones of hash database. .PP -Table database is also provided as a variant of hash database. Each record is identified by the primary key and has a set of named columns. Although there is no concept of data schema, it is possible to search for records with complex conditions efficiently by using indexes of arbitrary columns. +Table database is also provided as a variant of hash database. Each record is identified by the primary key and has a set of named columns. Although there is no concept of data schema, it is possible to search for records with complex conditions efficiently by using indices of arbitrary columns. .PP Tokyo Cabinet is written in the C language, and provided as API of C, Perl, Ruby, Java, and Lua. Tokyo Cabinet is available on platforms which have API conforming to C99 and POSIX. Tokyo Cabinet is a free software licensed under the GNU Lesser General Public License. @@ -78,7 +78,7 @@ Table database does not express simple key/value structure but expresses a struc .PP Table database supports query functions with not only the primary key but also with conditions about arbitrary columns. Each column condition is composed of the name of a column and a condition expression. Operators of full matching, forward matching, regular expression matching, and so on are provided for the string type. Operators of full matching, range matching and so on are provided for the number type. A query can contain multiple conditions. The order of the result set can be specified as the ascending or descending order of strings or numbers. .PP -You can create indexes for arbitrary columns to improve performance of search and sorting. Although columns do not have data types, indexes have types for strings or numbers. The query optimizer uses indexes in suitable way according to each query. Indexes are implemented as different files of B+ tree database. +You can create indices for arbitrary columns to improve performance of search and sorting. Although columns do not have data types, indices have types for strings or numbers. The query optimizer uses indices in suitable way according to each query. Indices are implemented as different files of B+ tree database. .SH PRACTICAL FUNCTIONALITY .PP man/tokyocabinet.3 @@ -872,6 +872,7 @@ bool tcadbsync(TCADB *adb){ tcmdbcutfront(adb->mdb, 1); } } + adb->capcnt = 0; break; case ADBONDB: if(adb->capnum > 0 && tcndbrnum(adb->ndb) > adb->capnum) @@ -881,6 +882,7 @@ bool tcadbsync(TCADB *adb){ tcndbcutfringe(adb->ndb, 0x100); } } + adb->capcnt = 0; break; case ADBOHDB: if(!tchdbsync(adb->hdb)) err = true; @@ -902,6 +904,117 @@ bool tcadbsync(TCADB *adb){ } +/* Optimize the storage of an abstract database object. */ +bool tcadboptimize(TCADB *adb, const char *params){ + assert(adb); + TCLIST *elems = params ? tcstrsplit(params, "#") : tclistnew(); + int64_t bnum = -1; + int64_t capnum = -1; + int64_t capsiz = -1; + int8_t apow = -1; + int8_t fpow = -1; + bool tdefault = true; + bool tlmode = false; + bool tdmode = false; + bool tbmode = false; + bool ttmode = false; + int32_t lmemb = -1; + int32_t nmemb = -1; + int32_t width = -1; + int64_t limsiz = -1; + int ln = TCLISTNUM(elems); + for(int i = 0; i < ln; i++){ + const char *elem = TCLISTVALPTR(elems, i); + char *pv = strchr(elem, '='); + if(!pv) continue; + *(pv++) = '\0'; + if(!tcstricmp(elem, "bnum")){ + bnum = tcatoix(pv); + } else if(!tcstricmp(elem, "capnum")){ + capnum = tcatoix(pv); + } else if(!tcstricmp(elem, "capsiz")){ + capsiz = tcatoix(pv); + } else if(!tcstricmp(elem, "apow")){ + apow = tcatoix(pv); + } else if(!tcstricmp(elem, "fpow")){ + fpow = tcatoix(pv); + } else if(!tcstricmp(elem, "opts")){ + tdefault = false; + if(strchr(pv, 'l') || strchr(pv, 'L')) tlmode = true; + if(strchr(pv, 'd') || strchr(pv, 'D')) tdmode = true; + if(strchr(pv, 'b') || strchr(pv, 'B')) tbmode = true; + if(strchr(pv, 't') || strchr(pv, 'T')) ttmode = true; + } else if(!tcstricmp(elem, "lmemb")){ + lmemb = tcatoix(pv); + } else if(!tcstricmp(elem, "nmemb")){ + nmemb = tcatoix(pv); + } else if(!tcstricmp(elem, "width")){ + width = tcatoix(pv); + } else if(!tcstricmp(elem, "limsiz")){ + limsiz = tcatoix(pv); + } + } + tclistdel(elems); + bool err = false; + int opts; + switch(adb->omode){ + case ADBOMDB: + adb->capnum = capnum; + adb->capsiz = capsiz; + tcadbsync(adb); + break; + case ADBONDB: + adb->capnum = capnum; + adb->capsiz = capsiz; + tcadbsync(adb); + break; + case ADBOHDB: + opts = 0; + if(tdefault){ + opts = UINT8_MAX; + } else { + if(tlmode) opts |= HDBTLARGE; + if(tdmode) opts |= HDBTDEFLATE; + if(tbmode) opts |= HDBTBZIP; + if(ttmode) opts |= HDBTTCBS; + } + if(!tchdboptimize(adb->hdb, bnum, apow, fpow, opts)) err = true; + break; + case ADBOBDB: + opts = 0; + if(tdefault){ + opts = UINT8_MAX; + } else { + if(tlmode) opts |= BDBTLARGE; + if(tdmode) opts |= BDBTDEFLATE; + if(tbmode) opts |= BDBTBZIP; + if(ttmode) opts |= BDBTTCBS; + } + if(!tcbdboptimize(adb->bdb, lmemb, nmemb, bnum, apow, fpow, opts)) err = true; + break; + case ADBOFDB: + if(!tcfdboptimize(adb->fdb, width, limsiz)) err = true; + break; + case ADBOTDB: + opts = 0; + if(tdefault){ + opts = UINT8_MAX; + } else { + if(tlmode) opts |= TDBTLARGE; + if(tdmode) opts |= TDBTDEFLATE; + if(tbmode) opts |= TDBTBZIP; + if(ttmode) opts |= TDBTTCBS; + } + if(!tctdboptimize(adb->tdb, bnum, apow, fpow, opts)) err = true; + break; + default: + err = true; + break; + } + return !err; +} + + /* Remove all records of an abstract database object. */ bool tcadbvanish(TCADB *adb){ assert(adb); @@ -1083,6 +1196,37 @@ bool tcadbtranabort(TCADB *adb){ } +/* Get the file path of an abstract database object. */ +const char *tcadbpath(TCADB *adb){ + assert(adb); + const char *rv; + switch(adb->omode){ + case ADBOMDB: + rv = "*"; + break; + case ADBONDB: + rv = "+"; + break; + case ADBOHDB: + rv = tchdbpath(adb->hdb); + break; + case ADBOBDB: + rv = tcbdbpath(adb->bdb); + break; + case ADBOFDB: + rv = tcfdbpath(adb->fdb); + break; + case ADBOTDB: + rv = tctdbpath(adb->tdb); + break; + default: + rv = NULL; + break; + } + return rv; +} + + /* Get the number of records of an abstract database object. */ uint64_t tcadbrnum(TCADB *adb){ assert(adb); tcadb.c @@ -324,6 +324,16 @@ double tcadbadddouble(TCADB *adb, const void *kbuf, int ksiz, double num); bool tcadbsync(TCADB *adb); +/* Optimize the storage of an abstract database object. + `adb' specifies the abstract database object. + `params' specifies the string of the tuning parameters, which works as with the tuning + of parameters the function `tcadbopen'. If it is `NULL', it is not used. + If successful, the return value is true, else, it is false. + This function is useful to reduce the size of the database storage with data fragmentation by + successive updating. */ +bool tcadboptimize(TCADB *adb, const char *params); + + /* Remove all records of an abstract database object. `adb' specifies the abstract database object. If successful, the return value is true, else, it is false. */ @@ -368,6 +378,14 @@ bool tcadbtrancommit(TCADB *adb); bool tcadbtranabort(TCADB *adb); +/* Get the file path of an abstract database object. + `adb' specifies the abstract database object. + The return value is the path of the database file or `NULL' if the object does not connect to + any database. "*" stands for on-memory hash database. "+" stands for on-memory tree + database. */ +const char *tcadbpath(TCADB *adb); + + /* Get the number of records of an abstract database object. `adb' specifies the abstract database object. The return value is the number of records or 0 if the object does not connect to any database tcadb.h @@ -39,6 +39,7 @@ static int runput(int argc, char **argv); static int runout(int argc, char **argv); static int runget(int argc, char **argv); static int runlist(int argc, char **argv); +static int runoptimize(int argc, char **argv); static int runmisc(int argc, char **argv); static int runmap(int argc, char **argv); static int runversion(int argc, char **argv); @@ -49,6 +50,7 @@ static int procput(const char *name, const char *kbuf, int ksiz, const char *vbu static int procout(const char *name, const char *kbuf, int ksiz); static int procget(const char *name, const char *kbuf, int ksiz, int sep, bool px, bool pz); static int proclist(const char *name, int sep, int max, bool pv, bool px, const char *fmstr); +static int procoptimize(const char *name, const char *params); static int procmisc(const char *name, const char *func, const TCLIST *args, int sep, bool px); static int procmap(const char *name, const char *dest, const char *fmstr); static int procversion(void); @@ -71,6 +73,8 @@ int main(int argc, char **argv){ rv = runget(argc, argv); } else if(!strcmp(argv[1], "list")){ rv = runlist(argc, argv); + } else if(!strcmp(argv[1], "optimize")){ + rv = runoptimize(argc, argv); } else if(!strcmp(argv[1], "misc")){ rv = runmisc(argc, argv); } else if(!strcmp(argv[1], "map")){ @@ -95,6 +99,7 @@ static void usage(void){ fprintf(stderr, " %s out [-sx] [-sep chr] name key\n", g_progname); fprintf(stderr, " %s get [-sx] [-sep chr] [-px] [-pz] name key\n", g_progname); fprintf(stderr, " %s list [-sep chr] [-m num] [-pv] [-px] [-fm str] name\n", g_progname); + fprintf(stderr, " %s optimize name [params]\n", g_progname); fprintf(stderr, " %s misc [-sx] [-sep chr] [-px] name func [arg...]\n", g_progname); fprintf(stderr, " %s map [-fm str] name dest\n", g_progname); fprintf(stderr, " %s version\n", g_progname); @@ -126,7 +131,8 @@ static char *strtozsv(const char *str, int sep, int *sp){ /* print error information */ static void printerr(TCADB *adb){ - fprintf(stderr, "%s: error\n", g_progname); + const char *path = tcadbpath(adb); + fprintf(stderr, "%s: %s: error\n", g_progname, path ? path : "-"); } @@ -389,6 +395,27 @@ static int runlist(int argc, char **argv){ } +/* parse arguments of optimize command */ +static int runoptimize(int argc, char **argv){ + char *name = NULL; + char *params = NULL; + for(int i = 2; i < argc; i++){ + if(!name && argv[i][0] == '-'){ + usage(); + } else if(!name){ + name = argv[i]; + } else if(!params){ + params = argv[i]; + } else { + usage(); + } + } + if(!name) usage(); + int rv = procoptimize(name, params); + return rv; +} + + /* parse arguments of misc command */ static int runmisc(int argc, char **argv){ char *name = NULL; @@ -498,6 +525,9 @@ static int procinform(const char *name){ return 1; } bool err = false; + const char *path = tcadbpath(adb); + if(!path) path = "(unknown)"; + printf("path: %s\n", path); const char *type = "(unknown)"; switch(tcadbomode(adb)){ case ADBOVOID: type = "not opened"; break; @@ -506,6 +536,7 @@ static int procinform(const char *name){ case ADBOHDB: type = "hash database"; break; case ADBOBDB: type = "B+ tree database"; break; case ADBOFDB: type = "fixed-length database"; break; + case ADBOTDB: type = "table database"; break; } printf("database type: %s\n", type); printf("record number: %llu\n", (unsigned long long)tcadbrnum(adb)); @@ -680,6 +711,28 @@ static int proclist(const char *name, int sep, int max, bool pv, bool px, const } +/* perform optimize command */ +static int procoptimize(const char *name, const char *params){ + TCADB *adb = tcadbnew(); + if(!tcadbopen(adb, name)){ + printerr(adb); + tcadbdel(adb); + return 1; + } + bool err = false; + if(!tcadboptimize(adb, params)){ + printerr(adb); + err = true; + } + if(!tcadbclose(adb)){ + if(!err) printerr(adb); + err = true; + } + tcadbdel(adb); + return err ? 1 : 0; +} + + /* perform misc command */ static int procmisc(const char *name, const char *func, const TCLIST *args, int sep, bool px){ TCADB *adb = tcadbnew(); tcamgr.c @@ -116,7 +116,8 @@ static void iputchar(int c){ /* print error message of abstract database */ static void eprint(TCADB *adb, const char *func){ - fprintf(stderr, "%s: %s: error\n", g_progname, func); + const char *path = tcadbpath(adb); + fprintf(stderr, "%s: %s: %s: error\n", g_progname, path ? path : "-", func); } @@ -680,6 +681,10 @@ static int procmisc(const char *name, int rnum){ eprint(adb, "tcadbsync"); err = true; } + if(myrand(10) == 0 && !tcadboptimize(adb, NULL)){ + eprint(adb, "tcadboptimize"); + err = true; + } if(!tcadbvanish(adb)){ eprint(adb, "tcadbvanish"); err = true; tcatest.c @@ -62,7 +62,7 @@ typedef struct { // type of structure for a page index typedef struct { // type of structure for a node page uint64_t id; // ID number of the node uint64_t heir; // ID of the child before the first index - TCPTRLIST *idxs; // list of indexes + TCPTRLIST *idxs; // list of indices bool dirty; // whether to be written back bool dead; // whether to be removed } BDBNODE; @@ -806,11 +806,17 @@ bool tcbdbcopy(TCBDB *bdb, const char *path){ /* Begin the transaction of a B+ tree database object. */ bool tcbdbtranbegin(TCBDB *bdb){ assert(bdb); - if(!BDBLOCKMETHOD(bdb, true)) return false; - if(!bdb->open || !bdb->wmode || bdb->tran){ - tcbdbsetecode(bdb, TCEINVALID, __FILE__, __LINE__, __func__); + for(double wsec = 1.0 / sysconf(_SC_CLK_TCK); true; wsec *= 2){ + if(!BDBLOCKMETHOD(bdb, true)) return false; + if(!bdb->open || !bdb->wmode){ + tcbdbsetecode(bdb, TCEINVALID, __FILE__, __LINE__, __func__); + BDBUNLOCKMETHOD(bdb); + return false; + } + if(!bdb->tran) break; BDBUNLOCKMETHOD(bdb); - return false; + if(wsec > 1.0) wsec = 1.0; + tcsleep(wsec); } if(!tcbdbmemsync(bdb, false)){ BDBUNLOCKMETHOD(bdb); @@ -3570,7 +3576,10 @@ static bool tcbdbcurjumpimpl(BDBCUR *cur, const char *kbuf, int ksiz, bool forwa } else { rv = bdb->cmp(kbuf, ksiz, dbuf, rec->ksiz, bdb->cmpop); } - if(rv < 0) return true; + if(rv < 0){ + cur->vidx = 0; + return true; + } cur->vidx = rec->rest ? TCLISTNUM(rec->rest) : 0; return tcbdbcurnextimpl(cur); } @@ -4012,7 +4021,7 @@ void tcbdbprintleaf(TCBDB *bdb, BDBLEAF *leaf){ } -/* Print indexes of a node object into the debugging output. +/* Print indices of a node object into the debugging output. `bdb' specifies the B+ tree database object. `node' specifies the node object. */ void tcbdbprintnode(TCBDB *bdb, BDBNODE *node){ tcbdb.c @@ -1328,6 +1328,33 @@ static void *threadwicked(void *targ){ break; default: if(id == 0) iputchar('@'); + if(tcbdbtranbegin(bdb)){ + if(myrand(2) == 0){ + if(!tcbdbput(bdb, kbuf, ksiz, vbuf, vsiz)){ + eprint(bdb, "tcbdbput"); + err = true; + } + } else { + if(!tcbdbout(bdb, kbuf, ksiz) && tcbdbecode(bdb) != TCENOREC){ + eprint(bdb, "tcbdbout"); + err = true; + } + } + if(nc && myrand(2) == 0){ + if(!tcbdbtrancommit(bdb)){ + eprint(bdb, "tcbdbtrancommit"); + err = true; + } + } else { + if(!tcbdbtranabort(bdb)){ + eprint(bdb, "tcbdbtranabort"); + err = true; + } + } + } else { + eprint(bdb, "tcbdbtranbegin"); + err = true; + } if(myrand(1000) == 0){ if(!tcbdbforeach(bdb, iterfunc, NULL)){ eprint(bdb, "tcbdbforeach"); @@ -1341,7 +1368,7 @@ static void *threadwicked(void *targ){ if(id == 0){ if(i % 50 == 0) iprintf(" (%08d)\n", i); if(id == 0 && i == rnum / 4){ - if(!tcbdboptimize(bdb, -1, -1, -1, -1, -1, -1)){ + if(!tcbdboptimize(bdb, -1, -1, -1, -1, -1, -1) && tcbdbecode(bdb) != TCEINVALID){ eprint(bdb, "tcbdboptimize"); err = true; } tcbmttest.c @@ -72,10 +72,6 @@ typedef struct { // type of structure for a duplication ((TC_fdb)->mmtx ? tcfdblockallrecords((TC_fdb), (TC_wr)) : true) #define FDBUNLOCKALLRECORDS(TC_fdb) \ ((TC_fdb)->mmtx ? tcfdbunlockallrecords(TC_fdb) : true) -#define FDBLOCKTRAN(TC_fdb) \ - ((TC_fdb)->mmtx ? tcfdblocktran(TC_fdb) : true) -#define FDBUNLOCKTRAN(TC_fdb) \ - ((TC_fdb)->mmtx ? tcfdbunlocktran(TC_fdb) : true) #define FDBLOCKWAL(TC_fdb) \ ((TC_fdb)->mmtx ? tcfdblockwal(TC_fdb) : true) #define FDBUNLOCKWAL(TC_fdb) \ @@ -115,8 +111,6 @@ static bool tcfdblockrecord(TCFDB *fdb, bool wr, uint64_t id); static bool tcfdbunlockrecord(TCFDB *fdb, uint64_t id); static bool tcfdblockallrecords(TCFDB *fdb, bool wr); static bool tcfdbunlockallrecords(TCFDB *fdb); -static bool tcfdblocktran(TCFDB *fdb); -static bool tcfdbunlocktran(TCFDB *fdb); static bool tcfdblockwal(TCFDB *fdb); static bool tcfdbunlockwal(TCFDB *fdb); @@ -881,7 +875,7 @@ bool tcfdbvanish(TCFDB *fdb){ bool tcfdbcopy(TCFDB *fdb, const char *path){ assert(fdb && path); if(!FDBLOCKMETHOD(fdb, false)) return false; - if(fdb->fd < 0 || !(fdb->omode & FDBOWRITER)){ + if(fdb->fd < 0){ tcfdbsetecode(fdb, TCEINVALID, __FILE__, __LINE__, __func__); FDBUNLOCKMETHOD(fdb); return false; @@ -901,11 +895,17 @@ bool tcfdbcopy(TCFDB *fdb, const char *path){ /* Begin the transaction of a fixed-length database object. */ bool tcfdbtranbegin(TCFDB *fdb){ assert(fdb); - if(!FDBLOCKMETHOD(fdb, true)) return false; - if(fdb->fd < 0 || !(fdb->omode & FDBOWRITER) || fdb->fatal || fdb->tran){ - tcfdbsetecode(fdb, TCEINVALID, __FILE__, __LINE__, __func__); + for(double wsec = 1.0 / sysconf(_SC_CLK_TCK); true; wsec *= 2){ + if(!FDBLOCKMETHOD(fdb, true)) return false; + if(fdb->fd < 0 || !(fdb->omode & FDBOWRITER) || fdb->fatal){ + tcfdbsetecode(fdb, TCEINVALID, __FILE__, __LINE__, __func__); + FDBUNLOCKMETHOD(fdb); + return false; + } + if(!fdb->tran) break; FDBUNLOCKMETHOD(fdb); - return false; + if(wsec > 1.0) wsec = 1.0; + tcsleep(wsec); } fdb->flags &= ~FDBFOPEN; if(!tcfdbmemsync(fdb, false)){ @@ -938,10 +938,6 @@ bool tcfdbtranbegin(TCFDB *fdb){ FDBUNLOCKMETHOD(fdb); return false; } - if(!FDBLOCKTRAN(fdb)){ - FDBUNLOCKMETHOD(fdb); - return false; - } fdb->tran = true; FDBUNLOCKMETHOD(fdb); return true; @@ -964,7 +960,6 @@ bool tcfdbtrancommit(TCFDB *fdb){ err = true; } fdb->tran = false; - FDBUNLOCKTRAN(fdb); FDBUNLOCKMETHOD(fdb); return !err; } @@ -993,7 +988,6 @@ bool tcfdbtranabort(TCFDB *fdb){ tcfdbloadmeta(fdb, hbuf); } fdb->tran = false; - FDBUNLOCKTRAN(fdb); FDBUNLOCKMETHOD(fdb); return !err; } @@ -1140,7 +1134,7 @@ uint64_t tcfdbmax(TCFDB *fdb){ /* Get the width of the value of each record of a fixed-length database object. */ -uint64_t tcfdbwidth(TCFDB *fdb){ +uint32_t tcfdbwidth(TCFDB *fdb){ assert(fdb); if(fdb->fd < 0){ tcfdbsetecode(fdb, TCEINVALID, __FILE__, __LINE__, __func__); @@ -1808,7 +1802,6 @@ static bool tcfdbcloseimpl(TCFDB *fdb){ if(fdb->tran){ if(!tcfdbwalrestore(fdb, fdb->path)) err = true; fdb->tran = false; - FDBUNLOCKTRAN(fdb); } if(fdb->walfd >= 0){ if(close(fdb->walfd) == -1){ @@ -2561,34 +2554,6 @@ static bool tcfdbunlockallrecords(TCFDB *fdb){ } -/* Lock the transaction of the fixed-length database object. - `fdb' specifies the fixed-length database object. - If successful, the return value is true, else, it is false. */ -static bool tcfdblocktran(TCFDB *fdb){ - assert(fdb); - if(pthread_mutex_lock(fdb->tmtx) != 0){ - tcfdbsetecode(fdb, TCETHREAD, __FILE__, __LINE__, __func__); - return false; - } - TCTESTYIELD(); - return true; -} - - -/* Unlock the transaction of the fixed-length database object. - `fdb' specifies the fixed-length database object. - If successful, the return value is true, else, it is false. */ -static bool tcfdbunlocktran(TCFDB *fdb){ - assert(fdb); - if(pthread_mutex_unlock(fdb->tmtx) != 0){ - tcfdbsetecode(fdb, TCETHREAD, __FILE__, __LINE__, __func__); - return false; - } - TCTESTYIELD(); - return true; -} - - /* Lock the write ahead logging file of the fixed-length database object. `fdb' specifies the fixed-length database object. If successful, the return value is true, else, it is false. */ tcfdb.c @@ -720,7 +720,7 @@ uint64_t tcfdbmax(TCFDB *fdb); `fdb' specifies the fixed-length database object. The return value is the width of the value of each record or 0 if the object does not connect to any database file. */ -uint64_t tcfdbwidth(TCFDB *fdb); +uint32_t tcfdbwidth(TCFDB *fdb); /* Get the limit file size of a fixed-length database object. tcfdb.h @@ -679,7 +679,7 @@ static int procwicked(const char *path, int tnum, int rnum, int omode, bool nc){ char *rbuf = tcfdbget2(fdb, kbuf, ksiz, &rsiz); if(vbuf){ iputchar('.'); - if(vsiz > RECBUFSIZ) vsiz = RECBUFSIZ; + if(vsiz > tcfdbwidth(fdb)) vsiz = tcfdbwidth(fdb); if(!rbuf){ eprint(fdb, "tcfdbget"); err = true; @@ -1023,6 +1023,33 @@ static void *threadwicked(void *targ){ break; default: if(id == 0) iputchar('@'); + if(tcfdbtranbegin(fdb)){ + if(myrand(2) == 0){ + if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){ + eprint(fdb, "tcfdbput"); + err = true; + } + } else { + if(!tcfdbout2(fdb, kbuf, ksiz) && tcfdbecode(fdb) != TCENOREC){ + eprint(fdb, "tcfdbout"); + err = true; + } + } + if(nc && myrand(2) == 0){ + if(!tcfdbtrancommit(fdb)){ + eprint(fdb, "tcfdbtrancommit"); + err = true; + } + } else { + if(!tcfdbtranabort(fdb)){ + eprint(fdb, "tcfdbtranabort"); + err = true; + } + } + } else { + eprint(fdb, "tcfdbtranbegin"); + err = true; + } if(myrand(1000) == 0){ if(!tcfdbforeach(fdb, iterfunc, NULL)){ eprint(fdb, "tcfdbforeach"); @@ -1036,7 +1063,7 @@ static void *threadwicked(void *targ){ if(id == 0){ if(i % 50 == 0) iprintf(" (%08d)\n", i); if(id == 0 && i == rnum / 4){ - if(!tcfdboptimize(fdb, RECBUFSIZ, -1)){ + if(!tcfdboptimize(fdb, RECBUFSIZ, -1) && tcfdbecode(fdb) != TCEINVALID){ eprint(fdb, "tcfdboptimize"); err = true; } tcfmttest.c @@ -1603,7 +1603,7 @@ static int procwicked(const char *path, int rnum, bool mt, int omode){ iputchar('+'); int vsiz; const char *vbuf = tcmapiterval(kbuf, &vsiz); - if(vsiz > RECBUFSIZ) vsiz = RECBUFSIZ; + if(vsiz > tcfdbwidth(fdb)) vsiz = tcfdbwidth(fdb); int rsiz; char *rbuf = tcfdbget2(fdb, kbuf, ksiz, &rsiz); if(!rbuf){ tcftest.c @@ -111,10 +111,6 @@ typedef struct { // type of structure for a duplication ((TC_hdb)->mmtx ? tchdblockdb(TC_hdb) : true) #define HDBUNLOCKDB(TC_hdb) \ ((TC_hdb)->mmtx ? tchdbunlockdb(TC_hdb) : true) -#define HDBLOCKTRAN(TC_hdb) \ - ((TC_hdb)->mmtx ? tchdblocktran(TC_hdb) : true) -#define HDBUNLOCKTRAN(TC_hdb) \ - ((TC_hdb)->mmtx ? tchdbunlocktran(TC_hdb) : true) #define HDBLOCKWAL(TC_hdb) \ ((TC_hdb)->mmtx ? tchdblockwal(TC_hdb) : true) #define HDBUNLOCKWAL(TC_hdb) \ @@ -187,8 +183,6 @@ static bool tchdblockallrecords(TCHDB *hdb, bool wr); static bool tchdbunlockallrecords(TCHDB *hdb); static bool tchdblockdb(TCHDB *hdb); static bool tchdbunlockdb(TCHDB *hdb); -static bool tchdblocktran(TCHDB *hdb); -static bool tchdbunlocktran(TCHDB *hdb); static bool tchdblockwal(TCHDB *hdb); static bool tchdbunlockwal(TCHDB *hdb); @@ -1019,7 +1013,7 @@ bool tchdbvanish(TCHDB *hdb){ bool tchdbcopy(TCHDB *hdb, const char *path){ assert(hdb && path); if(!HDBLOCKMETHOD(hdb, false)) return false; - if(hdb->fd < 0 || !(hdb->omode & HDBOWRITER)){ + if(hdb->fd < 0){ tchdbsetecode(hdb, TCEINVALID, __FILE__, __LINE__, __func__); HDBUNLOCKMETHOD(hdb); return false; @@ -1043,11 +1037,17 @@ bool tchdbcopy(TCHDB *hdb, const char *path){ /* Begin the transaction of a hash database object. */ bool tchdbtranbegin(TCHDB *hdb){ assert(hdb); - if(!HDBLOCKMETHOD(hdb, true)) return false; - if(hdb->fd < 0 || !(hdb->omode & HDBOWRITER) || hdb->fatal || hdb->tran){ - tchdbsetecode(hdb, TCEINVALID, __FILE__, __LINE__, __func__); + for(double wsec = 1.0 / sysconf(_SC_CLK_TCK); true; wsec *= 2){ + if(!HDBLOCKMETHOD(hdb, true)) return false; + if(hdb->fd < 0 || !(hdb->omode & HDBOWRITER) || hdb->fatal){ + tchdbsetecode(hdb, TCEINVALID, __FILE__, __LINE__, __func__); + HDBUNLOCKMETHOD(hdb); + return false; + } + if(!hdb->tran) break; HDBUNLOCKMETHOD(hdb); - return false; + if(wsec > 1.0) wsec = 1.0; + tcsleep(wsec); } if(hdb->async && !tchdbflushdrp(hdb)){ HDBUNLOCKMETHOD(hdb); @@ -1084,10 +1084,6 @@ bool tchdbtranbegin(TCHDB *hdb){ HDBUNLOCKMETHOD(hdb); return false; } - if(!HDBLOCKTRAN(hdb)){ - HDBUNLOCKMETHOD(hdb); - return false; - } hdb->tran = true; HDBUNLOCKMETHOD(hdb); return true; @@ -1111,7 +1107,6 @@ bool tchdbtrancommit(TCHDB *hdb){ err = true; } hdb->tran = false; - HDBUNLOCKTRAN(hdb); HDBUNLOCKMETHOD(hdb); return !err; } @@ -1140,11 +1135,11 @@ bool tchdbtranabort(TCHDB *hdb){ } else { tchdbloadmeta(hdb, hbuf); } + hdb->iter = 0; hdb->xfsiz = 0; hdb->fbpnum = 0; if(hdb->recc) tcmdbvanish(hdb->recc); hdb->tran = false; - HDBUNLOCKTRAN(hdb); HDBUNLOCKMETHOD(hdb); return !err; } @@ -1648,7 +1643,6 @@ bool tchdbtranvoid(TCHDB *hdb){ return false; } hdb->tran = false; - HDBUNLOCKTRAN(hdb); HDBUNLOCKMETHOD(hdb); return true; } @@ -3237,7 +3231,6 @@ static bool tchdbcloseimpl(TCHDB *hdb){ if(hdb->tran){ if(!tchdbwalrestore(hdb, hdb->path)) err = true; hdb->tran = false; - HDBUNLOCKTRAN(hdb); } if(hdb->walfd >= 0){ if(close(hdb->walfd) == -1){ @@ -4521,34 +4514,6 @@ static bool tchdbunlockdb(TCHDB *hdb){ } -/* Lock the transaction of the hash database object. - `hdb' specifies the hash database object. - If successful, the return value is true, else, it is false. */ -static bool tchdblocktran(TCHDB *hdb){ - assert(hdb); - if(pthread_mutex_lock(hdb->tmtx) != 0){ - tchdbsetecode(hdb, TCETHREAD, __FILE__, __LINE__, __func__); - return false; - } - TCTESTYIELD(); - return true; -} - - -/* Unlock the transaction of the hash database object. - `hdb' specifies the hash database object. - If successful, the return value is true, else, it is false. */ -static bool tchdbunlocktran(TCHDB *hdb){ - assert(hdb); - if(pthread_mutex_unlock(hdb->tmtx) != 0){ - tchdbsetecode(hdb, TCETHREAD, __FILE__, __LINE__, __func__); - return false; - } - TCTESTYIELD(); - return true; -} - - /* Lock the write ahead logging file of the hash database object. `hdb' specifies the hash database object. If successful, the return value is true, else, it is false. */ tchdb.c @@ -1377,6 +1377,33 @@ static void *threadwicked(void *targ){ break; default: if(id == 0) iputchar('@'); + if(tchdbtranbegin(hdb)){ + if(myrand(2) == 0){ + if(!tchdbput(hdb, kbuf, ksiz, vbuf, vsiz)){ + eprint(hdb, "tchdbput"); + err = true; + } + } else { + if(!tchdbout(hdb, kbuf, ksiz) && tchdbecode(hdb) != TCENOREC){ + eprint(hdb, "tchdbout"); + err = true; + } + } + if(nc && myrand(2) == 0){ + if(!tchdbtrancommit(hdb)){ + eprint(hdb, "tchdbtrancommit"); + err = true; + } + } else { + if(!tchdbtranabort(hdb)){ + eprint(hdb, "tchdbtranabort"); + err = true; + } + } + } else { + eprint(hdb, "tchdbtranbegin"); + err = true; + } if(myrand(1000) == 0){ if(!tchdbforeach(hdb, iterfunc, NULL)){ eprint(hdb, "tchdbforeach"); @@ -1390,7 +1417,7 @@ static void *threadwicked(void *targ){ if(id == 0){ if(i % 50 == 0) iprintf(" (%08d)\n", i); if(id == 0 && i == rnum / 4){ - if(!tchdboptimize(hdb, rnum / 50, -1, -1, -1)){ + if(!tchdboptimize(hdb, rnum / 50, -1, -1, -1) && tchdbecode(hdb) != TCEINVALID){ eprint(hdb, "tchdboptimize"); err = true; } @@ -1595,7 +1622,7 @@ static void *threadrace(void *targ){ if(myrand(mid) == 0){ iprintf("[o]"); if(!tchdboptimize(hdb, myrand(rnum) + 1, myrand(10), myrand(10), 0)){ - eprint(hdb, "tchdbvanish"); + eprint(hdb, "tchdboptimize"); err = true; } } tchmttest.c @@ -586,7 +586,7 @@ bool tctdbvanish(TCTDB *tdb){ bool tctdbcopy(TCTDB *tdb, const char *path){ assert(tdb && path); if(!TDBLOCKMETHOD(tdb, false)) return false; - if(!tdb->open || !tdb->wmode || tdb->tran){ + if(!tdb->open){ tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__); TDBUNLOCKMETHOD(tdb); return false; @@ -601,16 +601,23 @@ bool tctdbcopy(TCTDB *tdb, const char *path){ /* Begin the transaction of a table database object. */ bool tctdbtranbegin(TCTDB *tdb){ assert(tdb); - if(!TDBLOCKMETHOD(tdb, true)) return false; - if(!tdb->open || !tdb->wmode || tdb->tran){ - tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__); + for(double wsec = 1.0 / sysconf(_SC_CLK_TCK); true; wsec *= 2){ + if(!TDBLOCKMETHOD(tdb, true)) return false; + if(!tdb->open || !tdb->wmode){ + tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__); + TDBUNLOCKMETHOD(tdb); + return false; + } + if(!tdb->tran) break; TDBUNLOCKMETHOD(tdb); - return false; + if(wsec > 1.0) wsec = 1.0; + tcsleep(wsec); } if(!tctdbtranbeginimpl(tdb)){ TDBUNLOCKMETHOD(tdb); return false; } + tdb->tran = true; TDBUNLOCKMETHOD(tdb); return true; } @@ -625,12 +632,11 @@ bool tctdbtrancommit(TCTDB *tdb){ TDBUNLOCKMETHOD(tdb); return false; } - if(!tctdbtrancommitimpl(tdb)){ - TDBUNLOCKMETHOD(tdb); - return false; - } + tdb->tran = false; + bool err = false; + if(!tctdbtrancommitimpl(tdb)) err = true; TDBUNLOCKMETHOD(tdb); - return true; + return !err; } @@ -643,12 +649,11 @@ bool tctdbtranabort(TCTDB *tdb){ TDBUNLOCKMETHOD(tdb); return false; } - if(!tctdbtranabortimpl(tdb)){ - TDBUNLOCKMETHOD(tdb); - return false; - } + tdb->tran = false; + bool err = false; + if(!tctdbtranabortimpl(tdb)) err = true; TDBUNLOCKMETHOD(tdb); - return true; + return !err; } @@ -1060,7 +1065,7 @@ uint64_t tctdbbnumused(TCTDB *tdb){ } -/* Get the number of column indexes of a table database object. */ +/* Get the number of column indices of a table database object. */ int tctdbinum(TCTDB *tdb){ assert(tdb); if(!tdb->open){ @@ -1729,7 +1734,6 @@ static bool tctdbtranbeginimpl(TCTDB *tdb){ break; } } - tdb->tran = true; return !err; } @@ -1756,7 +1760,6 @@ static bool tctdbtrancommitimpl(TCTDB *tdb){ break; } } - tdb->tran = false; return !err; } @@ -1782,7 +1785,6 @@ static bool tctdbtranabortimpl(TCTDB *tdb){ break; } } - tdb->tran = false; return !err; } @@ -2830,7 +2832,7 @@ static bool tctdbqryallcondmatch(TDBQRY *qry, const char *pkbuf, int pksiz){ break; } } else { - if(!cond->sign){ + if(cond->sign){ ok = false; break; } @@ -3174,7 +3176,7 @@ static int tdbcmpsortkeynumdesc(const TDBSORTKEY *a, const TDBSORTKEY *b){ } -/* Add a record into indexes of a table database object. +/* Add a record into indices of a table database object. `tdb' specifies the table database object. `pkbuf' specifies the pointer to the region of the primary key. `pksiz' specifies the size of the region of the primary key. @@ -3222,7 +3224,7 @@ static bool tctdbidxput(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){ } -/* Remove a record from indexes of a table database object. +/* Remove a record from indices of a table database object. `tdb' specifies the table database object. `pkbuf' specifies the pointer to the region of the primary key. `pksiz' specifies the size of the region of the primary key. tctdb.c @@ -58,8 +58,8 @@ typedef struct { /* type of structure for a table databa uint8_t opts; /* options */ int32_t lcnum; /* max number of cached leaves */ int32_t ncnum; /* max number of cached nodes */ - TDBIDX *idxs; /* column indexes */ - int inum; /* number of column indexes */ + TDBIDX *idxs; /* column indices */ + int inum; /* number of column indices */ bool tran; /* whether in the transaction */ } TCTDB; @@ -229,7 +229,7 @@ bool tctdbtune(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts) the default value is specified. The default value is 512. If successful, the return value is true, else, it is false. Note that the caching parameters should be set before the database is opened. Leaf nodes and - non-leaf nodes are used in column indexes. */ + non-leaf nodes are used in column indices. */ bool tctdbsetcache(TCTDB *tdb, int32_t rcnum, int32_t lcnum, int32_t ncnum); @@ -616,7 +616,7 @@ uint64_t tctdbfsiz(TCTDB *tdb); removed. If `TDBITKEEP' is added by bitwise-or and the index exists, this function merely returns failure. If successful, the return value is true, else, it is false. - Note that the setting indexes should be set after the database is opened. */ + Note that the setting indices should be set after the database is opened. */ bool tctdbsetindex(TCTDB *tdb, const char *name, int type); @@ -811,9 +811,9 @@ char *tctdbopaque(TCTDB *tdb); uint64_t tctdbbnumused(TCTDB *tdb); -/* Get the number of column indexes of a table database object. +/* Get the number of column indices of a table database object. `tdb' specifies the table database object. - The return value is the number of column indexes or 0 if the object does not connect to any + The return value is the number of column indices or 0 if the object does not connect to any database file. */ int tctdbinum(TCTDB *tdb); tctdb.h @@ -1289,6 +1289,33 @@ static void *threadwicked(void *targ){ break; default: if(id == 0) iputchar('@'); + if(tctdbtranbegin(tdb)){ + if(myrand(2) == 0){ + if(!tctdbput(tdb, pkbuf, pksiz, cols)){ + eprint(tdb, "tctdbput"); + err = true; + } + } else { + if(!tctdbout(tdb, pkbuf, pksiz) && tctdbecode(tdb) != TCENOREC){ + eprint(tdb, "tctdbout"); + err = true; + } + } + if(myrand(2) == 0){ + if(!tctdbtrancommit(tdb)){ + eprint(tdb, "tctdbtrancommit"); + err = true; + } + } else { + if(!tctdbtranabort(tdb)){ + eprint(tdb, "tctdbtranabort"); + err = true; + } + } + } else { + eprint(tdb, "tctdbtranbegin"); + err = true; + } if(myrand(10000) == 0) srand((unsigned int)(tctime() * 1000) % UINT_MAX); break; } @@ -1296,7 +1323,7 @@ static void *threadwicked(void *targ){ if(id == 0){ if(i % 50 == 0) iprintf(" (%08d)\n", i); if(id == 0 && i == rnum / 4){ - if(!tctdboptimize(tdb, rnum / 50, -1, -1, -1)){ + if(!tctdboptimize(tdb, rnum / 50, -1, -1, -1) && tctdbecode(tdb) != TCEINVALID){ eprint(tdb, "tctdboptimize"); err = true; } tctmttest.c @@ -5758,6 +5758,23 @@ void *tcstrjoin4(const TCMAP *map, int *sp){ } +/* Suspend execution of the current thread. */ +bool tcsleep(double sec){ + if(!isnormal(sec) || sec <= 0.0) return false; + if(sec <= 1.0 / sysconf(_SC_CLK_TCK)) return sched_yield() == 0; + double integ, fract; + fract = modf(sec, &integ); + struct timespec req, rem; + req.tv_sec = integ; + req.tv_nsec = tclmin(fract * 1000.0 * 1000.0 * 1000.0, 999999999); + while(nanosleep(&req, &rem) != 0){ + if(errno != EINTR) return false; + req = rem; + } + return true; +} + + /************************************************************************************************* * filesystem utilities tcutil.c @@ -2596,6 +2596,12 @@ char *tcstrjoin3(const TCMAP *map, char delim); void *tcstrjoin4(const TCMAP *map, int *sp); +/* Suspend execution of the current thread. + `sec' specifies the interval of the suspension in seconds. + If successful, the return value is true, else, it is false. */ +bool tcsleep(double sec); + + /************************************************************************************************* * filesystem utilities @@ -3290,8 +3296,8 @@ typedef struct { /* type of structure for a bit stream o #include <stdio.h> -#define _TC_VERSION "1.4.14" -#define _TC_LIBVER 805 +#define _TC_VERSION "1.4.15" +#define _TC_LIBVER 806 #define _TC_FORMATVER "1.0" enum { /* enumeration for error codes */ tcutil.h @@ -19,13 +19,13 @@ */ module TokyoCabinet { //---------------------------------------------------------------- - // list of strings (substituted for by the native mechanism). + // list of strings (substituted for by the native mechanism) //---------------------------------------------------------------- interface List { string get(in long index); }; //---------------------------------------------------------------- - // map of strings (substituted for by the native mechanism). + // map of strings (substituted for by the native mechanism) //---------------------------------------------------------------- interface Map { string get(in string key); @@ -307,6 +307,7 @@ module TokyoCabinet { long addint(in string key, in long num); double adddouble(in string key, in double num); boolean sync(); + boolean optimize(in string params); boolean vanish(); boolean copy(in string path); boolean tranbegin(); tokyocabinet.idl 8a6686129b4d6a188e8762354035ca85c0d27775 Bob Ippolito bob@redivi.com http://github.com/etrepum/tokyo-cabinet/commit/6f562ea49ea5050bf435b0cb38fbf9ed41e11136 6f562ea49ea5050bf435b0cb38fbf9ed41e11136

2009-04-19T11:07:34-07:00

sync with tokyotyrant-1.4.15.tar.gz 5580f2ab6a07b70a25fe51bfff455fa8d06aa173 Bob Ippolito bob@redivi.com