@@ -1,3 +1,17 @@ +2009-03-11 Mikio Hirabayashi <mikio@users.sourceforge.net> + + * tcutil.c (tctopsort): new function. + + * tchdb.c (tchdbfbpinsert, tchdbfbpsearch, tchdbfbpsplice): performance was improved. + + * tchdb.c (tchdbwriterec): concurrency was improved. + + * tctdb.c (tctdbqrysearchimpl): a bug related to the skip parameter was fixed. + + * tctdb.c (tctdbputimpl, tctdbidxout, tctdbqrysearchimpl): performance was improved. + + - Release: 1.4.11 + 2009-03-02 Mikio Hirabayashi <mikio@users.sourceforge.net> * tcutil.c (tcmdbputproc, tcndbputfunc): removing mechanism was added. ChangeLog @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.61 for tokyocabinet 1.4.10. +# Generated by GNU Autoconf 2.61 for tokyocabinet 1.4.11. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. @@ -572,8 +572,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='tokyocabinet' PACKAGE_TARNAME='tokyocabinet' -PACKAGE_VERSION='1.4.10' -PACKAGE_STRING='tokyocabinet 1.4.10' +PACKAGE_VERSION='1.4.11' +PACKAGE_STRING='tokyocabinet 1.4.11' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1192,7 +1192,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures tokyocabinet 1.4.10 to adapt to many kinds of systems. +\`configure' configures tokyocabinet 1.4.11 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1253,7 +1253,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of tokyocabinet 1.4.10:";; + short | recursive ) echo "Configuration of tokyocabinet 1.4.11:";; esac cat <<\_ACEOF @@ -1354,7 +1354,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -tokyocabinet configure 1.4.10 +tokyocabinet configure 1.4.11 generated by GNU Autoconf 2.61 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1368,7 +1368,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by tokyocabinet $as_me 1.4.10, which was +It was created by tokyocabinet $as_me 1.4.11, which was generated by GNU Autoconf 2.61. Invocation command line was $ $0 $@ @@ -1724,7 +1724,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu # Package information MYLIBVER=8 -MYLIBREV=1 +MYLIBREV=2 MYFORMATVER="1.0" # Targets @@ -6351,7 +6351,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by tokyocabinet $as_me 1.4.10, which was +This file was extended by tokyocabinet $as_me 1.4.11, which was generated by GNU Autoconf 2.61. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -6394,7 +6394,7 @@ Report bugs to <bug-autoconf@gnu.org>." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF ac_cs_version="\\ -tokyocabinet config.status 1.4.10 +tokyocabinet config.status 1.4.11 configured by $0, generated by GNU Autoconf 2.61, with options \\"`echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" configure @@ -7,11 +7,11 @@ #================================================================ # Package name -AC_INIT(tokyocabinet, 1.4.10) +AC_INIT(tokyocabinet, 1.4.11) # Package information MYLIBVER=8 -MYLIBREV=1 +MYLIBREV=2 MYFORMATVER="1.0" # Targets configure.in @@ -47,15 +47,28 @@ switch((now.getMonth() + now.getDate() + now.getMinutes()) % 12){ default: text = "super hyper ultra database manager"; break; case 1: text = "much quicker database manager"; break; - case 3: text = "the uitimate database manager"; break; + case 3: text = "the ultimate database manager"; break; case 5: text = "the supreme database manager"; break; - case 7: text = "the lightning database manager"; break; + case 7: text = "the lightening database manager"; break; case 9: text = "the mighty unbeatable invincible database manager"; break; case 11: text = "the dinosaur wing of database managers"; break; } elem.firstChild.nodeValue = label + ": " + text; } } + elem = document.getElementById("logo"); + if(elem){ + var rnd = Math.floor(Math.random() * 5); + if(rnd == 0){ + elem.src = "logo-g.png"; + elem.removeAttribute("height"); + elem.removeAttribute("width"); + } else if(rnd == 1){ + elem.src = "logo-y.png"; + elem.removeAttribute("height"); + elem.removeAttribute("width"); + } + } } </script> </head> @@ -68,7 +81,7 @@ <div class="note">Last Update: Fri, 13 Feb 2009 15:37:50 +0900</div> <div class="navi">[<span class="void">English</span>/<a href="index.ja.html" hreflang="ja">Japanese</a>]</div> -<div class="logo"><img src="logo.png" alt="Tokyo Cabinet" width="300" height="110" /></div> +<div class="logo"><img src="logo.png" id="logo" alt="Tokyo Cabinet" width="300" height="110" /></div> <hr /> @@ -126,7 +139,7 @@ <p>The following are the source packages of Tokyo Cabinet. As for binary packages, see the site of each distributor.</p> <ul> -<li><a href="tokyocabinet-1.4.9.tar.gz">Latest Source Package (version 1.4.9)</a></li> +<li><a href="tokyocabinet-1.4.11.tar.gz">Latest Source Package (version 1.4.11)</a></li> <li><a href="pastpkg/">Past Versions</a></li> </ul> doc/index.html @@ -97,6 +97,19 @@ elem.firstChild.nodeValue = label + ": " + text; } } + elem = document.getElementById("logo"); + if(elem){ + var rnd = Math.floor(Math.random() * 5); + if(rnd == 0){ + elem.src = "logo-g.png"; + elem.removeAttribute("height"); + elem.removeAttribute("width"); + } else if(rnd == 1){ + elem.src = "logo-y.png"; + elem.removeAttribute("height"); + elem.removeAttribute("width"); + } + } } </script> </head> @@ -109,7 +122,7 @@ <div class="note">Last Update: Fri, 13 Feb 2009 15:37:50 +0900</div> <div class="navi">[<a href="index.html" hreflang="en">English</a>/<span class="void">Japanese</span>]</div> -<div class="logo"><img src="logo-ja.png" alt="Tokyo Cabinet" width="300" height="110" /></div> +<div class="logo"><img src="logo-ja.png" id="logo" alt="Tokyo Cabinet" width="300" height="110" /></div> <hr /> @@ -167,7 +180,7 @@ <p>以下のソースパッケージをダウンロードしてください。バイナリパッケージについては、各ディストリビュータのサイトをご覧ください。</p> <ul> -<li><a href="tokyocabinet-1.4.9.tar.gz">最新のソースパッケージ（バージョン1.4.9）</a></li> +<li><a href="tokyocabinet-1.4.11.tar.gz">最新のソースパッケージ（バージョン1.4.11）</a></li> <li><a href="pastpkg/">過去のバージョン</a></li> </ul> doc/index.ja.html @@ -2206,6 +2206,17 @@ <dd>`<var>buf</var>' specifies the pointer to the region into which the result string is written. The size of the buffer should be equal to or more than 48 bytes.</dd> </dl> +<p>The function `tctopsort' is used in order to sort top records of an array.</p> + +<dl class="api"> +<dt><code>void tctopsort(void *<var>base</var>, size_t <var>nmemb</var>, size_t <var>size</var>, size_t <var>top</var>, int(*<var>compar</var>)(const void *, const void *));</code></dt> +<dd>`<var>base</var>' spacifies the pointer to an array.</dd> +<dd>`<var>nmemb</var>' specifies the number of elements of the array.</dd> +<dd>`<var>size</var>' specifies the size of each element.</dd> +<dd>`<var>top</var>' specifies the number of top records.</dd> +<dd>`<var>compar</var>' specifies the pointer to comparing function. The two arguments specify the pointers of elements. The comparing function should returns positive if the former is big, negative if the latter is big, 0 if both are equal.</dd> +</dl> + <p>The function `tcchidxnew' is used in order to create a consistent hashing object.</p> <dl class="api"> @@ -5235,7 +5246,7 @@ int main(int argc, char **argv){ <dt><code>int tctdbecode(TCTDB *<var>tdb</var>);</code></dt> <dd>`<var>tdb</var>' specifies the table database object.</dd> <dd>The return value is the last happened error code.</dd> -<dd>The following error code is defined: `TCESUCCESS' for success, `TCETHREAD' for threading error, `TCEINVALID' for invalid operation, `TCENOFILE' for file not found, `TCENOPERM' for no permission, `TCEMETA' for invalid meta data, `TCERHEAD' for invalid record header, `TCEOPEN' for open error, `TCECLOSE' for close error, `TCETRUNC' for trunc error, `TCESYNC' for sync error, `TCESTAT' for stat error, `TCESEEK' for seek error, `TCEREAD' for read error, `TCEWRITE' for write error, `TCEMMAP' for mmap error, `TCELOCK' for lock error, `TCEUNLINK' for unlink error, `TCERENAME' for rename error, `TCEMKDIR' for mkdir error, `TCERMDIR' for rmdir error, `TCEKEEP' for existing record, `TCENOREC' for no record found, and `TCEMISC' for miscellaneous error.</dd> +<dd>The following error codes are defined: `TCESUCCESS' for success, `TCETHREAD' for threading error, `TCEINVALID' for invalid operation, `TCENOFILE' for file not found, `TCENOPERM' for no permission, `TCEMETA' for invalid meta data, `TCERHEAD' for invalid record header, `TCEOPEN' for open error, `TCECLOSE' for close error, `TCETRUNC' for trunc error, `TCESYNC' for sync error, `TCESTAT' for stat error, `TCESEEK' for seek error, `TCEREAD' for read error, `TCEWRITE' for write error, `TCEMMAP' for mmap error, `TCELOCK' for lock error, `TCEUNLINK' for unlink error, `TCERENAME' for rename error, `TCEMKDIR' for mkdir error, `TCERMDIR' for rmdir error, `TCEKEEP' for existing record, `TCENOREC' for no record found, and `TCEMISC' for miscellaneous error.</dd> </dl> <p>The function `tctdbsetmutex' is used in order to set mutual exclusion control of a table database object for threading.</p> doc/spex-en.html @@ -2209,6 +2209,17 @@ <dd>`<var>buf</var>' specifies the pointer to the region into which the result string is written. The size of the buffer should be equal to or more than 48 bytes.</dd> </dl> +<p>The function `tctopsort' is used in order to sort top records of an array.</p> + +<dl class="api"> +<dt><code>void tctopsort(void *<var>base</var>, size_t <var>nmemb</var>, size_t <var>size</var>, size_t <var>top</var>, int(*<var>compar</var>)(const void *, const void *));</code></dt> +<dd>`<var>base</var>' spacifies the pointer to an array.</dd> +<dd>`<var>nmemb</var>' specifies the number of elements of the array.</dd> +<dd>`<var>size</var>' specifies the size of each element.</dd> +<dd>`<var>top</var>' specifies the number of top records.</dd> +<dd>`<var>compar</var>' specifies the pointer to comparing function. The two arguments specify the pointers of elements. The comparing function should returns positive if the former is big, negative if the latter is big, 0 if both are equal.</dd> +</dl> + <p>The function `tcchidxnew' is used in order to create a consistent hashing object.</p> <dl class="api"> @@ -5238,7 +5249,7 @@ int main(int argc, char **argv){ <dt><code>int tctdbecode(TCTDB *<var>tdb</var>);</code></dt> <dd>`<var>tdb</var>' specifies the table database object.</dd> <dd>The return value is the last happened error code.</dd> -<dd>The following error code is defined: `TCESUCCESS' for success, `TCETHREAD' for threading error, `TCEINVALID' for invalid operation, `TCENOFILE' for file not found, `TCENOPERM' for no permission, `TCEMETA' for invalid meta data, `TCERHEAD' for invalid record header, `TCEOPEN' for open error, `TCECLOSE' for close error, `TCETRUNC' for trunc error, `TCESYNC' for sync error, `TCESTAT' for stat error, `TCESEEK' for seek error, `TCEREAD' for read error, `TCEWRITE' for write error, `TCEMMAP' for mmap error, `TCELOCK' for lock error, `TCEUNLINK' for unlink error, `TCERENAME' for rename error, `TCEMKDIR' for mkdir error, `TCERMDIR' for rmdir error, `TCEKEEP' for existing record, `TCENOREC' for no record found, and `TCEMISC' for miscellaneous error.</dd> +<dd>The following error codes are defined: `TCESUCCESS' for success, `TCETHREAD' for threading error, `TCEINVALID' for invalid operation, `TCENOFILE' for file not found, `TCENOPERM' for no permission, `TCEMETA' for invalid meta data, `TCERHEAD' for invalid record header, `TCEOPEN' for open error, `TCECLOSE' for close error, `TCETRUNC' for trunc error, `TCESYNC' for sync error, `TCESTAT' for stat error, `TCESEEK' for seek error, `TCEREAD' for read error, `TCEWRITE' for write error, `TCEMMAP' for mmap error, `TCELOCK' for lock error, `TCEUNLINK' for unlink error, `TCERENAME' for rename error, `TCEMKDIR' for mkdir error, `TCERMDIR' for rmdir error, `TCEKEEP' for existing record, `TCENOREC' for no record found, and `TCEMISC' for miscellaneous error.</dd> </dl> <p>The function `tctdbsetmutex' is used in order to set mutual exclusion control of a table database object for threading.</p> doc/spex-ja.html doc/tokyoproducts.pdf doc/tokyoproducts.ppt @@ -18,7 +18,7 @@ use constant { my @commands = ( "./tchtest write casket 1000000 1000000", "./tchtest read casket", - "./tchtest rcat casket 100000", + "./tchtest rcat casket 1000000", "./tcbtest write casket 1000000", "./tcbtest read casket", "./tcbtest rcat -lc 256 -nc 128 casket 100000", lab/stopwatch @@ -76,7 +76,7 @@ The function `tctdbecode' is used in order to get the last happened error code o The return value is the last happened error code. .RE .RS -The following error code is defined: `TCESUCCESS' for success, `TCETHREAD' for threading error, `TCEINVALID' for invalid operation, `TCENOFILE' for file not found, `TCENOPERM' for no permission, `TCEMETA' for invalid meta data, `TCERHEAD' for invalid record header, `TCEOPEN' for open error, `TCECLOSE' for close error, `TCETRUNC' for trunc error, `TCESYNC' for sync error, `TCESTAT' for stat error, `TCESEEK' for seek error, `TCEREAD' for read error, `TCEWRITE' for write error, `TCEMMAP' for mmap error, `TCELOCK' for lock error, `TCEUNLINK' for unlink error, `TCERENAME' for rename error, `TCEMKDIR' for mkdir error, `TCERMDIR' for rmdir error, `TCEKEEP' for existing record, `TCENOREC' for no record found, and `TCEMISC' for miscellaneous error. +The following error codes are defined: `TCESUCCESS' for success, `TCETHREAD' for threading error, `TCEINVALID' for invalid operation, `TCENOFILE' for file not found, `TCENOPERM' for no permission, `TCEMETA' for invalid meta data, `TCERHEAD' for invalid record header, `TCEOPEN' for open error, `TCECLOSE' for close error, `TCETRUNC' for trunc error, `TCESYNC' for sync error, `TCESTAT' for stat error, `TCESEEK' for seek error, `TCEREAD' for read error, `TCEWRITE' for write error, `TCEMMAP' for mmap error, `TCELOCK' for lock error, `TCEUNLINK' for unlink error, `TCERENAME' for rename error, `TCEMKDIR' for mkdir error, `TCERMDIR' for rmdir error, `TCEKEEP' for existing record, `TCENOREC' for no record found, and `TCEMISC' for miscellaneous error. .RE .RE .PP man/tctdb.3 @@ -3543,6 +3543,28 @@ The function `tcmd5hash' is used in order to get the MD5 hash value of a record. .RE .RE .PP +The function `tctopsort' is used in order to sort top records of an array. +.PP +.RS +.br +\fBvoid tctopsort(void *\fIbase\fB, size_t \fInmemb\fB, size_t \fIsize\fB, size_t \fItop\fB, int(*\fIcompar\fB)(const void *, const void *));\fR +.RS +`\fIbase\fR' spacifies the pointer to an array. +.RE +.RS +`\fInmemb\fR' specifies the number of elements of the array. +.RE +.RS +`\fIsize\fR' specifies the size of each element. +.RE +.RS +`\fItop\fR' specifies the number of top records. +.RE +.RS +`\fIcompar\fR' specifies the pointer to comparing function. The two arguments specify the pointers of elements. The comparing function should returns positive if the former is big, negative if the latter is big, 0 if both are equal. +.RE +.RE +.PP The function `tcchidxnew' is used in order to create a consistent hashing object. .PP .RS man/tcutil.3 @@ -44,9 +44,10 @@ #define HDBXFSIZINC 32768 // increment of extra file size #define HDBMINRUNIT 48 // minimum record reading unit #define HDBMAXHSIZ 32 // maximum record header size +#define HDBFBPALWRAT 2 // allowance ratio of the free block pool #define HDBFBPBSIZ 64 // base region size of the free block pool #define HDBFBPESIZ 4 // size of each region of the free block pool -#define HDBFBPMGFREQ 256 // frequency to merge the free block pool +#define HDBFBPMGFREQ 4096 // frequency to merge the free block pool #define HDBDRPUNIT 65536 // unit size of the delayed record pool #define HDBDRPLAT 2048 // latitude size of the delayed record pool #define HDBCACHEOUT 128 // number of records in a process of cacheout @@ -2039,7 +2040,7 @@ static void tchdbsetbucket(TCHDB *hdb, uint64_t bidx, uint64_t off){ The return value is true if successful, else, it is false. */ static bool tchdbsavefbp(TCHDB *hdb){ assert(hdb); - if(hdb->fbpnum > (hdb->fbpmax >> 1)){ + if(hdb->fbpnum > hdb->fbpmax){ tchdbfbpmerge(hdb); } else if(hdb->fbpnum > 1){ tcfbpsortbyoff(hdb->fbpool, hdb->fbpnum); @@ -2089,7 +2090,7 @@ static bool tchdbloadfbp(TCHDB *hdb){ } const char *rp = buf; HDBFB *cur = hdb->fbpool; - HDBFB *end = cur + hdb->fbpmax; + HDBFB *end = cur + hdb->fbpmax * HDBFBPALWRAT; uint64_t base = 0; while(cur < end && *rp != '\0'){ int step; @@ -2106,6 +2107,7 @@ static bool tchdbloadfbp(TCHDB *hdb){ } hdb->fbpnum = cur - (HDBFB *)hdb->fbpool; TCFREE(buf); + tcfbpsortbyrsiz(hdb->fbpool, hdb->fbpnum); return true; } @@ -2199,7 +2201,6 @@ static void tcfbpsortbyrsiz(HDBFB *fbpool, int fbpnum){ static void tchdbfbpmerge(TCHDB *hdb){ assert(hdb); TCDODEBUG(hdb->cnt_mergefbp++); - int32_t onum = hdb->fbpnum; tcfbpsortbyoff(hdb->fbpool, hdb->fbpnum); HDBFB *wp = hdb->fbpool;; HDBFB *cur = wp; @@ -2218,7 +2219,7 @@ static void tchdbfbpmerge(TCHDB *hdb){ } if(end->off > 0) *(wp++) = *end; hdb->fbpnum = wp - (HDBFB *)hdb->fbpool; - hdb->fbpmis = (hdb->fbpnum < onum) ? 0 : hdb->fbpnum * -2; + hdb->fbpmis = hdb->fbpnum * -1; } @@ -2231,18 +2232,41 @@ static void tchdbfbpinsert(TCHDB *hdb, uint64_t off, uint32_t rsiz){ TCDODEBUG(hdb->cnt_insertfbp++); if(hdb->fpow < 1) return; HDBFB *pv = hdb->fbpool; - if(hdb->fbpnum >= hdb->fbpmax){ + if(hdb->fbpnum >= hdb->fbpmax * HDBFBPALWRAT){ tchdbfbpmerge(hdb); tcfbpsortbyrsiz(hdb->fbpool, hdb->fbpnum); - if(hdb->fbpnum >= hdb->fbpmax){ + int diff = hdb->fbpnum - hdb->fbpmax; + if(diff > 0){ TCDODEBUG(hdb->cnt_reducefbp++); - int32_t dnum = (hdb->fbpmax >> 2) + 1; - memmove(pv, pv + dnum, (hdb->fbpnum - dnum) * sizeof(*pv)); - hdb->fbpnum -= dnum; + memmove(pv, pv + diff, (hdb->fbpnum - diff) * sizeof(*pv)); + hdb->fbpnum -= diff; } hdb->fbpmis = 0; } - pv = pv + hdb->fbpnum; + int num = hdb->fbpnum; + int left = 0; + int right = num; + int i = (left + right) / 2; + int cand = -1; + while(right >= left && i < num){ + int rv = (int)rsiz - (int)pv[i].rsiz; + if(rv == 0){ + cand = i; + break; + } else if(rv <= 0){ + cand = i; + right = i - 1; + } else { + left = i + 1; + } + i = (left + right) / 2; + } + if(cand >= 0){ + pv += cand; + memmove(pv + 1, pv, sizeof(*pv) * (num - cand)); + } else { + pv += num; + } pv->off = off; pv->rsiz = rsiz; hdb->fbpnum++; @@ -2263,30 +2287,43 @@ static bool tchdbfbpsearch(TCHDB *hdb, TCHREC *rec){ } uint32_t rsiz = rec->rsiz; HDBFB *pv = hdb->fbpool; - HDBFB *ep = pv + hdb->fbpnum; - while(pv < ep){ - if(pv->rsiz >= rsiz){ - if(pv->rsiz > rsiz * 2){ - uint32_t psiz = tchdbpadsize(hdb, pv->off + rsiz); - uint64_t noff = pv->off + rsiz + psiz; - if(pv->rsiz >= (noff - pv->off) * 2){ - TCDODEBUG(hdb->cnt_dividefbp++); - rec->off = pv->off; - rec->rsiz = noff - pv->off; - pv->off = noff; - pv->rsiz -= rec->rsiz; - return tchdbwritefb(hdb, pv->off, pv->rsiz); - } + int num = hdb->fbpnum; + int left = 0; + int right = num; + int i = (left + right) / 2; + int cand = -1; + while(right >= left && i < num){ + int rv = (int)rsiz - (int)pv[i].rsiz; + if(rv == 0){ + cand = i; + break; + } else if(rv <= 0){ + cand = i; + right = i - 1; + } else { + left = i + 1; + } + i = (left + right) / 2; + } + if(cand >= 0){ + pv += cand; + if(pv->rsiz > rsiz * 2){ + uint32_t psiz = tchdbpadsize(hdb, pv->off + rsiz); + uint64_t noff = pv->off + rsiz + psiz; + if(pv->rsiz >= (noff - pv->off) * 2){ + TCDODEBUG(hdb->cnt_dividefbp++); + rec->off = pv->off; + rec->rsiz = noff - pv->off; + pv->off = noff; + pv->rsiz -= rec->rsiz; + return tchdbwritefb(hdb, pv->off, pv->rsiz); } - rec->off = pv->off; - rec->rsiz = pv->rsiz; - ep--; - pv->off = ep->off; - pv->rsiz = ep->rsiz; - hdb->fbpnum--; - return true; } - pv++; + rec->off = pv->off; + rec->rsiz = pv->rsiz; + memmove(pv, pv + 1, sizeof(*pv) * (num - cand - 1)); + hdb->fbpnum--; + return true; } rec->off = hdb->fsiz; rec->rsiz = 0; @@ -2306,24 +2343,62 @@ static bool tchdbfbpsearch(TCHDB *hdb, TCHREC *rec){ The return value is whether splicing succeeded or not. */ static bool tchdbfbpsplice(TCHDB *hdb, TCHREC *rec, uint32_t nsiz){ assert(hdb && rec && nsiz > 0); - if(hdb->fbpnum < 1) return false; + if(hdb->mmtx){ + if(hdb->fbpnum < 1) return false; + uint64_t off = rec->off + rec->rsiz; + uint32_t rsiz = rec->rsiz; + uint8_t magic; + if(tchdbseekreadtry(hdb, off, &magic, sizeof(magic)) && magic != HDBMAGICFB) return false; + HDBFB *pv = hdb->fbpool; + HDBFB *ep = pv + hdb->fbpnum; + while(pv < ep){ + if(pv->off == off && rsiz + pv->rsiz >= nsiz){ + if(hdb->iter == pv->off) hdb->iter += pv->rsiz; + rec->rsiz += pv->rsiz; + ep--; + pv->off = ep->off; + pv->rsiz = ep->rsiz; + hdb->fbpnum--; + return true; + } + pv++; + } + return false; + } uint64_t off = rec->off + rec->rsiz; - uint32_t rsiz = rec->rsiz; - HDBFB *pv = hdb->fbpool; - HDBFB *ep = pv + hdb->fbpnum; - while(pv < ep){ - if(pv->off == off && rsiz + pv->rsiz >= nsiz){ - if(hdb->iter == pv->off) hdb->iter += pv->rsiz; - rec->rsiz += pv->rsiz; - ep--; - pv->off = ep->off; - pv->rsiz = ep->rsiz; - hdb->fbpnum--; - return true; + TCHREC nrec; + char nbuf[HDBIOBUFSIZ]; + while(off < hdb->fsiz){ + nrec.off = off; + if(!tchdbreadrec(hdb, &nrec, nbuf)) return false; + if(nrec.magic != HDBMAGICFB) break; + if(hdb->iter == off) hdb->iter += nrec.rsiz; + off += nrec.rsiz; + } + uint32_t jsiz = off - rec->off; + if(jsiz < nsiz) return false; + rec->rsiz = jsiz; + uint64_t base = rec->off; + HDBFB *wp = hdb->fbpool;; + HDBFB *cur = wp; + HDBFB *end = wp + hdb->fbpnum; + while(cur < end){ + if(cur->off < base || cur->off > off) *(wp++) = *cur; + cur++; + } + hdb->fbpnum = wp - (HDBFB *)hdb->fbpool; + if(jsiz > nsiz * 2){ + uint32_t psiz = tchdbpadsize(hdb, rec->off + nsiz); + uint64_t noff = rec->off + nsiz + psiz; + if(jsiz >= (noff - rec->off) * 2){ + TCDODEBUG(hdb->cnt_dividefbp++); + nsiz = off - noff; + if(!tchdbwritefb(hdb, noff, nsiz)) return false; + rec->rsiz = noff - rec->off; + tchdbfbpinsert(hdb, noff, nsiz); } - pv++; } - return false; + return true; } @@ -2404,16 +2479,27 @@ static bool tchdbwriterec(TCHDB *hdb, TCHREC *rec, uint64_t bidx, off_t entoff){ finc = rec->rsiz; } else if(rsiz > rec->rsiz){ if(rbuf != stack) TCFREE(rbuf); + if(!HDBLOCKDB(hdb)) return false; if(tchdbfbpsplice(hdb, rec, rsiz)){ TCDODEBUG(hdb->cnt_splicefbp++); - return tchdbwriterec(hdb, rec, bidx, entoff); + bool rv = tchdbwriterec(hdb, rec, bidx, entoff); + HDBUNLOCKDB(hdb); + return rv; } TCDODEBUG(hdb->cnt_moverec++); - if(!tchdbwritefb(hdb, rec->off, rec->rsiz)) return false; + if(!tchdbwritefb(hdb, rec->off, rec->rsiz)){ + HDBUNLOCKDB(hdb); + return false; + } tchdbfbpinsert(hdb, rec->off, rec->rsiz); rec->rsiz = rsiz; - if(!tchdbfbpsearch(hdb, rec)) return false; - return tchdbwriterec(hdb, rec, bidx, entoff); + if(!tchdbfbpsearch(hdb, rec)){ + HDBUNLOCKDB(hdb); + return false; + } + bool rv = tchdbwriterec(hdb, rec, bidx, entoff); + HDBUNLOCKDB(hdb); + return rv; } else { TCDODEBUG(hdb->cnt_reuserec++); uint32_t psiz = rec->rsiz - rsiz; @@ -2424,8 +2510,16 @@ static bool tchdbwriterec(TCHDB *hdb, TCHREC *rec, uint64_t bidx, off_t entoff){ uint32_t nsiz = rec->rsiz - rsiz - psiz; rec->rsiz = noff - rec->off; rec->psiz = psiz; - if(!tchdbwritefb(hdb, noff, nsiz)) return false; + if(!tchdbwritefb(hdb, noff, nsiz)){ + if(rbuf != stack) TCFREE(rbuf); + return false; + } + if(!HDBLOCKDB(hdb)){ + if(rbuf != stack) TCFREE(rbuf); + return false; + } tchdbfbpinsert(hdb, noff, nsiz); + HDBUNLOCKDB(hdb); } rec->psiz = psiz; } @@ -3056,7 +3150,7 @@ static bool tchdbopenimpl(TCHDB *hdb, const char *path, int omode){ } hdb->fbpmax = 1 << hdb->fpow;; if(omode & HDBOWRITER){ - TCMALLOC(hdb->fbpool, hdb->fbpmax * sizeof(HDBFB)); + TCMALLOC(hdb->fbpool, hdb->fbpmax * HDBFBPALWRAT * sizeof(HDBFB)); } else { hdb->fbpool = NULL; } @@ -3208,7 +3302,7 @@ static bool tchdbputimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, entoff = rec.off + (sizeof(uint8_t) + sizeof(uint8_t)) + (hdb->ba64 ? sizeof(uint64_t) : sizeof(uint32_t)); } else { - bool rv, lock; + bool rv; int nvsiz; char *nvbuf; HDBPDPROCOP *procptr; @@ -3227,7 +3321,6 @@ static bool tchdbputimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, return false; } nvsiz = rec.vsiz + vsiz; - lock = vsiz + TCCALCVNUMSIZE(nvsiz) - TCCALCVNUMSIZE(rec.vsiz) > rec.psiz; if(rec.bbuf){ TCREALLOC(rec.bbuf, rec.bbuf, rec.ksiz + nvsiz); memcpy(rec.bbuf + rec.ksiz + rec.vsiz, vbuf, vsiz); @@ -3241,16 +3334,6 @@ static bool tchdbputimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, rec.vbuf = rec.bbuf; rec.vsiz = nvsiz; } - if(lock){ - if(!HDBLOCKDB(hdb)){ - TCFREE(rec.bbuf); - return false; - } - rv = tchdbwriterec(hdb, &rec, bidx, entoff); - HDBUNLOCKDB(hdb); - TCFREE(rec.bbuf); - return rv; - } rv = tchdbwriterec(hdb, &rec, bidx, entoff); TCFREE(rec.bbuf); return rv; @@ -3311,22 +3394,10 @@ static bool tchdbputimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, if(nvbuf == (void *)-1){ return tchdbremoverec(hdb, &rec, rbuf, bidx, entoff); } else if(nvbuf){ - lock = nvsiz + TCCALCVNUMSIZE(nvsiz) - TCCALCVNUMSIZE(rec.vsiz) > - rec.vsiz + rec.psiz; rec.kbuf = kbuf; rec.ksiz = ksiz; rec.vbuf = nvbuf; rec.vsiz = nvsiz; - if(lock){ - if(!HDBLOCKDB(hdb)){ - TCFREE(nvbuf); - return false; - } - rv = tchdbwriterec(hdb, &rec, bidx, entoff); - HDBUNLOCKDB(hdb); - TCFREE(nvbuf); - return rv; - } rv = tchdbwriterec(hdb, &rec, bidx, entoff); TCFREE(nvbuf); return rv; @@ -3337,17 +3408,10 @@ static bool tchdbputimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, break; } TCFREE(rec.bbuf); - lock = vsiz + TCCALCVNUMSIZE(vsiz) - TCCALCVNUMSIZE(rec.vsiz) > rec.vsiz + rec.psiz; rec.ksiz = ksiz; rec.vsiz = vsiz; rec.kbuf = kbuf; rec.vbuf = vbuf; - if(lock){ - if(!HDBLOCKDB(hdb)) return false; - rv = tchdbwriterec(hdb, &rec, bidx, entoff); - HDBUNLOCKDB(hdb); - return rv; - } return tchdbwriterec(hdb, &rec, bidx, entoff); } } tchdb.c @@ -1399,23 +1399,41 @@ static bool tctdbputimpl(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols, return false; } TCMAP *ocols = tcmapload(obuf, osiz); - if(!tctdbidxout(tdb, pkbuf, pksiz, ocols)) err = true; if(dmode == TDBPDCAT){ + TCMAP *ncols = tcmapnew2(tcmaprnum(cols) + 1); tcmapiterinit(cols); const char *kbuf; int ksiz; while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){ int vsiz; const char *vbuf = tcmapiterval(kbuf, &vsiz); - tcmapput(ocols, kbuf, ksiz, vbuf, vsiz); + if(tcmapputkeep(ocols, kbuf, ksiz, vbuf, vsiz)) tcmapput(ncols, kbuf, ksiz, vbuf, vsiz); } - if(!tctdbidxput(tdb, pkbuf, pksiz, ocols)) err = true; + if(!tctdbidxput(tdb, pkbuf, pksiz, ncols)) err = true; + tcmapdel(ncols); int csiz; char *cbuf = tcmapdump(ocols, &csiz); if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true; TCFREE(cbuf); } else { - if(!tctdbidxput(tdb, pkbuf, pksiz, cols)) err = true; + TCMAP *ncols = tcmapnew2(tcmaprnum(cols) + 1); + tcmapiterinit(cols); + const char *kbuf; + int ksiz; + while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){ + int vsiz; + const char *vbuf = tcmapiterval(kbuf, &vsiz); + int osiz; + const char *obuf = tcmapget(ocols, kbuf, ksiz, &osiz); + if(obuf && osiz == vsiz && !memcmp(obuf, vbuf, osiz)){ + tcmapout(ocols, kbuf, ksiz); + } else { + tcmapput(ncols, kbuf, ksiz, vbuf, vsiz); + } + } + if(!tctdbidxout(tdb, pkbuf, pksiz, ocols)) err = true; + if(!tctdbidxput(tdb, pkbuf, pksiz, ncols)) err = true; + tcmapdel(ncols); int csiz; char *cbuf = tcmapdump(cols, &csiz); if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true; @@ -2680,24 +2698,28 @@ static TCLIST *tctdbqrysearchimpl(TDBQRY *qry){ key->vbuf = vbuf; key->vsiz = vsiz; } + int (*compar)(const TDBSORTKEY *a, const TDBSORTKEY *b) = NULL; switch(otype){ case TDBQOSTRASC: - qsort(keys, rnum, sizeof(*keys), - (int(*)(const void *, const void *))tdbcmpsortkeystrasc); + compar = tdbcmpsortkeystrasc; break; case TDBQOSTRDESC: - qsort(keys, rnum, sizeof(*keys), - (int(*)(const void *, const void *))tdbcmpsortkeystrdesc); + compar = tdbcmpsortkeystrdesc; break; case TDBQONUMASC: - qsort(keys, rnum, sizeof(*keys), - (int(*)(const void *, const void *))tdbcmpsortkeynumasc); + compar = tdbcmpsortkeynumasc; break; case TDBQONUMDESC: - qsort(keys, rnum, sizeof(*keys), - (int(*)(const void *, const void *))tdbcmpsortkeynumdesc); + compar = tdbcmpsortkeynumdesc; break; } + if(compar){ + if(max <= rnum / 16){ + tctopsort(keys, rnum, sizeof(*keys), max, (int (*)(const void *, const void *))compar); + } else { + qsort(keys, rnum, sizeof(*keys), (int (*)(const void *, const void *))compar); + } + } TCLIST *nres = tclistnew2(rnum); for(int i = 0; i < rnum; i++){ TDBSORTKEY *key = keys + i; @@ -2719,6 +2741,7 @@ static TCLIST *tctdbqrysearchimpl(TDBQRY *qry){ int rsiz; TCFREE(tclistshift(res, &rsiz)); } + max -= qry->skip; } if(TCLISTNUM(res) > max){ int left = TCLISTNUM(res) - max; @@ -3215,34 +3238,32 @@ static bool tctdbidxout(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){ for(int i = 0; i < inum; i++){ TDBIDX *idx = idxs + i; if(strcmp(idx->name, kbuf)) continue; - TCLIST *ovals; + BDBCUR *cur; switch(idx->type){ case TDBITLEXICAL: case TDBITDECIMAL: - if((ovals = tcbdbget4(idx->db, vbuf, vsiz)) != NULL){ - int on = TCLISTNUM(ovals); - for(int j = 0; j < on; j++){ - const char *obuf; - int osiz; - TCLISTVAL(obuf, ovals, j, osiz); - if(osiz == pksiz && !memcmp(obuf, pkbuf, osiz)){ - TCFREE(tclistremove(ovals, j, &osiz)); + cur = tcbdbcurnew(idx->db); + if(tcbdbcurjump(cur, vbuf, vsiz)){ + int oksiz; + const char *okbuf; + while((okbuf = tcbdbcurkey3(cur, &oksiz)) != NULL){ + if(oksiz != vsiz || memcmp(okbuf, vbuf, oksiz)) break; + int ovsiz; + const char *ovbuf = tcbdbcurval3(cur, &ovsiz); + if(ovsiz == pksiz && !memcmp(ovbuf, pkbuf, ovsiz)){ + if(!tcbdbcurout(cur)){ + tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__); + err = true; + } break; } + tcbdbcurnext(cur); } - if(!tcbdbout3(idx->db, vbuf, vsiz)){ - tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__); - err = true; - } - if(!tcbdbputdup3(idx->db, vbuf, vsiz, ovals)){ - tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__); - err = true; - } - tclistdel(ovals); } else { tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__); err = true; } + tcbdbcurdel(cur); break; } } tctdb.c @@ -33,6 +33,7 @@ static void iputchar(int c); static void *pdprocfunc(const void *vbuf, int vsiz, int *sp, void *op); static bool iterfunc(const void *kbuf, int ksiz, const void *vbuf, int vsiz, void *op); static int myrand(int range); +static int intcompar(const void *ap, const void *bp); static int runxstr(int argc, char **argv); static int runlist(int argc, char **argv); static int runmap(int argc, char **argv); @@ -152,6 +153,12 @@ static int myrand(int range){ } +/* compare two integers */ +static int intcompar(const void *ap, const void *bp){ + return *(int *)ap - *(int *)bp; +} + + /* parse arguments of xstr command */ static int runxstr(int argc, char **argv){ char *rstr = NULL; @@ -885,6 +892,23 @@ static int procmisc(int rnum){ tcmd5hash(kbuf, ksiz, buf); } tcfree(buf); + anum = myrand(30) + 1; + int tary[anum], qary[anum]; + for(int j = 0; j < anum; j++){ + int val = myrand(anum * 2 + 1); + tary[j] = val; + qary[j] = val; + } + int tnum = myrand(anum); + tctopsort(tary, anum, sizeof(*tary), tnum, intcompar); + qsort(qary, anum, sizeof(*qary), intcompar); + for(int j = 0; j < tnum; j++){ + if(tary[j] != qary[j]) err = true; + } + qsort(tary, anum, sizeof(*tary), intcompar); + for(int j = 0; j < anum; j++){ + if(tary[j] != qary[j]) err = true; + } TCCHIDX *chidx = tcchidxnew(5); for(int i = 0; i < 10; i++){ char kbuf[RECBUFSIZ]; tcutest.c @@ -5083,6 +5083,68 @@ void tcmd5hash(const void *ptr, int size, char *buf){ } +/* Sort top records of an array. */ +void tctopsort(void *base, size_t nmemb, size_t size, size_t top, + int(*compar)(const void *, const void *)){ + assert(base && size > 0 && compar); + if(nmemb < 1) return; + if(top > nmemb) top = nmemb; + char *bp = base; + char *ep = bp + nmemb * size; + char *rp = bp + size; + int num = 1; + char swap[size]; + while(rp < ep){ + if(num < top){ + int cidx = num; + while(cidx > 0){ + int pidx = (cidx - 1) / 2; + if(compar(bp + cidx * size, bp + pidx * size) <= 0) break; + memcpy(swap, bp + cidx * size, size); + memcpy(bp + cidx * size, bp + pidx * size, size); + memcpy(bp + pidx * size, swap, size); + cidx = pidx; + } + num++; + } else if(compar(rp, bp) < 0){ + memcpy(swap, bp, size); + memcpy(bp, rp, size); + memcpy(rp, swap, size); + int pidx = 0; + int bot = num / 2; + while(pidx < bot){ + int cidx = pidx * 2 + 1; + if(cidx < num - 1 && compar(bp + cidx * size, bp + (cidx + 1) * size) < 0) cidx++; + if(compar(bp + pidx * size, bp + cidx * size) > 0) break; + memcpy(swap, bp + pidx * size, size); + memcpy(bp + pidx * size, bp + cidx * size, size); + memcpy(bp + cidx * size, swap, size); + pidx = cidx; + } + } + rp += size; + } + num = top - 1; + while(num > 0){ + memcpy(swap, bp, size); + memcpy(bp, bp + num * size, size); + memcpy(bp + num * size, swap, size); + int pidx = 0; + int bot = num / 2; + while(pidx < bot){ + int cidx = pidx * 2 + 1; + if(cidx < num - 1 && compar(bp + cidx * size, bp + (cidx + 1) * size) < 0) cidx++; + if(compar(bp + pidx * size, bp + cidx * size) > 0) break; + memcpy(swap, bp + pidx * size, size); + memcpy(bp + pidx * size, bp + cidx * size, size); + memcpy(bp + cidx * size, swap, size); + pidx = cidx; + } + num--; + } +} + + /* Create a consistent hashing object. */ TCCHIDX *tcchidxnew(int range){ assert(range > 0); tcutil.c @@ -2420,6 +2420,18 @@ char *tcregexreplace(const char *str, const char *regex, const char *alt); void tcmd5hash(const void *ptr, int size, char *buf); +/* Sort top records of an array. + `base' spacifies the pointer to an array. + `nmemb' specifies the number of elements of the array. + `size' specifies the size of each element. + `top' specifies the number of top records. + `compar' specifies the pointer to comparing function. The two arguments specify the pointers + of elements. The comparing function should returns positive if the former is big, negative + if the latter is big, 0 if both are equal. */ +void tctopsort(void *base, size_t nmemb, size_t size, size_t top, + int(*compar)(const void *, const void *)); + + /* Create a consistent hashing object. `range' specifies the number of nodes. It should be more than 0. The range of hash values is from 0 to less than the specified number. @@ -3278,8 +3290,8 @@ typedef struct { /* type of structure for a bit stream o #include <stdio.h> -#define _TC_VERSION "1.4.10" -#define _TC_LIBVER 801 +#define _TC_VERSION "1.4.11" +#define _TC_LIBVER 802 #define _TC_FORMATVER "1.0" enum { /* enumeration for error codes */ tcutil.h a203b0e05a815d8da472fa774fae46fdce93fe9d Bob Ippolito bob@redivi.com http://github.com/etrepum/tokyo-cabinet/commit/43aeab6c8c84e72658158fdffdee784760b93da0 43aeab6c8c84e72658158fdffdee784760b93da0 2009-03-23T15:01:48-07:00 2009-03-23T15:01:48-07:00 sync with tokyocabinet-1.4.11.tar.gz 9b76d00ae8655e4f8456806f8ead776e15dc3d52 Bob Ippolito bob@redivi.com