diff --git a/api/java/SphinxClient.java b/api/java/SphinxClient.java index 321fd436..b2c91b97 100644 --- a/api/java/SphinxClient.java +++ b/api/java/SphinxClient.java @@ -142,7 +142,7 @@ public class SphinxClient /** Creates a new SphinxClient instance. */ public SphinxClient() { - this("localhost", 3312); + this("localhost", 9312); } /** Creates a new SphinxClient instance, with host:port specification. */ @@ -536,7 +536,7 @@ public void SetWeights(int[] weights) throws SphinxException * Bind per-field weights by field name. * @param fieldWeights hash which maps String index names to Integer weights */ - public void SetFieldeights ( Map fieldWeights ) throws SphinxException + public void SetFieldWeights ( Map fieldWeights ) throws SphinxException { /* FIXME! implement checks here */ _fieldWeights = ( fieldWeights==null ) ? new LinkedHashMap () : fieldWeights; diff --git a/api/java/test.java b/api/java/test.java index 2061bc99..f1b0f80f 100644 --- a/api/java/test.java +++ b/api/java/test.java @@ -41,7 +41,7 @@ public static void main ( String[] argv ) throws SphinxException StringBuffer q = new StringBuffer(); String host = "localhost"; - int port = 3312; + int port = 9312; int mode = SphinxClient.SPH_MATCH_ALL; String index = "*"; int offset = 0; diff --git a/api/libsphinxclient/sphinxclient.c b/api/libsphinxclient/sphinxclient.c index 1ba6d010..6558e6bf 100644 --- a/api/libsphinxclient/sphinxclient.c +++ b/api/libsphinxclient/sphinxclient.c @@ -61,14 +61,17 @@ enum SEARCHD_COMMAND_SEARCH = 0, SEARCHD_COMMAND_EXCERPT = 1, SEARCHD_COMMAND_UPDATE = 2, - SEARCHD_COMMAND_KEYWORDS = 3 + SEARCHD_COMMAND_KEYWORDS = 3, + SEARCHD_COMMAND_PERSIST = 4, + SEARCHD_COMMAND_STATUS = 5 }; enum { VER_COMMAND_EXCERPT = 0x100, VER_COMMAND_UPDATE = 0x101, - VER_COMMAND_KEYWORDS = 0x100 + VER_COMMAND_KEYWORDS = 0x100, + VER_COMMAND_STATUS = 0x100 }; ////////////////////////////////////////////////////////////////////////// @@ -164,6 +167,8 @@ struct st_sphinx_client int num_results; sphinx_result results [ MAX_REQS ]; + + int sock; ///< open socket for pconns; -1 if none }; ////////////////////////////////////////////////////////////////////////// @@ -193,7 +198,7 @@ sphinx_client * sphinx_create ( sphinx_bool copy_args ) client->local_error_buf[0] = '\0'; client->host = strchain ( client, "localhost" ); - client->port = 3312; + client->port = 9312; client->timeout = 0.0f; client->offset = 0; client->limit = 20; @@ -246,6 +251,7 @@ sphinx_client * sphinx_create ( sphinx_bool copy_args ) client->results[i].attr_types = NULL; } + client->sock = -1; return client; } @@ -271,6 +277,8 @@ static void sphinx_free_results ( sphinx_client * client ) } +void sock_close ( int sock ); + void sphinx_destroy ( sphinx_client * client ) { int i; @@ -291,6 +299,9 @@ void sphinx_destroy ( sphinx_client * client ) if ( client->response_buf ) free ( client->response_buf ); + if ( client->sock>=0 ) + sock_close ( client->sock ); + free ( client ); } @@ -1224,13 +1235,68 @@ void SPH_FD_SET ( int fd, fd_set * fdset ) { FD_SET ( fd, fdset ); } #endif +static sphinx_bool net_write ( int fd, const char * bytes, int len, sphinx_client * client ) +{ + int res; + res = send ( fd, bytes, len, 0 ); + + if ( res<0 ) + { + set_error ( client, "send() error: %s", sock_error() ); + return SPH_FALSE; + } + + if ( res!=len ) + { + set_error ( client, "send() error: incomplete write (len=%d, sent=%d)", len, res ); + return SPH_FALSE; + } + + return SPH_TRUE; +} + + +static sphinx_bool net_read ( int fd, char * buf, int len, sphinx_client * client ) +{ + int res, err; + for ( ;; ) + { + res = recv ( fd, buf, len, 0 ); + + if ( res<0 ) + { + err = sock_errno(); + if ( err==EINTR || err==EWOULDBLOCK ) // FIXME! remove non-blocking mode here; add timeout + continue; + set_error ( client, "recv(): read error (error=%s)", sock_error() ); + return SPH_FALSE; + } + + len -= res; + buf += res; + + if ( len==0 ) + return SPH_TRUE; + + if ( res==0 ) + { + set_error ( client, "recv(): incomplete read (len=%d, recv=%d)", len, res ); + return SPH_FALSE; + } + } +} + + static int net_connect ( sphinx_client * client ) { struct hostent * hp; struct sockaddr_in sa; struct timeval timeout; fd_set fds_write; - int sock, to_wait, res, err; + int sock, to_wait, res, err, my_proto; + + if ( client->sock>=0 ) + return client->sock; hp = gethostbyname ( client->host ); if ( !hp ) @@ -1287,6 +1353,31 @@ static int net_connect ( sphinx_client * client ) if ( res>=0 && FD_ISSET ( sock, &fds_write ) ) { sock_set_blocking ( sock ); + + // now send major client protocol version + my_proto = htonl ( 1 ); + if ( !net_write ( sock, (char*)&my_proto, sizeof(my_proto), client ) ) + { + sock_close ( sock ); + set_error ( client, "failed to send client protocol version" ); + return -1; + } + + // check daemon version + if ( !net_read ( sock, (char*)&my_proto, sizeof(my_proto), client ) ) + { + sock_close ( sock ); + return -1; + } + + my_proto = ntohl ( my_proto ); + if ( my_proto<1 ) + { + sock_close ( sock ); + set_error ( client, "expected searchd protocol version 1+, got version %d", my_proto ); + return -1; + } + return sock; } @@ -1299,58 +1390,6 @@ static int net_connect ( sphinx_client * client ) } -static sphinx_bool net_write ( int fd, const char * bytes, int len, sphinx_client * client ) -{ - int res; - res = send ( fd, bytes, len, 0 ); - - if ( res<0 ) - { - set_error ( client, "send() error: %s", sock_error() ); - return SPH_FALSE; - } - - if ( res!=len ) - { - set_error ( client, "send() error: incomplete write (len=%d, sent=%d)", len, res ); - return SPH_FALSE; - } - - return SPH_TRUE; -} - - -static sphinx_bool net_read ( int fd, char * buf, int len, sphinx_client * client ) -{ - int res, err; - for ( ;; ) - { - res = recv ( fd, buf, len, 0 ); - - if ( res<0 ) - { - err = sock_errno(); - if ( err==EINTR || err==EWOULDBLOCK ) // FIXME! remove non-blocking mode here; add timeout - continue; - set_error ( client, "recv(): read error (error=%s)", sock_error() ); - return SPH_FALSE; - } - - len -= res; - buf += res; - - if ( len==0 ) - return SPH_TRUE; - - if ( res==0 ) - { - set_error ( client, "recv(): incomplete read (len=%d, recv=%d)", len, res ); - return SPH_FALSE; - } - } -} - - static unsigned short unpack_short ( char ** cur ) { unsigned short v; @@ -1405,7 +1444,7 @@ static float unpack_float ( char ** cur ) static void net_get_response ( int fd, sphinx_client * client ) { - int i, len; + int len; char header_buf[32], *cur, *response; unsigned short status, ver; @@ -1418,11 +1457,15 @@ static void net_get_response ( int fd, sphinx_client * client ) } // read and parse the header - if ( !net_read ( fd, header_buf, 12, client ) ) + if ( !net_read ( fd, header_buf, 8, client ) ) + { + sock_close ( fd ); + if ( client->sock>0 ) + client->sock = -1; return; + } cur = header_buf; - i = unpack_int ( &cur ); // major searchd version status = unpack_short ( &cur ); ver = unpack_short ( &cur ); len = unpack_int ( &cur ); @@ -1445,6 +1488,8 @@ static void net_get_response ( int fd, sphinx_client * client ) if ( !net_read ( fd, response, len, client ) ) { sock_close ( fd ); + if ( client->sock>0 ) + client->sock = -1; free ( response ); return; } @@ -1474,8 +1519,52 @@ static void net_get_response ( int fd, sphinx_client * client ) break; } - // close the socket - sock_close ( fd ); + // close one-time socket on success + if ( client->sock<0 ) + sock_close ( fd ); +} + + +sphinx_bool sphinx_open ( sphinx_client * client ) +{ + char buf[16], *pbuf; + + if ( client->sock>=0 ) + { + set_error ( client, "already connected" ); + return SPH_FALSE; + } + + client->sock = net_connect ( client ); + if ( client->sock<0 ) + return SPH_FALSE; + + pbuf = buf; + send_word ( &pbuf, SEARCHD_COMMAND_PERSIST ); + send_word ( &pbuf, 0 ); // dummy version + send_int ( &pbuf, 4 ); // dummy body len + send_int ( &pbuf, 1 ); // dummy body + if ( !net_write ( client->sock, buf, (int)(pbuf-buf), client ) ) + { + sock_close ( client->sock ); + client->sock = -1; + return SPH_FALSE; + } + return SPH_TRUE; +} + + +sphinx_bool sphinx_close ( sphinx_client * client ) +{ + if ( client->sock<0 ) + { + set_error ( client, "not connected" ); + return SPH_FALSE; + } + + sock_close ( client->sock ); + client->sock = -1; + return SPH_TRUE; } @@ -1526,7 +1615,6 @@ sphinx_result * sphinx_run_queries ( sphinx_client * client ) len += client->req_lens[i]; req = req_header; - send_int ( &req, 1 ); // major protocol version send_word ( &req, SEARCHD_COMMAND_SEARCH ); send_word ( &req, client->ver_search ); send_int ( &req, len ); @@ -1734,7 +1822,7 @@ static sphinx_bool net_simple_query ( sphinx_client * client, char * buf, int re return SPH_FALSE; } - if ( !net_write ( fd, buf, 12+req_len, client ) ) + if ( !net_write ( fd, buf, 8+req_len, client ) ) { free ( buf ); return SPH_FALSE; @@ -1822,7 +1910,6 @@ char ** sphinx_build_excerpts ( sphinx_client * client, int num_docs, const char // build request req = buf; - send_int ( &req, 1 ); // major protocol version send_word ( &req, SEARCHD_COMMAND_EXCERPT ); send_word ( &req, VER_COMMAND_EXCERPT ); send_int ( &req, req_len ); @@ -1848,7 +1935,7 @@ char ** sphinx_build_excerpts ( sphinx_client * client, int num_docs, const char for ( i=0; iresponse_start; + pmax = client->response_start + client->response_len; // max position for checks, to protect against broken responses + + *num_rows = unpack_int ( &p ); + *num_cols = unpack_int ( &p ); + n = (*num_rows)*(*num_cols); + + res = (char**) malloc ( n*sizeof(char*) ); + for ( i=0; i_host = "localhost"; - $this->_port = 3312; + $this->_port = 9312; // per-query settings $this->_offset = 0; diff --git a/api/ruby/spec/sphinx/sphinx-id64.conf b/api/ruby/spec/sphinx/sphinx-id64.conf index 9b6d533a..027defea 100644 --- a/api/ruby/spec/sphinx/sphinx-id64.conf +++ b/api/ruby/spec/sphinx/sphinx-id64.conf @@ -57,7 +57,7 @@ indexer searchd { - port = 3312 + port = 9312 log = /opt/sphinx-0.9.9-id64/var/log/searchd.log query_log = /opt/sphinx-0.9.9-id64/var/log/query.log read_timeout = 5 diff --git a/api/ruby/spec/sphinx/sphinx.conf b/api/ruby/spec/sphinx/sphinx.conf index c792afaa..0721e605 100644 --- a/api/ruby/spec/sphinx/sphinx.conf +++ b/api/ruby/spec/sphinx/sphinx.conf @@ -57,7 +57,7 @@ indexer searchd { - port = 3312 + port = 9312 log = /opt/sphinx-0.9.9/var/log/searchd.log query_log = /opt/sphinx-0.9.9/var/log/query.log read_timeout = 5 diff --git a/api/ruby/test.rb b/api/ruby/test.rb index c81126b3..7b0685e9 100644 --- a/api/ruby/test.rb +++ b/api/ruby/test.rb @@ -1,14 +1,27 @@ -require File.dirname(__FILE__) + "/lib/sphinx" +# +# $Id$ +# -sphinx = Sphinx::Client.new -result = sphinx.Query("test", "test1") +require 'init.rb' -print "Found ", result["total_found"], " matches.\n\n" +q = ARGV.join(' ') +@sphinx = Sphinx::Client.new +# @sphinx.SetSortMode(Sphinx::Client::SPH_SORT_ATTR_ASC, 'created_at') +results = @sphinx.Query(q) + +puts "Query '#{q}' retrieved #{results['total']} of #{results['total_found']} matches in #{results['time']} sec."; +puts "Query stats:"; +results['words'].each do |word, info| + puts " '#{word}' found #{info['hits']} times in #{info['docs']} documents\n" +end +puts n = 1 -result['matches'].each do |m| - print "#{n}. id=#{m['id']}, weight=#{m['weight']}" - m['attrs'].each { |a| print ", #{a[0]}=#{a[1]}" } - print "\n" +results['matches'].each do |doc| + print "#{n}. doc_id=#{doc['id']}, weight=#{doc['weight']}" + doc['attrs'].each do |attr, value| + print ", #{attr}=#{value}" + end + puts n = n+1 end diff --git a/api/sphinxapi.php b/api/sphinxapi.php index a6d2b697..3b97fa93 100644 --- a/api/sphinxapi.php +++ b/api/sphinxapi.php @@ -349,6 +349,11 @@ function sphUnpackI64 ( $v ) $mq = floor($m/10000000.0); $l = $m - $mq*10000000.0 + $c; $h = $q*4294967296.0 + $r*429.0 + $mq; + if ( $l==10000000 ) + { + $l = 0; + $h += 1; + } $h = sprintf ( "%.0f", $h ); $l = sprintf ( "%07.0f", $l ); @@ -358,11 +363,27 @@ function sphUnpackI64 ( $v ) } +function sphFixUint ( $value ) +{ + if ( PHP_INT_SIZE>=8 ) + { + // x64 route, workaround broken unpack() in 5.2.2+ + if ( $value<0 ) $value += (1<<32); + return $value; + } + else + { + // x32 route, workaround php signed/unsigned braindamage + return sprintf ( "%u", $value ); + } +} + + /// sphinx searchd client class class SphinxClient { var $_host; ///< searchd host (default is "localhost") - var $_port; ///< searchd port (default is 3312) + var $_port; ///< searchd port (default is 9312) var $_offset; ///< how many records to seek from result-set start (default is 0) var $_limit; ///< how many records to return from result-set starting at offset (default is 20) var $_mode; ///< query matching mode (default is SPH_MATCH_ALL) @@ -406,7 +427,7 @@ function SphinxClient () { // per-client-object settings $this->_host = "localhost"; - $this->_port = 3312; + $this->_port = 9312; $this->_path = false; $this->_socket = false; @@ -534,8 +555,16 @@ function _MBPop () /// connect to searchd server function _Connect () { - if ( $this->_socket !== false ) - return $this->_socket; + if ( $this->_socket!==false ) + { + // we are in persistent connection mode, so we have a socket + // however, need to check whether it's still alive + if ( !@feof ( $this->_socket ) ) + return $this->_socket; + + // force reopen + $this->_socket = false; + } $errno = 0; $errstr = ""; @@ -1172,16 +1201,7 @@ function _ParseSearchResponse ( $response, $nreqs ) list ( $doc, $weight ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; - - if ( PHP_INT_SIZE>=8 ) - { - // x64 route, workaround broken unpack() in 5.2.2+ - if ( $doc<0 ) $doc += (1<<32); - } else - { - // x32 route, workaround php signed/unsigned braindamage - $doc = sprintf ( "%u", $doc ); - } + $doc = sphFixUint($doc); } $weight = sprintf ( "%u", $weight ); @@ -1220,7 +1240,7 @@ function _ParseSearchResponse ( $response, $nreqs ) while ( $nvalues-->0 && $p<$max ) { list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attrvals[$attr][] = sprintf ( "%u", $val ); + $attrvals[$attr][] = sphFixUint($val); } } else if ( $type==SPH_ATTR_STRING ) { @@ -1228,7 +1248,7 @@ function _ParseSearchResponse ( $response, $nreqs ) $p += $val; } else { - $attrvals[$attr] = sprintf ( "%u", $val ); + $attrvals[$attr] = sphFixUint($val); } } @@ -1640,4 +1660,4 @@ function FlushAttrs () // // $Id$ -// \ No newline at end of file +// diff --git a/api/sphinxapi.py b/api/sphinxapi.py index c8033f4b..76450983 100644 --- a/api/sphinxapi.py +++ b/api/sphinxapi.py @@ -108,7 +108,7 @@ def __init__ (self): Create a new client object, and fill defaults. """ self._host = 'localhost' # searchd host (default is "localhost") - self._port = 3312 # searchd port (default is 3312) + self._port = 9312 # searchd port (default is 9312) self._path = None # searchd unix-domain socket path self._socket = None self._offset = 0 # how much records to seek from result-set start (default is 0) @@ -181,7 +181,17 @@ def _Connect (self): INTERNAL METHOD, DO NOT CALL. Connects to searchd server. """ if self._socket: - return self._socket + # we have a socket, but is it still alive? + sr, sw, _ = select.select ( [self._socket], [self._socket], [], 0 ) + + # this is how alive socket should look + if len(sr)==0 and len(sw)==1: + return self._socket + + # oops, looks like it was closed, lets reopen + self._socket.close() + self._socket = None + try: if self._path: af = socket.AF_UNIX diff --git a/api/test.php b/api/test.php index b55dcd56..93348395 100644 --- a/api/test.php +++ b/api/test.php @@ -50,7 +50,7 @@ $sql = ""; $mode = SPH_MATCH_ALL; $host = "localhost"; -$port = 3312; +$port = 9312; $index = "*"; $groupby = ""; $groupsort = "@group desc"; diff --git a/api/test.py b/api/test.py index 0e08a34c..1741e294 100644 --- a/api/test.py +++ b/api/test.py @@ -25,7 +25,7 @@ q = '' mode = SPH_MATCH_ALL host = 'localhost' -port = 3312 +port = 9312 index = '*' filtercol = 'group_id' filtervals = [] diff --git a/configure.ac b/configure.ac index 0fbee49c..5dda40ae 100644 --- a/configure.ac +++ b/configure.ac @@ -240,6 +240,10 @@ else fi AM_CONDITIONAL(USE_PGSQL, test x$ac_cv_use_pgsql != xno) +# add macports include directory +if (echo $MYSQL_LIBS | grep -q -- -L/opt/local/lib); then + MYSQL_CFLAGS="$MYSQL_CFLAGS -I/opt/local/include" +fi # we can now set preprocessor flags for both C and C++ compilers CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS" diff --git a/contrib/perlapi/Sphinx.pm b/contrib/perlapi/Sphinx.pm deleted file mode 100644 index a949269a..00000000 --- a/contrib/perlapi/Sphinx.pm +++ /dev/null @@ -1,560 +0,0 @@ -# -# Copyright (c) 2001-2006, Len Kranendonk. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License. You should have -# received a copy of the GPL license along with this program; if you -# did not, you can find it at http://www.gnu.org/ -# - -#------------------------------------------------------------- -# Sphinx Perl searchd client API -#------------------------------------------------------------- - -package Sphinx; - -use strict; -use Carp; -use Socket; -use base 'Exporter'; - -# Constants to export. -our @EXPORT = qw( - SPH_MATCH_ALL SPH_MATCH_ANY SPH_MATCH_PHRASE SPH_MATCH_BOOLEAN SPH_MATCH_EXTENDED - SPH_SORT_RELEVANCE SPH_SORT_ATTR_DESC SPH_SORT_ATTR_ASC SPH_SORT_TIME_SEGMENTS SPH_SORT_EXTENDED - SPH_GROUPBY_DAY SPH_GROUPBY_WEEK SPH_GROUPBY_MONTH SPH_GROUPBY_YEAR SPH_GROUPBY_ATTR - ); - -# known searchd commands -use constant SEARCHD_COMMAND_SEARCH => 0; -use constant SEARCHD_COMMAND_EXCERPT => 1; - -# current client-side command implementation versions -use constant VER_COMMAND_SEARCH => 0x104; -use constant VER_COMMAND_EXCERPT => 0x100; - -# known searchd status codes -use constant SEARCHD_OK => 0; -use constant SEARCHD_ERROR => 1; -use constant SEARCHD_RETRY => 2; - -# known match modes -use constant SPH_MATCH_ALL => 0; -use constant SPH_MATCH_ANY => 1; -use constant SPH_MATCH_PHRASE => 2; -use constant SPH_MATCH_BOOLEAN => 3; -use constant SPH_MATCH_EXTENDED => 4; - -# known sort modes -use constant SPH_SORT_RELEVANCE => 0; -use constant SPH_SORT_ATTR_DESC => 1; -use constant SPH_SORT_ATTR_ASC => 2; -use constant SPH_SORT_TIME_SEGMENTS => 3; -use constant SPH_SORT_EXTENDED => 4; - -# known attribute types -use constant SPH_ATTR_INTEGER => 1; -use constant SPH_ATTR_TIMESTAMP => 2; - -# known grouping functions -use constant SPH_GROUPBY_DAY => 0; -use constant SPH_GROUPBY_WEEK => 1; -use constant SPH_GROUPBY_MONTH => 2; -use constant SPH_GROUPBY_YEAR => 3; -use constant SPH_GROUPBY_ATTR => 4; - - -#------------------------------------------------------------- -# common stuff -#------------------------------------------------------------- - -# create a new client object and fill defaults -sub new { - my $class = shift; - my $self = { - _host => 'localhost', - _port => 3312, - _offset => 0, - _limit => 20, - _mode => SPH_MATCH_ALL, - _weights => [], - _sort => SPH_SORT_RELEVANCE, - _sortby => "", - _min_id => 0, - _max_id => 0xFFFFFFFF, - _min => {}, - _max => {}, - _filter => {}, - _groupby => "", - _groupfunc => SPH_GROUPBY_DAY, - _maxmatches => 1000, - _error => '', - _warning => '', - }; - bless($self, $class); - return $self; -} - -# get last error message (string) -sub GetLastError { - my $self = shift; - return $self->{_error}; -} - -# get last warning message (string) -sub GetLastWarning { - my $self = shift; - return $self->{_warning}; -} - -# set searchd server -sub SetServer { - my $self = shift; - my $host = shift; - my $port = shift; - - croak("host is not defined") unless defined($host); - croak("port is not defined") unless defined($port); - - $self->{_host} = $host; - $self->{_port} = $port; -} - -#------------------------------------------------------------- - -# connect to searchd server - -sub _Connect { - my $self = shift; - - # connect socket - my $fp; - socket($fp, PF_INET, SOCK_STREAM, getprotobyname('tcp')) || Carp::croak("socket: ".$!); - my $dest = sockaddr_in($self->{_port}, inet_aton($self->{_host})); - connect($fp, $dest); - - if($!) { - $self->{_error} = "connection to {$self->{_host}}:{$self->{_port}} failed: $!"; - return 0; - } - - # check version - my $buf = ''; - recv($fp, $buf, 4, 0) ne "" || croak("recv: ".$!); - my $v = unpack("N*", $buf); - $v = int($v); - if($v < 1) { - close($fp) || croak("close: $!"); - $self->{_error} = "expected searchd protocol version 1+, got version '$v'"; - } - - # All ok, send my version - send($fp, pack("N", 1),0); - return $fp; -} - -#------------------------------------------------------------- - -# get and check response packet from searchd server -sub _GetResponse { - my $self = shift; - my $fp = shift; - my $client_ver = shift; - - my $header; - recv($fp, $header, 8, 0) ne "" || croak("recv: ".$!); - - my ($status, $ver, $len ) = unpack("n2N", $header); - my ($chunk, $response); - while(defined($chunk = <$fp>)) { - $response .= $chunk; - } - close ( $fp ); - - # check response - if ( !$response || length($response) != $len ) { - $self->{_error} = $len - ? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=". length($response) . ")" - : "received zero-sized searchd response"; - return 0; - } - - # check status - if ( $status==SEARCHD_ERROR ) { - $self->{_error} = "searchd error: " . substr ( $response, 4 ); - return 0; - } - if ( $status==SEARCHD_RETRY ) { - $self->{_error} = "temporary searchd error: " . substr ( $response, 4 ); - return 0; - } - if ( $status!=SEARCHD_OK ) { - $self->{_error} = "unknown status code '$status'"; - return 0; - } - - # check version - if ( $ver<$client_ver ) { - $self->{_warning} = sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work", - $ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ); - } - return $response; -} - - -#------------------------------------------------------------- -# searching -#------------------------------------------------------------- - -# set match offset/limits -sub SetLimits { - my $self = shift; - my $offset = shift; - my $limit = shift; - my $max = shift || 0; - croak("offset should be an integer >= 0") unless ($offset =~ /^\d+$/ && $offset >= 0) ; - croak("limit should be an integer >= 0") unless ($limit =~ /^\d+/ && $offset >= 0); - $self->{_offset} = $offset; - $self->{_limit} = $limit; - if($max > 0) { - $self->{_maxmatches} = $max; - } -} - -# set match mode -sub SetMatchMode { - my $self = shift; - my $mode = shift; - croak("Match mode not defined") unless defined($mode); - croak("Unknown matchmode: $mode") unless ( $mode==SPH_MATCH_ALL || $mode==SPH_MATCH_ANY - || $mode==SPH_MATCH_PHRASE || $mode==SPH_MATCH_BOOLEAN || $mode==SPH_MATCH_EXTENDED ); - $self->{_mode} = $mode; -} - -# set sort mode -sub SetSortMode { - my $self = shift; - my $mode = shift; - my $sortby = shift; - croak("Sort mode not defined") unless defined($mode); - croak("Unknown sort mode: $mode") unless ( $mode==SPH_SORT_RELEVANCE - || $mode==SPH_SORT_ATTR_DESC || $mode==SPH_SORT_ATTR_ASC - || $mode==SPH_SORT_TIME_SEGMENTS || $mode==SPH_SORT_EXTENDED ); - croak("Sortby must be defined") unless ($mode==SPH_SORT_RELEVANCE || length($sortby)); - $self->{_sort} = $mode; - $self->{_sortby} = $sortby; -} - -# set per-field weights -sub SetWeights { - my $self = shift; - my $weights = shift; - croak("Weights is not an array reference") unless (ref($weights) eq 'ARRAY'); - foreach my $weight (@$weights) { - croak("Weight: $weight is not an integer") unless ($weight =~ /^\d+$/); - } - $self->{_weights} = $weights; -} - -# set IDs range to match -# only match those records where document ID -# is beetwen $min and $max (including $min and $max) -sub SetIDRange { - my $self = shift; - my $min = shift; - my $max = shift; - croak("min_id is not an integer") unless ($min =~ /^\d+$/); - croak("max_id is not an integer") unless ($max =~ /^\d+$/); - croak("min_id is larger than or equal to max_id") unless ($min < $max); - $self->{_min_id} = $min; - $self->{_max_id} = $max; -} - -sub SetFilter { - my $self = shift; - my $attribute = shift; - my $values = shift; - croak("attribute is not defined") unless (defined $attribute); - croak("values is not an array reference") unless (ref($values) eq 'ARRAY'); - croak("values reference is empty") unless (scalar(@$values)); - - foreach my $value (@$values) { - croak("value $value is not an integer") unless ($value =~ /^\d+$/); - } - $self->{_filter}{$attribute} = $values; -} - -# set range filter -# only match those records where $attribute column value -# is beetwen $min and $max (including $min and $max) -sub SetFilterRange { - my $self = shift; - my $attribute = shift; - my $min = shift; - my $max = shift; - croak("attribute is not defined") unless (defined $attribute); - croak("min: $min is not an integer") unless ($min =~ /^\d+$/); - croak("max: $max is not an integer") unless ($max =~ /^\d+$/); - croak("min value should be <= max") unless ($min <= $max); - - $self->{_min}{$attribute} = $min; - $self->{_max}{$attribute} = $max; -} - -# set grouping -# if grouping -sub SetGroupBy { - my $self = shift; - my $attribute = shift; - my $func = shift; - croak("attribute is not defined") unless (defined $attribute); - croak("Unknown grouping function: $func") unless ($func==SPH_GROUPBY_DAY - || $func==SPH_GROUPBY_WEEK - || $func==SPH_GROUPBY_MONTH - || $func==SPH_GROUPBY_YEAR - || $func==SPH_GROUPBY_ATTR ); - - $self->{_groupby} = $attribute; - $self->{_groupfunc} = $func; -} - -# connect to searchd server and run given search query -# -# $query is query string -# $query is index name to query, default is "*" which means to query all indexes -# -# returns false on failure -# returns hash which has the following keys on success: -# "matches" -# array containing hashes with found documents ( "doc", "weight", "group", "stamp" ) -# "total" -# total amount of matches retrieved (upto SPH_MAX_MATCHES, see sphinx.h) -# "total_found" -# total amount of matching documents in index -# "time" -# search time -# "words" -# hash which maps query terms (stemmed!) to ( "docs", "hits" ) hash -sub Query { - my $self = shift; - my $query = shift; - my $index = shift; - $index ||= "*"; - - my $fp = $self->_Connect(); - return 0 unless ($fp); - - ################## - # build request - ################## - - my $req; - $req = pack ( "NNNN", $self->{_offset}, $self->{_limit}, $self->{_mode}, $self->{_sort} ); # mode and limits - $req .= pack ( "N", length($self->{_sortby}) ) . $self->{_sortby}; - $req .= pack ( "N", length($query) ) . $query; # query itself - $req .= pack ( "N", scalar(@{$self->{_weights}}) ); # weights - foreach my $weight (@{$self->{_weights}}) { - $req .= pack ( "N", int($weight)); - } - $req .= pack ( "N", length($index) ) . $index; # indexes - $req .= # id range - pack ( "N", int($self->{_min_id}) ) . - pack ( "N", int($self->{_max_id}) ); - - # filters - $req .= pack ( "N", scalar(keys %{$self->{_min}}) + scalar(keys %{$self->{_filter}}) ); - - foreach my $attr (keys %{$self->{_min}}) { - $req .= - pack ( "N", length($attr) ) . $attr . - pack ( "NNN", 0, $self->{_min}{$attr}, $self->{_max}{$attr} ); - } - - foreach my $attr (keys %{$self->{_filter}}) { - my $values = $self->{_filter}{$attr}; - $req .= - pack ( "N", length($attr) ) . $attr . - pack ( "N", scalar(@$values) ); - - foreach my $value ( @$values ) { - $req .= pack ( "N", $value ); - } - } - - # group-by - $req .= pack ( "NN", $self->{_groupfunc}, length($self->{_groupby}) ) . $self->{_groupby}; - - # max matches to retrieve - $req .= pack ( "N", $self->{_maxmatches} ); - - ################## - # send query, get response - ################## - - my $len = length($req); - $req = pack ( "nnN", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $len ) . $req; # add header - send($fp, $req ,0); - - my $response = $self->_GetResponse ( $fp, VER_COMMAND_SEARCH ); - return 0 unless ($response); - - ################## - # parse response - ################## - - my $result = {}; # Empty hash ref - $result->{matches} = []; # Empty array ref - my $max = length($response); # Protection from broken response - - # read schema - my $p = 0; - my @fields; - my (%attrs, @attr_list); - - my $nfields = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - while ( $nfields-->0 && $p<$max ) { - my $len = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - push(@fields, substr ( $response, $p, $len )); $p += $len; - } - $result->{"fields"} = \@fields; - - my $nattrs = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - while ( $nattrs-->0 && $p<$max ) { - my $len = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - my $attr = substr ( $response, $p, $len ); $p += $len; - my $type = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attrs{$attr} = $type; - push(@attr_list, $attr); - } - $result->{"attrs"} = \%attrs; - - # read match count - my $count = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - - # read matches - while ( $count-->0 && $p<$max ) { - my $data = {}; - ( $data->{doc}, $data->{weight} ) = unpack("N*N*", substr($response,$p,8)); - $p += 8; - - foreach my $attr (@attr_list) { - $data->{$attr} = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - } - push(@{$result->{matches}}, $data); - } - my $words; - ($result->{total}, $result->{total_found}, $result->{time}, $words) = unpack("N*N*N*N*", substr($response, $p, 16)); - $result->{time} = sprintf ( "%.3f", $result->{"time"}/1000 ); - $p += 16; - - while ( $words-->0 ) { - my $len = unpack ( "N*", substr ( $response, $p, 4 ) ); - $p += 4; - my $word = substr ( $response, $p, $len ); - $p += $len; - my ($docs, $hits) = unpack ("N*N*", substr($response, $p, 8)); - $p += 8; - $result->{words}{$word} = { - "docs" => $docs, - "hits" => $hits - }; - } - return $result; -} - -#------------------------------------------------------------- -# excerpts generation -#------------------------------------------------------------- - -# connect to searchd server and generate exceprts from given documents -# -# $index is a string specifiying the index which settings will be used -# for stemming, lexing and case folding -# $docs is an array reference of strings which represent the documents' contents -# $words is a string which contains the words to highlight -# $opts is a hash which contains additional optional highlighting parameters: -# "before_match" -# a string to insert before a set of matching words, default is "" -# "after_match" -# a string to insert after a set of matching words, default is "" -# "chunk_separator" -# a string to insert between excerpts chunks, default is " ... " -# "limit" -# max excerpt size in symbols (codepoints), default is 256 -# "around" -# how much words to highlight around each match, default is 5 -# -# returns false on failure -# retrurns an array of string excerpts on success -sub BuildExcerpts { - my ($self, $docs, $index, $words, $opts) = @_; - $opts ||= {}; - croak("BuildExcepts() called with incorrect parameters") unless (ref($docs) eq 'ARRAY' - && defined($index) - && defined($words) - && ref($opts) eq 'HASH'); - my $fp = $self->_Connect(); - return 0 unless ($fp); - - ################## - # fixup options - ################## - $opts->{"before_match"} ||= ""; - $opts->{"after_match"} ||= ""; - $opts->{"chunk_separator"} ||= " ... "; - $opts->{"limit"} ||= 256; - $opts->{"around"} ||= 5; - - ################## - # build request - ################## - - # v.1.0 req - my $req; - $req = pack ( "NN", 0, 1 ); # mode=0, flags=1 (remove spaces) - $req .= pack ( "N", length($index) ) . $index; # req index - $req .= pack ( "N", length($words) ) . $words; # req words - - # options - $req .= pack ( "N", length($opts->{"before_match"}) ) . $opts->{"before_match"}; - $req .= pack ( "N", length($opts->{"after_match"}) ) . $opts->{"after_match"}; - $req .= pack ( "N", length($opts->{"chunk_separator"}) ) . $opts->{"chunk_separator"}; - $req .= pack ( "N", int($opts->{"limit"}) ); - $req .= pack ( "N", int($opts->{"around"}) ); - - # documents - $req .= pack ( "N", scalar(@$docs) ); - foreach my $doc (@$docs) { - croak('BuildExcepts: Found empty document in $docs') unless ($doc); - $req .= pack("N", length($doc)) . $doc; - } - - ########################## - # send query, get response - ########################## - - my $len = length($req); - $req = pack ( "nnN", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $len ) . $req; # add header - send($fp, $req ,0); - - my $response = $self->_GetResponse($fp, VER_COMMAND_EXCERPT); - return 0 unless ($response); - - my ($pos, $i) = 0; - my $res = []; # Empty hash ref - my $rlen = length($response); - for ( $i=0; $i< scalar(@$docs); $i++ ) { - my $len = unpack ( "N*", substr ( $response, $pos, 4 ) ); - $pos += 4; - - if ( $pos+$len > $rlen ) { - $self->_error = "incomplete reply"; - return 0; - } - push(@$res, substr ( $response, $pos, $len )); - $pos += $len; - } - return $res; -} - -1; diff --git a/contrib/perlapi/excerpts.pl b/contrib/perlapi/excerpts.pl deleted file mode 100644 index 28c201e5..00000000 --- a/contrib/perlapi/excerpts.pl +++ /dev/null @@ -1,29 +0,0 @@ -use strict; -use Sphinx; - -my $spx = Sphinx->new; -$spx->SetServer('localhost' , 3312); - -my @docs = ( - "this is my test text to be highlighted", - "this is another test text to be highlighted", -); -my $words = "test text"; -my $index = "test1"; -my %opts = ( - "before_match" => "", - "after_match" => "", - "chunk_separator" => " ... ", - "limit" => 400, - "around" => 15 -); -my $res = $spx->BuildExcerpts(\@docs,$index,$words,\%opts ); -if ( !$res ) { - die ( "ERROR: " . $spx->GetLastError() . ".\n" ); -} else { - my $n; - foreach my $entry (@$res) { - $n++; - print "n=$n, res=$entry\n"; - } -} diff --git a/contrib/perlapi/search.pl b/contrib/perlapi/search.pl deleted file mode 100644 index 0ae7c0ef..00000000 --- a/contrib/perlapi/search.pl +++ /dev/null @@ -1,51 +0,0 @@ -use strict; -use Sphinx; - -my $spx = Sphinx->new; - -# SetServer(host, port) -$spx->SetServer('localhost' , 3312); - -# SetLimits(offset, limit, [max]) -# Limit to 10 results, with offset=20 -# $spx->SetLimits(20,10); - -# SetMatchMode(mode) -# mode can be SPH_MATCH_ALL, SPH_MATCH_ANY, SPH_MATCH_PHRASE, or SPH_MATCH_BOOLEAN. -# $spx->SetMatchMode(SPH_MATCH_ANY); - -# SetSortMode(mode, [sortby]) -# sort can be SPH_SORT_RELEVANCE, SPH_SORT_ATTR_DESC, SPH_SORT_ATTR_ASC or SPH_SORT_TIME_SEGMENTS -# sortby is the attribute to sort by. -# $spx->SetSortMode(SPH_SORT_ATTR_DESC, "date_added"); - -# SetWeights(values) -# $spx->SetWeights([4,1,3]); - -# SetIDRange(min_id, max_id) -# $spx->SetIDRange(0,1000); - -# SetFilter(attr, values) -# $spx->SetFilter('group_id', [1961,4228]); - -# SetFilterRange(attr, min, max) -# $spx->SetFilterRange('group_id', 10, 1000); - -# SetGroupBy(attr, func) -# func can be SPH_GROUPBY_DAY SPH_GROUPBY_WEEK SPH_GROUPBY_MONTH SPH_GROUPBY_YEAR SPH_GROUPBY_ATTR -# $spx->SetGroupBy('group_id', SPH_GROUPBY_ATTR); - -my $results = $spx->Query('test', 'lj1'); -print "Total documents found: " . $results->{total_found} . "\n"; -print "Search took: " . $results->{time} . "s\n"; - -foreach my $match (@{$results->{matches}}) { - # Print doc id and weight - print "Doc: $match->{doc}\tWeight: $match->{weight}"; - - # Print attributes for this result - foreach my $attr (keys %{$results->{attrs}}) { - print "\t" . $attr . ": " . $match->{$attr}; - } - print "\n"; -} diff --git a/doc/sphinx.txt b/doc/sphinx.txt index 210e16d3..d978c8b3 100644 --- a/doc/sphinx.txt +++ b/doc/sphinx.txt @@ -981,9 +981,11 @@ Example 3. xmlpipe2 document stream | | | - | this is the main content entry must be handled properly by xml parser lib]]> + | this is the main content entry + | must be handled properly by xml parser lib]]> | 1012325463 - | note how field/attr tags can be in randomized order + | note how field/attr tags can be + | in randomized order | some undeclared element | | @@ -1804,7 +1806,8 @@ of data after where the index(es) searched are listed. A query log entry might take the form of: - | [Fri Jun 29 21:17:58 2007] 0.004 sec [all/0/rel 35254 (0,20)] [lj] [ios=6 kb=111.1 ms=0.5] test + | [Fri Jun 29 21:17:58 2007] 0.004 sec [all/0/rel 35254 (0,20)] [lj] + | [ios=6 kb=111.1 ms=0.5] test This additional block is information regarding I/O operations in performing the search: the number of file I/O operations carried out, the amount of @@ -1820,7 +1823,7 @@ binary network protocol and can be accessed with regular MySQL API. For instance, 'mysql' CLI client program works well. Here's an example of querying Sphinx using MySQL client: - | $ mysql -P 3307 + | $ mysql -P 9306 | Welcome to the MySQL monitor. Commands end with ; or \g. | Your MySQL connection id is 1 | Server version: 0.9.9-dev (r1734) @@ -1848,7 +1851,7 @@ protocol support needs to be additionally configured. This is a matter of 1-line config change, adding a new listener with mysql41 specified as a protocol: - | listen = localhost:3307:mysql41 + | listen = localhost:9306:mysql41 Just supporting the protocol and not the SQL syntax would be useless so Sphinx now also supports a tiny subset of SQL that we dubbed SphinxQL. @@ -1942,7 +1945,8 @@ latest query. The error message will be returned along with the query itself: | mysql> SELECT * FROM test1 WHERE MATCH('@@title hello') \G - | ERROR 1064 (42000): index test1: syntax error, unexpected TOK_FIELDLIMIT near '@title hello' + | ERROR 1064 (42000): index test1: syntax error, unexpected TOK_FIELDLIMIT + | near '@title hello' | | mysql> SELECT * FROM test1 WHERE MATCH('@title -hello') \G | ERROR 1064 (42000): index test1: query is non-computable (single NOT operator) @@ -1959,7 +1963,8 @@ itself: | *************************** 1. row *************************** | Level: warning | Code: 1000 - | Message: quorum threshold too high (words=2, thresh=3); replacing quorum operator with AND operator + | Message: quorum threshold too high (words=2, thresh=3); replacing quorum operator + | with AND operator | 1 row in set (0.00 sec) SHOW STATUS shows a number of useful performance counters. IO and CPU @@ -2388,13 +2393,13 @@ The options available to searchd on all builds are: * --port portnumber (-p for short) is used to specify the post that searchd should listen on, usually for debugging purposes. This will - usually default to 3312, but sometimes you need to run it on + usually default to 9312, but sometimes you need to run it on a different port. Specifying it on the command line will override anything specified in the configuration file. The valid range is 0 to 65535, but ports numbered 1024 and below usually require a privileged account in order to run. An example of usage: - | $ searchd --port 3313 + | $ searchd --port 9313 * --index forces this instance of searchd only to serve the specified index. Like --port, above, this is usually for debugging @@ -2421,13 +2426,15 @@ install it as a service. console, so any methods you could use for starting, stopping and restarting services would also apply to searchd. Example: - | C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install --config C:\Sphinx\sphinx.conf + | C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install + | --config C:\Sphinx\sphinx.conf If you wanted to have the I/O stats every time you started searchd, you would specify its option on the same line as the --install command thus: - | C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install --config C:\Sphinx\sphinx.conf --iostats + | C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install + | --config C:\Sphinx\sphinx.conf --iostats * --delete removes the service from the Microsoft Management Console and other places where services are registered, after previously installed @@ -2450,7 +2457,8 @@ install it as a service. Note that unless combined with --install or --delete, this option does not do anything. Example: - | C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install --config C:\Sphinx\sphinx.conf --servicename SphinxSearch + | C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install + | --config C:\Sphinx\sphinx.conf --servicename SphinxSearch * --ntservice is the option that is passed by the Management Console to searchd to invoke it as a service on Windows platforms. It would not @@ -2749,7 +2757,7 @@ several times if needed. Prototype: function SetServer ( $host, $port ) Sets searchd host name and TCP port. All subsequent requests will use the -new host and port settings. Default host and port are 'localhost' and 3312, +new host and port settings. Default host and port are 'localhost' and 9312, respectively. 6.1.4. SetRetries @@ -2924,7 +2932,8 @@ using this computed expressions mechanism, using magic names '@expr' and Example: | $cl->SetSelect ( "*, @weight+(user_karma+ln(pageviews))*0.1 AS myweight" ); - | $cl->SetSelect ( "exp_years, salary_gbp*{$gbp_usd_rate} AS salary_usd, IF(age>40,1,0) AS over40" ); + | $cl->SetSelect ( "exp_years, salary_gbp*{$gbp_usd_rate} AS salary_usd, + | IF(age>40,1,0) AS over40" ); | $cl->SetSelect ( "*, AVG(price) AS avgprice" ); 6.3. Full-text search query settings @@ -3679,17 +3688,16 @@ newly built servers, run mysql client and issue SHOW ENGINES query. You should see a list of all available engines. Sphinx should be present and "Support" column should contain "YES": - | | mysql> show engines; - | +------------+----------+----------------------------------------------------------------+ - | | Engine | Support | Comment | - | +------------+----------+----------------------------------------------------------------+ - | | MyISAM | DEFAULT | Default engine as of MySQL 3.23 with great performance | + | +------------+----------+-------------------------------------------------------------+ + | | Engine | Support | Comment | + | +------------+----------+-------------------------------------------------------------+ + | | MyISAM | DEFAULT | Default engine as of MySQL 3.23 with great performance | | ... - | | SPHINX | YES | Sphinx storage engine | + | | SPHINX | YES | Sphinx storage engine | | ... - | +------------+----------+----------------------------------------------------------------+ - | 13 rows in set (0.00 sec) + | +------------+----------+-------------------------------------------------------------+ + | 13 rows in set (0.00 sec) 7.3. Using SphinxSE ------------------- @@ -3707,7 +3715,7 @@ Let's begin with an example create statement and search query: | query VARCHAR(3072) NOT NULL, | group_id INTEGER, | INDEX(query) - | ) ENGINE=SPHINX CONNECTION="sphinx://localhost:3312/test"; + | ) ENGINE=SPHINX CONNECTION="sphinx://localhost:9312/test"; | | SELECT * FROM t1 WHERE query='test it;mode=any'; @@ -3729,7 +3737,7 @@ _sph_groupby, _sph_count or _sph_distinct column names, respectively. CONNECTION string parameter can be used to specify default searchd host, port and indexes for queries issued using this table. If no connection string is specified in CREATE TABLE, index name "*" (ie. search all -indexes) and localhost:3312 are assumed. Connection string syntax is as +indexes) and localhost:9312 are assumed. Connection string syntax is as follows: | CONNECTION="sphinx://HOST:PORT/INDEXNAME" @@ -5601,9 +5609,9 @@ process each query: | { | type = distributed | local = chunk1 - | agent = localhost:3312:chunk2 - | agent = localhost:3312:chunk3 - | agent = localhost:3312:chunk4 + | agent = localhost:9312:chunk2 + | agent = localhost:9312:chunk3 + | agent = localhost:9312:chunk4 | } Note how one of the chunks is searched locally and the same instance of @@ -5612,9 +5620,9 @@ parallel. Example: - | agent = localhost:3312:chunk2 # contact itself + | agent = localhost:9312:chunk2 # contact itself | agent = /var/run/searchd.s:chunk2 - | agent = searchbox2:3312:chunk3,chunk4 # search remote indexes + | agent = searchbox2:9312:chunk3,chunk4 # search remote indexes 9.2.29. agent_blackhole ----------------------- @@ -5634,7 +5642,7 @@ directive. Example: - | agent_blackhole = testbox:3312:testindex1,testindex2 + | agent_blackhole = testbox:9312:testindex1,testindex2 9.2.30. agent_connect_timeout ----------------------------- @@ -5957,13 +5965,13 @@ Examples: | listen = localhost:5000 | listen = 192.168.0.1:5000 | listen = /var/run/sphinx.s - | listen = 3312 - | listen = localhost:3307:mysql41 + | listen = 9312 + | listen = localhost:9306:mysql41 There can be multiple listen directives, searchd will listen for client connections on all specified ports and sockets. If no listen directive is found then the server will listen on all available interfaces using the -default port (which is 3312). +default port (which is 9312). Unix-domain sockets are not supported on Windows. @@ -5986,11 +5994,11 @@ Example: ----------- searchd TCP port number. DEPRECATED, use listen instead. Used to be -mandatory. Default port number is 3312. +mandatory. Default port number is 9312. Example: - | port = 3312 + | port = 9312 9.4.4. log ---------- diff --git a/doc/sphinx.xml b/doc/sphinx.xml index 191ff351..8a039ea2 100644 --- a/doc/sphinx.xml +++ b/doc/sphinx.xml @@ -1697,7 +1697,8 @@ Sort mode can take one of the following values: Additionally, if searchd was started with , there will be a block of data after where the index(es) searched are listed. A query log entry might take the form of: -[Fri Jun 29 21:17:58 2007] 0.004 sec [all/0/rel 35254 (0,20)] [lj] [ios=6 kb=111.1 ms=0.5] test +[Fri Jun 29 21:17:58 2007] 0.004 sec [all/0/rel 35254 (0,20)] [lj] + [ios=6 kb=111.1 ms=0.5] test This additional block is information regarding I/O operations in performing the search: the number of file I/O operations carried out, the amount of data in kilobytes read from the index files and time spent on I/O operations (although there is a background processing component, the bulk of this time is the I/O operation time) @@ -1711,7 +1712,7 @@ network protocol and can be accessed with regular MySQL API. For instance, 'mysql' CLI client program works well. Here's an example of querying Sphinx using MySQL client: -$ mysql -P 3307 +$ mysql -P 9306 Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 1 Server version: 0.9.9-dev (r1734) @@ -1743,7 +1744,7 @@ configured. This is a matter of 1-line config change, adding a new listener with mysql41 specified as a protocol: -listen = localhost:3307:mysql41 +listen = localhost:9306:mysql41 @@ -1851,7 +1852,8 @@ produced by the latest query. The error message will be returned along with the query itself: mysql> SELECT * FROM test1 WHERE MATCH('@@title hello') \G -ERROR 1064 (42000): index test1: syntax error, unexpected TOK_FIELDLIMIT near '@title hello' +ERROR 1064 (42000): index test1: syntax error, unexpected TOK_FIELDLIMIT +near '@title hello' mysql> SELECT * FROM test1 WHERE MATCH('@title -hello') \G ERROR 1064 (42000): index test1: query is non-computable (single NOT operator) @@ -1868,7 +1870,8 @@ mysql> SHOW WARNINGS \G *************************** 1. row *************************** Level: warning Code: 1000 -Message: quorum threshold too high (words=2, thresh=3); replacing quorum operator with AND operator +Message: quorum threshold too high (words=2, thresh=3); replacing quorum operator + with AND operator 1 row in set (0.00 sec) @@ -2204,9 +2207,9 @@ call and might therefore be unavailable on certain systems. You might start $ searchd --config /home/myuser/sphinx.conf --cpustats - ( for short) is used to specify the post that searchd should listen on, usually for debugging purposes. This will usually default to 3312, but sometimes you need to run it on a different port. Specifying it on the command line will override anything specified in the configuration file. The valid range is 0 to 65535, but ports numbered 1024 and below usually require a privileged account in order to run. An example of usage: + ( for short) is used to specify the post that searchd should listen on, usually for debugging purposes. This will usually default to 9312, but sometimes you need to run it on a different port. Specifying it on the command line will override anything specified in the configuration file. The valid range is 0 to 65535, but ports numbered 1024 and below usually require a privileged account in order to run. An example of usage: -$ searchd --port 3313 +$ searchd --port 9313 forces this instance of searchd only to serve the specified index. Like , above, this is usually for debugging purposes; more long-term changes would generally be applied to the configuration file itself. Example usage: @@ -2220,11 +2223,13 @@ $ searchd --index myindex installs searchd as a service into the Microsoft Management Console (Control Panel / Administrative Tools / Services). Any other parameters specified on the command line, where is specified will also become part of the command line on future starts of the service. For example, as part of calling searchd, you will likely also need to specify the configuration file with , and you would do that as well as specifying . Once called, the usual start/stop facilities will become available via the management console, so any methods you could use for starting, stopping and restarting services would also apply to searchd. Example: -C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install --config C:\Sphinx\sphinx.conf +C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install + --config C:\Sphinx\sphinx.conf If you wanted to have the I/O stats every time you started searchd, you would specify its option on the same line as the command thus: -C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install --config C:\Sphinx\sphinx.conf --iostats +C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install + --config C:\Sphinx\sphinx.conf --iostats removes the service from the Microsoft Management Console and other places where services are registered, after previously installed with . Note, this does not uninstall the software or delete the indexes. It means the service will not be called from the services systems, and will not be started on the machine's next start. If currently running as a service, the current instance will not be terminated (until the next reboot, or searchd is called with ). If the service was installed with a custom name (with ), the same name will need to be specified with when calling to uninstall. Example: @@ -2234,7 +2239,8 @@ C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --delete applies the given name to searchd when installing or deleting the service, as would appear in the Management Console; this will default to searchd, but if being deployed on servers where multiple administrators may log into the system, or a system with multiple searchd instances, a more descriptive name may be applicable. Note that unless combined with or , this option does not do anything. Example: -C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install --config C:\Sphinx\sphinx.conf --servicename SphinxSearch +C:\WINDOWS\system32> C:\Sphinx\bin\searchd.exe --install + --config C:\Sphinx\sphinx.conf --servicename SphinxSearch is the option that is passed by the Management Console to searchd to invoke it as a service on Windows platforms. It would not normally be necessary to call this directly; this would normally be called by Windows when the service would be started, although if you wanted to call this as a regular service from the command-line (as the complement to ) you could do so in theory. @@ -2452,7 +2458,7 @@ call it several times if needed. Sets searchd host name and TCP port. All subsequent requests will use the new host and port settings. -Default host and port are 'localhost' and 3312, respectively. +Default host and port are 'localhost' and 9312, respectively. @@ -2640,7 +2646,8 @@ respectively. Example: $cl->SetSelect ( "*, @weight+(user_karma+ln(pageviews))*0.1 AS myweight" ); -$cl->SetSelect ( "exp_years, salary_gbp*{$gbp_usd_rate} AS salary_usd, IF(age>40,1,0) AS over40" ); +$cl->SetSelect ( "exp_years, salary_gbp*{$gbp_usd_rate} AS salary_usd, + IF(age>40,1,0) AS over40" ); $cl->SetSelect ( "*, AVG(price) AS avgprice" ); @@ -3512,17 +3519,17 @@ issue SHOW ENGINES query. You should see a list of all available engines. Sphinx should be present and "Support" column should contain "YES": - + mysql> show engines; -+------------+----------+----------------------------------------------------------------+ -| Engine | Support | Comment | -+------------+----------+----------------------------------------------------------------+ -| MyISAM | DEFAULT | Default engine as of MySQL 3.23 with great performance | ++------------+----------+-------------------------------------------------------------+ +| Engine | Support | Comment | ++------------+----------+-------------------------------------------------------------+ +| MyISAM | DEFAULT | Default engine as of MySQL 3.23 with great performance | ... -| SPHINX | YES | Sphinx storage engine | +| SPHINX | YES | Sphinx storage engine | ... -+------------+----------+----------------------------------------------------------------+ -13 rows in set (0.00 sec) ++------------+----------+-------------------------------------------------------------+ +13 rows in set (0.00 sec) @@ -3538,25 +3545,29 @@ Let's begin with an example create statement and search query: CREATE TABLE t1 ( - id INTEGER NOT NULL, + id INTEGER UNSIGNED NOT NULL, weight INTEGER NOT NULL, query VARCHAR(3072) NOT NULL, group_id INTEGER, INDEX(query) -) ENGINE=SPHINX CONNECTION="sphinx://localhost:3312/test"; +) ENGINE=SPHINX CONNECTION="sphinx://localhost:9312/test"; SELECT * FROM t1 WHERE query='test it;mode=any'; -First 3 columns of search table must be INTEGER, -INTEGER and VARCHAR which will be mapped to document ID, -match weight and search query accordingly. Query column must be indexed; +First 3 columns of search table must have a types of +INTEGER UNSINGED or BIGINT for the 1st column (document id), +INTEGER or BIGINT for the 2nd column (match weight), and +VARCHAR or TEXT for the 3rd column (your query), respectively. +This mapping is fixed; you can not omit any of these three required columns, +or move them around, or change types. Also, query column must be indexed; all the others must be kept unindexed. Columns' names are ignored so you can use arbitrary ones. -Additional columns must be either INTEGER or TIMESTAMP. +Additional columns must be either INTEGER, TIMESTAMP, +BIGINT, VARCHAR, or FLOAT. They will be bound to attributes provided in Sphinx result set by name, so their names must match attribute names specified in sphinx.conf. If there's no such attribute name in Sphinx search results, column will have @@ -3573,7 +3584,7 @@ or @distinct virtual attributes, use _sph_groupby, CONNECTION string parameter can be used to specify default searchd host, port and indexes for queries issued using this table. If no connection string is specified in CREATE TABLE, -index name "*" (ie. search all indexes) and localhost:3312 are assumed. +index name "*" (ie. search all indexes) and localhost:9312 are assumed. Connection string syntax is as follows: CONNECTION="sphinx://HOST:PORT/INDEXNAME" @@ -5881,9 +5892,9 @@ index dist { type = distributed local = chunk1 - agent = localhost:3312:chunk2 - agent = localhost:3312:chunk3 - agent = localhost:3312:chunk4 + agent = localhost:9312:chunk2 + agent = localhost:9312:chunk3 + agent = localhost:9312:chunk4 } Note how one of the chunks is searched locally and the same instance @@ -5892,9 +5903,9 @@ in parallel. Example: -agent = localhost:3312:chunk2 # contact itself +agent = localhost:9312:chunk2 # contact itself agent = /var/run/searchd.s:chunk2 -agent = searchbox2:3312:chunk3,chunk4 # search remote indexes +agent = searchbox2:9312:chunk3,chunk4 # search remote indexes @@ -5919,7 +5930,7 @@ The value format is completely identical to regular Example: -agent_blackhole = testbox:3312:testindex1,testindex2 +agent_blackhole = testbox:9312:testindex1,testindex2 @@ -6333,14 +6344,14 @@ listen = localhost listen = localhost:5000 listen = 192.168.0.1:5000 listen = /var/run/sphinx.s -listen = 3312 -listen = localhost:3307:mysql41 +listen = 9312 +listen = localhost:9306:mysql41 There can be multiple listen directives, searchd will listen for client connections on all specified ports and sockets. If no listen directive is found then the server will listen -on all available interfaces using the default port (which is 3312). +on all available interfaces using the default port (which is 9312). Unix-domain sockets are not supported on Windows. @@ -6370,11 +6381,11 @@ address = 192.168.0.1 searchd TCP port number. DEPRECATED, use listen instead. -Used to be mandatory. Default port number is 3312. +Used to be mandatory. Default port number is 9312. Example: -port = 3312 +port = 9312 diff --git a/misc/suggest/suggest.conf b/misc/suggest/suggest.conf index 6ee72597..53eec85f 100644 --- a/misc/suggest/suggest.conf +++ b/misc/suggest/suggest.conf @@ -31,7 +31,7 @@ indexer searchd { - port = 3312 + port = 9312 log = searchd.log query_log = query.log read_timeout = 5 diff --git a/mysqlse/ha_sphinx.cc b/mysqlse/ha_sphinx.cc index 9a6a968b..94e7e3a4 100644 --- a/mysqlse/ha_sphinx.cc +++ b/mysqlse/ha_sphinx.cc @@ -30,6 +30,8 @@ #include #define RECV_FLAGS MSG_WAITALL + + #define sphSockClose(_sock) ::close(_sock) #else // Windows-specific #include @@ -37,6 +39,8 @@ #define snprintf _snprintf #define RECV_FLAGS 0 + + #define sphSockClose(_sock) ::closesocket(_sock) #endif #include @@ -109,7 +113,7 @@ void sphUnalignedWrite ( void * pPtr, const T & tVal ) #define SPHINXSE_MAX_FILTERS 32 #define SPHINXSE_DEFAULT_HOST "127.0.0.1" -#define SPHINXSE_DEFAULT_PORT 3312 +#define SPHINXSE_DEFAULT_PORT 9312 #define SPHINXSE_DEFAULT_INDEX "*" #define SPHINXSE_SYSTEM_COLUMNS 3 @@ -411,12 +415,12 @@ struct CSphSEFilter public: ESphFilter m_eType; char * m_sAttrName; - uint32 m_uMinValue; - uint32 m_uMaxValue; + longlong m_uMinValue; + longlong m_uMaxValue; float m_fMinValue; float m_fMaxValue; int m_iValues; - uint32 * m_pValues; + longlong * m_pValues; int m_bExclude; public: @@ -530,7 +534,7 @@ struct CSphSEQuery int m_iBufLeft; bool m_bBufOverrun; - int ParseArray ( uint32 ** ppValues, const char * sValue ); + template < typename T > int ParseArray ( T ** ppValues, const char * sValue ); bool ParseField ( char * sField ); void SendBytes ( const void * pBytes, int iBytes ); @@ -542,6 +546,9 @@ struct CSphSEQuery void SendFloat ( float v ) { SendDword ( sphF2DW(v) ); } }; +template int CSphSEQuery::ParseArray ( uint32 **, const char * ); +template int CSphSEQuery::ParseArray ( longlong **, const char * ); + ////////////////////////////////////////////////////////////////////////////// #if MYSQL_VERSION_ID>50100 @@ -1195,21 +1202,22 @@ CSphSEQuery::~CSphSEQuery () } -int CSphSEQuery::ParseArray ( uint32 ** ppValues, const char * sValue ) +template < typename T > +int CSphSEQuery::ParseArray ( T ** ppValues, const char * sValue ) { SPH_ENTER_METHOD(); -// assert ( ppValues ); -// assert ( !(*ppValues) ); + assert ( ppValues ); + assert ( !(*ppValues) ); - const char * p; + const char * pValue; bool bPrevDigit = false; int iValues = 0; // count the values - for ( p=sValue; *p; p++ ) + for ( pValue=sValue; *pValue; pValue++ ) { - bool bDigit = ( (*p)>='0' && (*p)<='9' ); + bool bDigit = (*pValue)>='0' && (*pValue)<='9'; if ( bDigit && !bPrevDigit ) iValues++; bPrevDigit = bDigit; @@ -1218,33 +1226,34 @@ int CSphSEQuery::ParseArray ( uint32 ** ppValues, const char * sValue ) SPH_RET(0); // extract the values - uint32 * pValues = new uint32 [ iValues ]; + T * pValues = new T [ iValues ]; *ppValues = pValues; - int iIndex = 0; - uint32 uValue = 0; + int iIndex = 0, iSign = 1; + T uValue = 0; bPrevDigit = false; - for ( p=sValue; ; p++ ) + for ( pValue=sValue ;; pValue++ ) { - bool bDigit = ( (*p)>='0' && (*p)<='9' ); + bool bDigit = (*pValue)>='0' && (*pValue)<='9'; if ( bDigit ) { if ( !bPrevDigit ) uValue = 0; - uValue = uValue*10 + ( (*p)-'0' ); + uValue = uValue*10 + ( (*pValue)-'0' ); } - - if ( !bDigit && bPrevDigit ) + else if ( bPrevDigit ) { assert ( iIndex ( &m_pWeights, sValue ); else if ( !strcmp ( sName, "minid" ) ) m_iMinID = iValue; else if ( !strcmp ( sName, "maxid" ) ) m_iMaxID = iValue; else if ( !strcmp ( sName, "maxmatches" ) ) m_iMaxMatches = iValue; @@ -1345,6 +1354,7 @@ bool CSphSEQuery::ParseField ( char * sField ) else if ( !strcmp ( sValue, "ext2") ) m_eMode = SPH_MATCH_EXTENDED2; else if ( !strcmp ( sValue, "extended2") ) m_eMode = SPH_MATCH_EXTENDED2; else if ( !strcmp ( sValue, "all") ) m_eMode = SPH_MATCH_ALL; + else if ( !strcmp ( sValue, "fullscan") ) m_eMode = SPH_MATCH_FULLSCAN; else { snprintf ( m_sParseError, sizeof(m_sParseError), "unknown matching mode '%s'", sValue ); @@ -1450,8 +1460,8 @@ bool CSphSEQuery::ParseField ( char * sField ) if ( tFilter.m_eType==SPH_FILTER_RANGE ) { - tFilter.m_uMinValue = atoi(sValue); - tFilter.m_uMaxValue = atoi(p); + tFilter.m_uMinValue = strtoll ( sValue, NULL, 0 ); + tFilter.m_uMaxValue = strtoll ( p, NULL, 0 ); } else { tFilter.m_fMinValue = (float)atof(sValue); @@ -1486,7 +1496,7 @@ bool CSphSEQuery::ParseField ( char * sField ) *sValue++ = '\0'; // get the values - tFilter.m_iValues = ParseArray ( &tFilter.m_pValues, sValue ); + tFilter.m_iValues = ParseArray ( &tFilter.m_pValues, sValue ); if ( !tFilter.m_iValues ) { assert ( !tFilter.m_pValues ); @@ -1728,7 +1738,7 @@ int CSphSEQuery::BuildRequest ( char ** ppBuffer ) SPH_ENTER_METHOD(); // calc request length - int iReqSize = 116 + 4*m_iWeights + int iReqSize = 124 + 4*m_iWeights + strlen ( m_sSortBy ) + strlen ( m_sQuery ) + strlen ( m_sIndex ) @@ -1739,8 +1749,13 @@ int CSphSEQuery::BuildRequest ( char ** ppBuffer ) for ( int i=0; istore ( iValue64, 1 ); + af->store ( iValue64, 0 ); break; case ( SPH_ATTR_MULTI | SPH_ATTR_INTEGER ): @@ -3104,6 +3119,17 @@ int sphinx_showfunc_words ( THD * thd, SHOW_VAR * out, char * sBuffer ) return 0; } +int sphinx_showfunc_error ( THD * thd, SHOW_VAR * out, char * ) +{ + CSphSEStats * pStats = sphinx_get_stats ( thd, out ); + if ( pStats && pStats->m_bLastError ) + { + out->type = SHOW_CHAR; + out->value = pStats->m_sLastMessage; + } + return 0; +} + #if MYSQL_VERSION_ID>50100 struct st_mysql_storage_engine sphinx_storage_engine = { @@ -3117,6 +3143,7 @@ struct st_mysql_show_var sphinx_status_vars[] = {"sphinx_time", (char *)sphinx_showfunc_time, SHOW_FUNC}, {"sphinx_word_count", (char *)sphinx_showfunc_word_count, SHOW_FUNC}, {"sphinx_words", (char *)sphinx_showfunc_words, SHOW_FUNC}, + {"sphinx_error", (char *)sphinx_showfunc_error, SHOW_FUNC}, {0, 0, (enum_mysql_show_type)0} }; diff --git a/mysqlse/snippets_udf.cc b/mysqlse/snippets_udf.cc index f49f4f2f..5c11a876 100644 --- a/mysqlse/snippets_udf.cc +++ b/mysqlse/snippets_udf.cc @@ -174,7 +174,7 @@ enum #define SPHINXSE_DEFAULT_SCHEME "sphinx" #define SPHINXSE_DEFAULT_HOST "127.0.0.1" -#define SPHINXSE_DEFAULT_PORT 3312 +#define SPHINXSE_DEFAULT_PORT 9312 #define SPHINXSE_DEFAULT_INDEX "*" class CSphBuffer @@ -357,7 +357,7 @@ int CSphUrl::Connect() { struct sockaddr_in sin; #ifndef __WIN__ - struct sockaddr_un sun; + struct sockaddr_un saun; #endif int iDomain = 0; @@ -406,12 +406,12 @@ int CSphUrl::Connect() { #ifndef __WIN__ iDomain = AF_UNIX; - iSockaddrSize = sizeof(sun); - pSockaddr = (struct sockaddr *) &sun; + iSockaddrSize = sizeof(saun); + pSockaddr = (struct sockaddr *) &saun; - memset ( &sun, 0, sizeof(sun) ); - sun.sun_family = AF_UNIX; - strncpy ( sun.sun_path, m_sHost, sizeof(sun.sun_path)-1 ); + memset ( &saun, 0, sizeof(saun) ); + saun.sun_family = AF_UNIX; + strncpy ( saun.sun_path, m_sHost, sizeof(saun.sun_path)-1 ); #else my_error ( ER_CONNECT_TO_FOREIGN_DATA_SOURCE, MYF(0), "Unix-domain sockets are not supported on Windows" ); return -1; diff --git a/sphinx-min.conf.in b/sphinx-min.conf.in index 2795a627..319f6d63 100644 --- a/sphinx-min.conf.in +++ b/sphinx-min.conf.in @@ -40,7 +40,7 @@ indexer searchd { - port = 3312 + port = 9312 log = @CONFDIR@/log/searchd.log query_log = @CONFDIR@/log/query.log read_timeout = 5 diff --git a/sphinx.conf.in b/sphinx.conf.in index 0ef127b8..2ab73905 100644 --- a/sphinx.conf.in +++ b/sphinx.conf.in @@ -542,14 +542,14 @@ index dist1 # multiple remote agents may be specified # syntax for TCP connections is 'hostname:port:index1,[index2[,...]]' # syntax for local UNIX connections is '/path/to/socket:index1,[index2[,...]]' - agent = localhost:3313:remote1 - agent = localhost:3314:remote2,remote3 + agent = localhost:9313:remote1 + agent = localhost:9314:remote2,remote3 # agent = /var/run/searchd.sock:remote4 # blackhole remote agent, for debugging/testing # network errors and search results will be ignored # - # agent_blackhole = testbox:3312:testindex1,testindex2 + # agent_blackhole = testbox:9312:testindex1,testindex2 # remote agent connection timeout, milliseconds @@ -605,11 +605,11 @@ searchd { # hostname, port, or hostname:port, or /unix/socket/path to listen on # multi-value, multiple listen points are allowed - # optional, default is 0.0.0.0:3312 (listen on all interfaces, port 3312) + # optional, default is 0.0.0.0:9312 (listen on all interfaces, port 9312) # # listen = 127.0.0.1 - # listen = 192.168.0.1:3312 - # listen = 3312 + # listen = 192.168.0.1:9312 + # listen = 9312 # listen = /var/run/searchd.sock diff --git a/src/llsphinxql.c b/src/llsphinxql.c index 88c50095..b5d40b53 100644 --- a/src/llsphinxql.c +++ b/src/llsphinxql.c @@ -367,7 +367,7 @@ static yyconst flex_int16_t yy_accept[156] = 34, 0, 0, 0, 42, 42, 1, 42, 42, 5, 42, 42, 42, 42, 42, 13, 12, 42, 42, 42, 42, 42, 42, 29, 42, 42, 42, 42, 42, 42, - 42, 42, 35, 36, 0, 39, 0, 28, 2, 3, + 42, 42, 0, 36, 0, 39, 0, 28, 2, 3, 42, 42, 42, 42, 42, 42, 42, 42, 15, 42, 17, 30, 42, 42, 42, 42, 42, 23, 42, 42, @@ -423,23 +423,23 @@ static yyconst flex_int32_t yy_meta[59] = static yyconst flex_int16_t yy_base[158] = { 0, - 0, 0, 304, 324, 57, 59, 292, 58, 58, 292, - 60, 59, 255, 56, 50, 55, 48, 59, 0, 55, - 56, 83, 64, 97, 62, 65, 102, 104, 84, 324, - 115, 324, 121, 121, 95, 128, 83, 73, 324, 324, - 324, 104, 121, 122, 0, 124, 129, 128, 120, 0, - 121, 125, 129, 133, 134, 0, 0, 138, 135, 136, - 142, 140, 142, 158, 153, 152, 166, 160, 158, 159, - 170, 161, 189, 70, 165, 324, 176, 0, 0, 0, - 167, 180, 187, 176, 184, 179, 191, 198, 0, 202, - 0, 0, 195, 200, 201, 187, 192, 0, 199, 208, - - 202, 212, 209, 216, 221, 214, 0, 225, 0, 221, - 219, 230, 0, 226, 226, 240, 0, 227, 239, 241, - 246, 243, 236, 248, 252, 0, 248, 0, 0, 0, - 251, 0, 248, 251, 257, 253, 0, 261, 324, 260, - 266, 279, 0, 0, 0, 276, 0, 0, 324, 0, - 267, 269, 0, 0, 324, 321, 71 + 0, 0, 304, 327, 57, 59, 286, 58, 58, 244, + 60, 59, 210, 56, 50, 55, 48, 59, 0, 55, + 56, 83, 64, 97, 62, 65, 102, 104, 84, 327, + 115, 327, 100, 120, 191, 122, 83, 73, 327, 327, + 327, 109, 121, 122, 0, 124, 129, 128, 120, 0, + 121, 124, 125, 131, 133, 0, 0, 136, 132, 135, + 142, 139, 140, 156, 151, 152, 166, 157, 156, 165, + 170, 161, 189, 70, 164, 327, 177, 0, 0, 0, + 167, 180, 187, 176, 184, 178, 191, 198, 0, 202, + 0, 0, 195, 200, 202, 188, 194, 0, 202, 211, + + 205, 214, 213, 216, 225, 215, 0, 225, 0, 221, + 219, 231, 0, 227, 227, 242, 0, 229, 241, 243, + 247, 244, 237, 249, 256, 0, 252, 0, 0, 0, + 254, 0, 250, 253, 259, 255, 0, 266, 327, 262, + 269, 259, 0, 0, 0, 276, 0, 0, 327, 0, + 269, 272, 0, 0, 327, 324, 71 } ; static yyconst flex_int16_t yy_def[158] = @@ -463,7 +463,7 @@ static yyconst flex_int16_t yy_def[158] = 157, 157, 157, 157, 0, 155, 155 } ; -static yyconst flex_int16_t yy_nxt[383] = +static yyconst flex_int16_t yy_nxt[386] = { 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 4, 13, 14, 15, 16, 17, 18, 19, 20, 21, 19, @@ -476,40 +476,40 @@ static yyconst flex_int16_t yy_nxt[383] = 74, 48, 54, 55, 58, 29, 29, 62, 44, 50, 37, 63, 64, 33, 42, 49, 51, 46, 56, 52, - 43, 47, 37, 53, 48, 54, 55, 57, 58, 59, + 43, 47, 155, 53, 48, 54, 55, 57, 58, 59, 62, 44, 50, 60, 63, 64, 69, 61, 65, 32, - 70, 66, 56, 71, 72, 73, 34, 35, 36, 75, - 57, 67, 68, 59, 38, 36, 76, 60, 77, 78, + 70, 66, 56, 71, 72, 34, 35, 36, 38, 36, + 57, 67, 68, 59, 75, 73, 76, 60, 77, 78, 69, 61, 65, 79, 70, 66, 80, 71, 72, 81, - 33, 82, 75, 83, 67, 68, 33, 84, 85, 86, - 76, 87, 77, 78, 88, 90, 91, 79, 89, 92, - 80, 93, 81, 94, 82, 95, 83, 96, 97, 100, - 84, 85, 86, 98, 87, 99, 101, 88, 90, 91, - 102, 89, 92, 32, 93, 103, 104, 94, 95, 105, - - 96, 107, 97, 100, 106, 108, 98, 109, 99, 110, - 101, 111, 112, 102, 113, 114, 115, 116, 103, 117, - 104, 118, 105, 119, 33, 107, 120, 106, 108, 121, - 109, 122, 110, 123, 124, 111, 112, 125, 113, 114, - 115, 116, 117, 126, 118, 127, 119, 128, 129, 130, - 120, 131, 121, 132, 133, 122, 123, 134, 124, 135, - 136, 125, 137, 138, 41, 139, 126, 140, 141, 127, - 128, 129, 142, 130, 131, 143, 132, 144, 133, 145, - 134, 146, 147, 135, 136, 148, 137, 138, 139, 149, - 150, 140, 141, 151, 152, 142, 153, 154, 143, 37, - - 144, 30, 145, 155, 146, 147, 155, 155, 148, 155, - 155, 155, 149, 150, 155, 155, 155, 151, 152, 153, - 154, 31, 31, 3, 155, 155, 155, 155, 155, 155, + 33, 82, 83, 84, 67, 68, 85, 75, 86, 87, + 76, 88, 77, 78, 90, 89, 91, 79, 92, 93, + 80, 94, 81, 95, 82, 83, 84, 96, 97, 85, + 98, 86, 87, 99, 88, 100, 101, 90, 89, 91, + 102, 92, 93, 32, 103, 94, 95, 104, 37, 105, + + 96, 107, 97, 98, 106, 108, 99, 109, 110, 100, + 101, 111, 112, 102, 113, 114, 115, 103, 116, 41, + 117, 104, 105, 118, 33, 107, 119, 106, 108, 120, + 109, 110, 121, 122, 124, 111, 112, 123, 113, 114, + 115, 125, 116, 117, 126, 127, 118, 128, 129, 119, + 130, 37, 131, 120, 132, 121, 133, 122, 124, 134, + 123, 135, 136, 137, 138, 125, 139, 126, 140, 127, + 128, 129, 141, 151, 130, 131, 142, 132, 143, 144, + 133, 145, 134, 146, 147, 135, 136, 137, 138, 139, + 148, 149, 140, 150, 152, 30, 141, 151, 153, 142, + + 154, 143, 144, 155, 145, 155, 146, 147, 155, 155, + 155, 155, 155, 148, 149, 155, 150, 155, 152, 155, + 155, 153, 155, 154, 31, 31, 3, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155 + 155, 155, 155, 155, 155 } ; -static yyconst flex_int16_t yy_chk[383] = +static yyconst flex_int16_t yy_chk[386] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -522,37 +522,37 @@ static yyconst flex_int16_t yy_chk[383] = 38, 15, 20, 21, 23, 29, 29, 25, 14, 16, 37, 26, 26, 8, 14, 16, 17, 15, 22, 18, - 14, 15, 35, 18, 15, 20, 21, 22, 23, 24, + 14, 15, 33, 18, 15, 20, 21, 22, 23, 24, 25, 14, 16, 24, 26, 26, 28, 24, 27, 31, - 28, 27, 22, 28, 28, 33, 34, 34, 34, 42, - 22, 27, 27, 24, 36, 36, 43, 24, 44, 46, + 28, 27, 22, 28, 28, 34, 34, 34, 36, 36, + 22, 27, 27, 24, 42, 33, 43, 24, 44, 46, 28, 24, 27, 47, 28, 27, 48, 28, 28, 49, - 31, 51, 42, 52, 27, 27, 33, 53, 54, 55, - 43, 58, 44, 46, 59, 60, 61, 47, 59, 62, - 48, 63, 49, 64, 51, 65, 52, 66, 67, 70, - 53, 54, 55, 68, 58, 69, 71, 59, 60, 61, - 72, 59, 62, 73, 63, 75, 77, 64, 65, 81, - - 66, 83, 67, 70, 82, 84, 68, 85, 69, 86, - 71, 87, 88, 72, 90, 93, 94, 95, 75, 96, - 77, 97, 81, 99, 73, 83, 100, 82, 84, 101, - 85, 102, 86, 103, 104, 87, 88, 105, 90, 93, - 94, 95, 96, 106, 97, 108, 99, 110, 111, 112, - 100, 114, 101, 115, 116, 102, 103, 118, 104, 119, - 120, 105, 121, 122, 13, 123, 106, 124, 125, 108, - 110, 111, 127, 112, 114, 131, 115, 133, 116, 134, - 118, 135, 136, 119, 120, 138, 121, 122, 123, 140, - 141, 124, 125, 142, 146, 127, 151, 152, 131, 10, - - 133, 7, 134, 3, 135, 136, 0, 0, 138, 0, - 0, 0, 140, 141, 0, 0, 0, 142, 146, 151, - 152, 156, 156, 155, 155, 155, 155, 155, 155, 155, + 31, 51, 52, 53, 27, 27, 54, 42, 55, 58, + 43, 59, 44, 46, 60, 59, 61, 47, 62, 63, + 48, 64, 49, 65, 51, 52, 53, 66, 67, 54, + 68, 55, 58, 69, 59, 70, 71, 60, 59, 61, + 72, 62, 63, 73, 75, 64, 65, 77, 35, 81, + + 66, 83, 67, 68, 82, 84, 69, 85, 86, 70, + 71, 87, 88, 72, 90, 93, 94, 75, 95, 13, + 96, 77, 81, 97, 73, 83, 99, 82, 84, 100, + 85, 86, 101, 102, 104, 87, 88, 103, 90, 93, + 94, 105, 95, 96, 106, 108, 97, 110, 111, 99, + 112, 10, 114, 100, 115, 101, 116, 102, 104, 118, + 103, 119, 120, 121, 122, 105, 123, 106, 124, 108, + 110, 111, 125, 142, 112, 114, 127, 115, 131, 133, + 116, 134, 118, 135, 136, 119, 120, 121, 122, 123, + 138, 140, 124, 141, 146, 7, 125, 142, 151, 127, + + 152, 131, 133, 3, 134, 0, 135, 136, 0, 0, + 0, 0, 0, 138, 140, 0, 141, 0, 146, 0, + 0, 151, 0, 152, 156, 156, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155 + 155, 155, 155, 155, 155 } ; /* The intent behind this definition is that it'll catch @@ -862,7 +862,7 @@ YY_DECL yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } - while ( yy_base[yy_current_state] != 324 ); + while ( yy_base[yy_current_state] != 327 ); yy_find_action: yy_act = yy_accept[yy_current_state]; diff --git a/src/searchd.cpp b/src/searchd.cpp index c73a9055..900596e7 100644 --- a/src/searchd.cpp +++ b/src/searchd.cpp @@ -27,7 +27,7 @@ #include #define SEARCHD_BACKLOG 5 -#define SEARCHD_DEFAULT_PORT 3312 +#define SEARCHD_DEFAULT_PORT 9312 #define SPH_ADDRESS_SIZE sizeof("000.000.000.000") @@ -1317,6 +1317,14 @@ void sphSockSetErrno ( int iErr ) } +int sphSockPeekErrno () +{ + int iRes = sphSockGetErrno(); + sphSockSetErrno ( iRes ); + return iRes; +} + + /// formats IP address given in network byte order into sBuffer /// returns the buffer char * sphFormatIP ( char * sBuffer, int iBufferSize, DWORD uAddress ) @@ -1610,11 +1618,11 @@ int sphSetSockNB ( int iSock ) } -int sphSockRead ( int iSock, void * buf, int iLen, int iReadTimeout ) +int sphSockRead ( int iSock, void * buf, int iLen, int iReadTimeout, bool bIntr ) { assert ( iLen>0 ); - int64_t tmMaxTimer = sphMicroTimer() + 1000000*Max ( 1, iReadTimeout ); // in microseconds + int64_t tmMaxTimer = sphMicroTimer() + I64C(1000000)*Max ( 1, iReadTimeout ); // in microseconds int iLeftBytes = iLen; // bytes to read left char * pBuf = (char*) buf; @@ -1644,7 +1652,7 @@ int sphSockRead ( int iSock, void * buf, int iLen, int iReadTimeout ) if ( iRes==-1 ) { iErr = sphSockGetErrno(); - if ( iErr==EINTR && !g_bGotSigterm ) + if ( iErr==EINTR && !g_bGotSigterm && !bIntr ) continue; sphSockSetErrno ( iErr ); @@ -1672,7 +1680,7 @@ int sphSockRead ( int iSock, void * buf, int iLen, int iReadTimeout ) if ( iRes==-1 ) { iErr = sphSockGetErrno(); - if ( iErr==EINTR ) + if ( iErr==EINTR && !bIntr ) continue; sphSockSetErrno ( iErr ); @@ -1682,6 +1690,9 @@ int sphSockRead ( int iSock, void * buf, int iLen, int iReadTimeout ) // update pBuf += iRes; iLeftBytes -= iRes; + + // avoid partial buffer loss in case of signal during the 2nd (!) read + bIntr = false; } // if there was a timeout, report it as an error @@ -1798,17 +1809,19 @@ class NetInputBuffer_c : public InputBuffer_c NetInputBuffer_c ( int iSock ); virtual ~NetInputBuffer_c (); - bool ReadFrom ( int iLen, int iTimeout ); + bool ReadFrom ( int iLen, int iTimeout, bool bIntr=false ); bool ReadFrom ( int iLen ) { return ReadFrom ( iLen, g_iReadTimeout ); }; virtual void SendErrorReply ( const char *, ... ); const BYTE * GetBufferPtr () const { return m_pBuf; } + bool IsIntr () const { return m_bIntr; } protected: static const int NET_MINIBUFFER_SIZE = 4096; int m_iSock; + bool m_bIntr; BYTE m_dMinibufer[NET_MINIBUFFER_SIZE]; int m_iMaxibuffer; @@ -2143,6 +2156,7 @@ template < typename T > bool InputBuffer_c::GetQwords ( CSphVector & dBuffer, NetInputBuffer_c::NetInputBuffer_c ( int iSock ) : InputBuffer_c ( m_dMinibufer, sizeof(m_dMinibufer) ) , m_iSock ( iSock ) + , m_bIntr ( false ) , m_iMaxibuffer ( 0 ) , m_pMaxibuffer ( NULL ) {} @@ -2154,8 +2168,9 @@ NetInputBuffer_c::~NetInputBuffer_c () } -bool NetInputBuffer_c::ReadFrom ( int iLen, int iTimeout ) +bool NetInputBuffer_c::ReadFrom ( int iLen, int iTimeout, bool bIntr ) { + m_bIntr = false; if ( iLen<=0 || iLen>g_iMaxPacketSize || m_iSock<0 ) return false; @@ -2172,11 +2187,12 @@ bool NetInputBuffer_c::ReadFrom ( int iLen, int iTimeout ) } m_pCur = m_pBuf = pBuf; - int iGot = sphSockRead ( m_iSock, pBuf, iLen, iTimeout ); + int iGot = sphSockRead ( m_iSock, pBuf, iLen, iTimeout, bIntr ); if ( g_bGotSigterm ) return false; m_bError = ( iGot!=iLen ); + m_bIntr = m_bError && ( sphSockPeekErrno()==EINTR ); m_iLen = m_bError ? 0 : iLen; return !m_bError; } @@ -6276,7 +6292,19 @@ void HandleClientSphinx ( int iSock, const char * sClientIP, int iPipeFD ) { g_pCrashLog_LastQuery = NULL; - tBuf.ReadFrom ( 8, iTimeout ); + // in "persistent connection" mode, we want interruptable waits + // so that the worker child could be forcibly restarted + tBuf.ReadFrom ( 8, iTimeout, bPersist ); + if ( bPersist && tBuf.IsIntr() ) + { + // SIGHUP means restart + if ( g_bGotSighup ) + break; + + // otherwise, keep waiting + continue; + } + int iCommand = tBuf.GetWord (); int iCommandVer = tBuf.GetWord (); int iLength = tBuf.GetInt (); @@ -7038,8 +7066,8 @@ int CreatePipe ( bool bFatal, int iHandler ) int PipeAndFork ( bool bFatal, int iHandler ) { int iChildPipe = CreatePipe ( bFatal, iHandler ); - int iRes = fork(); - switch ( iRes ) + int iFork = fork(); + switch ( iFork ) { // fork() failed case -1: @@ -7056,6 +7084,7 @@ int PipeAndFork ( bool bFatal, int iHandler ) // parent process, continue accept()ing default: g_iChildren++; + g_dChildren.Add ( iFork ); SafeClose ( iChildPipe ); g_dChildren.Add ( iRes ); break; @@ -7294,6 +7323,21 @@ void RotationThreadFunc ( void * ) } +void IndexRotationDone () +{ +#if !USE_WINDOWS + // forcibly restart children serving persistent connections and/or preforked ones + // FIXME! check how both signals are handled in both cases + int iSignal = ( g_eWorks==MPM_PREFORK ) ? SIGTERM : SIGHUP; + ARRAY_FOREACH ( i, g_dChildren ) + kill ( g_dChildren[i], iSignal ); +#endif + + g_bDoRotate = false; + sphInfo ( "rotating finished" ); +} + + void SeamlessTryToForkPrereader () { // next in line @@ -7391,10 +7435,7 @@ void SeamlessForkPrereader () // if there's no more candidates, and nothing in the works, we're done if ( !g_sPrereading && !g_dRotating.GetLength() ) - { - g_bDoRotate = false; - sphInfo ( "rotating finished" ); - } + IndexRotationDone (); } @@ -8150,15 +8191,7 @@ void CheckRotate () } SafeDelete ( pCP ); - -#if !USE_WINDOWS - if ( g_eWorkers==MPM_PREFORK ) - ARRAY_FOREACH ( i, g_dChildren ) - kill ( g_dChildren[i], SIGTERM ); -#endif - - g_bDoRotate = false; - sphInfo ( "rotating finished" ); + IndexRotationDone (); return; } diff --git a/src/sphinx.cpp b/src/sphinx.cpp index b2fe2676..d89a803e 100644 --- a/src/sphinx.cpp +++ b/src/sphinx.cpp @@ -9739,7 +9739,7 @@ int CSphIndex_VLN::Build ( const CSphVector & dSources, int iMemory } assert ( Id.m_uDocID == DOCINFO2ID(pEntry) ); - DOCINFO2ATTRS(pEntry) [ dOrdinalAttrs[i] ] = Id.m_uId; + sphSetRowAttr ( DOCINFO2ATTRS(pEntry), m_tSchema.GetAttr(dOrdinalAttrs[i]).m_tLocator, Id.m_uId ); } iOrd++; m_uMinMaxIndex += iDocinfoStride; @@ -16604,7 +16604,7 @@ bool CSphSource_SQL::IterateHitsStart ( CSphString & sError ) // check it if ( m_tSchema.m_dFields.GetLength()>SPH_MAX_FIELDS ) - LOC_ERROR2 ( "too many fields (fields=%d, max=%d); raise SPH_MAX_FIELDS in sphinx.h and rebuild", + LOC_ERROR2 ( "too many fields (fields=%d, max=%d)", m_tSchema.m_dFields.GetLength(), SPH_MAX_FIELDS ); return true; diff --git a/src/sphinxexpr.cpp b/src/sphinxexpr.cpp index ce3582e7..d862070d 100644 --- a/src/sphinxexpr.cpp +++ b/src/sphinxexpr.cpp @@ -657,6 +657,16 @@ int ExprParser_t::GetToken ( YYSTYPE * lvalp ) m_pCur++; if ( *m_pCur=='=' ) m_pCur++; return TOK_EQ; + + // special case for float values without leading zero + case '.': + char * pEnd = NULL; + lvalp->fConst = (float) strtod ( m_pCur, &pEnd ); + if ( pEnd ) + { + m_pCur = pEnd; + return TOK_CONST_FLOAT; + } } m_sLexerError.SetSprintf ( "unknown operator '%c' near '%s'", *m_pCur, m_pCur ); diff --git a/src/sphinxql.l b/src/sphinxql.l index f70f7c62..7c3dfebe 100644 --- a/src/sphinxql.l +++ b/src/sphinxql.l @@ -63,7 +63,7 @@ SPACE [ \t\n\r] "<=" { YYSTOREBOUNDS; return TOK_LTE; } ">=" { YYSTOREBOUNDS; return TOK_GTE; } -'([^']|\\')*' { YYSTOREBOUNDS; SqlUnescape ( lvalp->m_sValue, yytext, yyleng ); return TOK_QUOTED_STRING; } +'([^'\\]|\\.|\\\\)*' { YYSTOREBOUNDS; SqlUnescape ( lvalp->m_sValue, yytext, yyleng ); return TOK_QUOTED_STRING; } -*{DIGIT}+\.{DIGIT}* { YYSTOREBOUNDS; lvalp->m_sValue = yytext; lvalp->m_fValue = (float)strtod ( yytext, NULL ); return TOK_CONST_FLOAT; } -*\.{DIGIT}+ { YYSTOREBOUNDS; lvalp->m_sValue = yytext; lvalp->m_fValue = (float)strtod ( yytext, NULL ); return TOK_CONST_FLOAT; } diff --git a/src/sphinxsearch.cpp b/src/sphinxsearch.cpp index 090c651a..9eb4caff 100644 --- a/src/sphinxsearch.cpp +++ b/src/sphinxsearch.cpp @@ -2247,18 +2247,21 @@ ExtProximity_c::ExtProximity_c ( CSphVector & dQwords, DWORD uDupeM const ExtDoc_t * ExtProximity_c::GetDocsChunk ( SphDocID_t * pMaxID ) { m_uMaxID = 0; - const ExtDoc_t * pDocs = NULL; - const ExtHit_t * pHits = NULL; - SphDocID_t uMaxID = 0; - if ( !pDocs ) pDocs = m_pNode->GetDocsChunk ( &uMaxID ); - if ( !pDocs ) return NULL; - if ( !pHits ) pHits = m_pNode->GetHitsChunk ( pDocs, uMaxID ); - assert ( pHits ); + // shortcuts + const ExtDoc_t * pDoc = m_pDoc; + const ExtHit_t * pHit = m_pHit; - const ExtHit_t * pHit = pHits; - const ExtDoc_t * pDoc = pDocs; + // warmup + if ( !pDoc ) + { + if ( !m_pDocs ) pDoc = m_pDocs = m_pNode->GetDocsChunk ( &uMaxID ); + if ( !pDoc ) return NULL; + + pHit = m_pHits = m_pNode->GetHitsChunk ( m_pDocs, uMaxID ); + assert ( pHit ); + } CSphVector dProx; // proximity hit position for i-th word int iProxWords = 0; @@ -2276,21 +2279,18 @@ const ExtDoc_t * ExtProximity_c::GetDocsChunk ( SphDocID_t * pMaxID ) // update hitlist if ( pHit->m_uDocid==DOCID_MAX ) { - pHits = m_pNode->GetHitsChunk ( pDocs, uMaxID ); - if ( pHits ) - { - pHit = pHits; + pHit = m_pHits = m_pNode->GetHitsChunk ( m_pDocs, uMaxID ); + if ( pHit ) continue; - } // no more hits for current document? *now* we can reset m_uExpID = m_uExpPos = m_uExpQpos = 0; - pDoc = pDocs = m_pNode->GetDocsChunk ( &uMaxID ); - if ( !pDocs ) + pDoc = m_pDocs = m_pNode->GetDocsChunk ( &uMaxID ); + if ( !pDoc ) break; - pHit = pHits = m_pNode->GetHitsChunk ( pDocs, uMaxID ); + pHit = m_pHits = m_pNode->GetHitsChunk ( m_pDocs, uMaxID ); assert ( pHit ); continue; } @@ -2416,6 +2416,10 @@ const ExtDoc_t * ExtProximity_c::GetDocsChunk ( SphDocID_t * pMaxID ) m_pMyDoc = m_dDocs; m_pMyHit = m_dMyHits; + // save shortcuts + m_pDoc = pDoc; + m_pHit = pHit; + assert ( iHit>=0 && iHit +uint attrs + + +indexer +{ + mem_limit = 16M +} + +searchd +{ + +} + +source test +{ + type = mysql + + + sql_query = SELECT id, n, 'text' FROM test_table; + sql_attr_uint = n + sql_attr_multi = uint mva from query; select id, n mva from test_table +} + +index test +{ + source = test + path = /test +} + + + +CREATE TABLE test_table +( + id INT NOT NULL, + n INT UNSIGNED NOT NULL +); + + + +DROP TABLE IF EXISTS test_table; + + + +INSERT INTO test_table (id, n) VALUES +( 1, 2582995467 ), +( 2, 3650268775 ), +( 3, 1452351953 ), +( 4, 1022026391 ), +( 5, 3802901620 ), +( 6, 1329722356 ); + + + + n + mva + + + + + + + + + diff --git a/test/test_86/model.bin b/test/test_86/model.bin new file mode 100644 index 00000000..df100401 --- /dev/null +++ b/test/test_86/model.bin @@ -0,0 +1 @@ +a:1:{i:0;a:1:{i:0;a:2:{i:0;a:9:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:5:"title";}s:5:"attrs";a:0:{}s:7:"matches";a:3:{i:1;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}i:2;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}i:4;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"3";s:11:"total_found";s:1:"3";s:5:"words";a:1:{s:5:"sonne";a:2:{s:4:"docs";s:1:"3";s:4:"hits";s:1:"3";}}}i:1;a:9:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:5:"title";}s:5:"attrs";a:0:{}s:7:"matches";a:3:{i:101;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}i:102;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}i:104;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"3";s:11:"total_found";s:1:"3";s:5:"words";a:1:{s:5:"sonne";a:2:{s:4:"docs";s:1:"3";s:4:"hits";s:1:"3";}}}}}} \ No newline at end of file diff --git a/test/test_86/test.xml b/test/test_86/test.xml new file mode 100644 index 00000000..ee5fb35a --- /dev/null +++ b/test/test_86/test.xml @@ -0,0 +1,78 @@ + + + +index rotation vs pconns + + + + + + +indexer +{ + mem_limit = 16M +} + +searchd +{ + +} + +source test1 +{ + type = mysql + + sql_query = SELECT * FROM test_table + sql_query_post_index = UPDATE test_table SET id=id+100 +} + +index test1 +{ + source = test1 + path = /test1 +} + + + +CREATE TABLE test_table +( + id INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT, + title varchar(255) NOT NULL +); + + + +DROP TABLE IF EXISTS test_table; + + + +INSERT INTO test_table VALUES +( 1, 'hier kommt die sonne' ), +( 2, 'hier kommt die sonne' ), +( 3, 'sie est der hellste stern von allen' ), +( 4, 'hier kommt die sonne' ); + + +Open (); +$res1 = $client->Query ( "sonne", "test1" ); +unset ( $res1["time"] ); + +$rv = 0; +$err = ""; +exec ( "$indexer_path --config config.conf --rotate --all", $err, $rv ); + +usleep ( 1500000 ); +$res2 = $client->Query ( "sonne", "test1" ); +unset ( $res2["time"] ); +$client->Close(); + +$results = array ( $res1, $res2 ); +if ( !$res1 || !$res2 || $rv!=0 ) + $results = false; +]]> + + + diff --git a/test/test_93/model.bin b/test/test_93/model.bin new file mode 100644 index 00000000..c577417c --- /dev/null +++ b/test/test_93/model.bin @@ -0,0 +1 @@ +a:1:{i:0;a:5:{i:0;a:13:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:4:"body";}s:5:"attrs";a:0:{}s:7:"matches";a:1:{i:101;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"1";s:11:"total_found";s:1:"1";s:4:"time";s:5:"0.003";s:5:"words";a:1:{s:3:"one";a:2:{s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}s:8:"resarray";i:0;s:8:"roundoff";i:0;s:5:"query";s:3:"one";}i:1;a:13:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:4:"body";}s:5:"attrs";a:0:{}s:7:"matches";a:3:{i:102;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}i:1002;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}i:1003;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"3";s:11:"total_found";s:1:"3";s:4:"time";s:5:"0.003";s:5:"words";a:1:{s:3:"two";a:2:{s:4:"docs";s:1:"3";s:4:"hits";s:1:"3";}}s:8:"resarray";i:0;s:8:"roundoff";i:0;s:5:"query";s:3:"two";}i:2;a:13:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:4:"body";}s:5:"attrs";a:0:{}s:7:"matches";a:1:{i:103;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"1";s:11:"total_found";s:1:"1";s:4:"time";s:5:"0.004";s:5:"words";a:1:{s:5:"three";a:2:{s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}s:8:"resarray";i:0;s:8:"roundoff";i:0;s:5:"query";s:5:"three";}i:3;a:13:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:4:"body";}s:5:"attrs";a:0:{}s:7:"matches";a:2:{i:104;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}i:1004;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"2";s:11:"total_found";s:1:"2";s:4:"time";s:5:"0.003";s:5:"words";a:1:{s:4:"four";a:2:{s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}}s:8:"resarray";i:0;s:8:"roundoff";i:0;s:5:"query";s:4:"four";}i:4;a:13:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:4:"body";}s:5:"attrs";a:0:{}s:7:"matches";a:1:{i:1003;a:2:{s:6:"weight";s:1:"1";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"1";s:11:"total_found";s:1:"1";s:4:"time";s:5:"0.003";s:5:"words";a:1:{s:4:"five";a:2:{s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}s:8:"resarray";i:0;s:8:"roundoff";i:0;s:5:"query";s:4:"five";}}} \ No newline at end of file diff --git a/test/test_93/test.xml b/test/test_93/test.xml new file mode 100644 index 00000000..1c6daaf7 --- /dev/null +++ b/test/test_93/test.xml @@ -0,0 +1,81 @@ + + + +merge vs different min docids + + +indexer +{ + mem_limit = 16M +} + +searchd +{ + +} + +source srcmain +{ + type = mysql + + + sql_query = SELECT * FROM test_table WHERE document_id in (101,102,103,104) +} + +source srcdelta : srcmain +{ + sql_query = SELECT * FROM test_table WHERE document_id in (1001,1002,1003,1004) +} + +index main +{ + source = srcmain + path = /main + charset_type = utf-8 +} + +index delta +{ + source = srcdelta + path = /delta + charset_type = utf-8 +} + + + +--merge main delta + + + +one +two +three +four +five + + + +CREATE TABLE `test_table` +( + `document_id` int(11) NOT NULL default '0', + `body` varchar(255) NOT NULL default '' +) + + + +DROP TABLE IF EXISTS `test_table` + + + +INSERT INTO `test_table` VALUES +( 101, 'one' ), +( 102, 'two' ), +( 103, 'three crazy mice live happly' ), +( 104, 'four' ), +( 1001, 'seven' ), +( 1002, 'six two' ), +( 1003, 'five but now two' ), +( 1004, 'four' ) + + + \ No newline at end of file diff --git a/test/test_94/model.bin b/test/test_94/model.bin new file mode 100644 index 00000000..bd456a79 --- /dev/null +++ b/test/test_94/model.bin @@ -0,0 +1 @@ +a:1:{i:0;a:1:{i:0;a:13:{s:5:"error";s:0:"";s:7:"warning";s:0:"";s:6:"status";i:0;s:6:"fields";a:1:{i:0;s:4:"body";}s:5:"attrs";a:0:{}s:7:"matches";a:2:{i:1;a:2:{s:6:"weight";s:6:"255186";s:5:"attrs";a:0:{}}i:2;a:2:{s:6:"weight";s:4:"1356";s:5:"attrs";a:0:{}}}s:5:"total";s:1:"2";s:11:"total_found";s:1:"2";s:4:"time";s:5:"0.002";s:5:"words";a:2:{s:3:"one";a:2:{s:4:"docs";s:1:"2";s:4:"hits";s:3:"258";}s:3:"two";a:2:{s:4:"docs";s:1:"2";s:4:"hits";s:3:"258";}}s:8:"resarray";i:0;s:8:"roundoff";i:0;s:5:"query";s:12:""one two"~10";}}} \ No newline at end of file diff --git a/test/test_94/test.xml b/test/test_94/test.xml new file mode 100644 index 00000000..e22d9202 --- /dev/null +++ b/test/test_94/test.xml @@ -0,0 +1,55 @@ + + + +proximity queries + + +indexer +{ + mem_limit = 16M +} + +searchd +{ + +} + +source srctest +{ + type = mysql + + sql_query = SELECT * FROM test_table +} + +index test +{ + source = srctest + path = /test + charset_type = utf-8 +} + + + + +"one two"~10 + + + +CREATE TABLE `test_table` +( + `document_id` int(11) NOT NULL default '0', + `body` TEXT NOT NULL +) + + + +DROP TABLE IF EXISTS `test_table` + + + +INSERT INTO `test_table` +SELECT 1, REPEAT('one two ',257) UNION +SELECT 2, 'two one' + + + \ No newline at end of file