Item9806: restored missing 1.0 support. Some work on memory consumpti…

…on and wide-byte character support. git-svn-id: http://svn.foswiki.org/trunk/NativeSearchContrib@9530 0b4bb1d4-4e5a-0410-9cc4-b2b747904278
foswiki · Oct 7, 2010 · 1212088 · 1212088
1 parent 69bbde1
commit 1212088
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 63 deletions.
diff --git a/data/System/NativeSearchContrib.txt b/data/System/NativeSearchContrib.txt
@@ -12,6 +12,8 @@ By default the RCS-based store in Foswiki searches content by [[http://en.wikipe
 
 *You require shell access to the server to install this module*.
 
+Tested with Foswiki 1.0.9 and 1.1
+
 ---++ Installation Instructions
 
 Before you start, there are some prerequisites.
@@ -62,6 +64,7 @@ Many thanks to the following sponsors for supporting this work:
 |  Version: | %$VERSION% |
 |  Release: | %$RELEASE% |
 |  Change History: | <!-- versions below in reverse order -->&nbsp; |
+|  1.3 | Restored missing 1.0 support :-8 |
 |  1.2 | Changes for Foswiki 1.1 |
 |  1.1 | Initial release |
 |  Dependencies: | %$DEPENDENCIES% |

diff --git a/lib/Foswiki/Contrib/NativeSearchContrib.pm b/lib/Foswiki/Contrib/NativeSearchContrib.pm
@@ -4,7 +4,7 @@ package Foswiki::Contrib::NativeSearchContrib;
 use strict;
 
 our $VERSION          = '$Rev$';
-our $RELEASE          = '1.2';
+our $RELEASE          = '1.3';
 our $SHORTDESCRIPTION = 'Accelerated search for use with mod_perl';
 
 1;

diff --git a/lib/Foswiki/Store/SearchAlgorithms/Native.pm b/lib/Foswiki/Store/SearchAlgorithms/Native.pm
@@ -3,8 +3,6 @@ package Foswiki::Store::SearchAlgorithms::Native;
 
 use Assert;
 use FoswikiNativeSearch ();
-use Foswiki::Search::InfoCache ();
-use Foswiki::Search::ResultSet ();
 
 =pod
 
@@ -13,11 +11,13 @@ use Foswiki::Search::ResultSet ();
 Native implementation of the RCS cache search. Requires tools/native_search
 to be built and installed.
 
+Rude and crude, this makes no attempt to handle UTF-8.
+
 ---++ query($searchString, $topics, $options, $sDir) -> \%seen
 Search .txt files in $dir for $string. See RcsFile::searchInWebContent
 for details.
 
-Rude and crude, this makes no attempt to handle UTF-8.
+Foswiki 1.1 only
 
 SMELL: 'query' and '_webQuery' are duplicated from
 Foswiki::Store::SearchAlgorithms::Forking - doesn't seem to be any
@@ -28,6 +28,10 @@ sensible way to enable re-use.
 sub query {
     my ( $query, $inputTopicSet, $session, $options ) = @_;
 
+    # These are loaded by require because 1.0 doesn't have them
+    require Foswiki::Search::InfoCache;
+    require Foswiki::Search::ResultSet;
+
     if (( @{ $query->{tokens} } ) == 0) {
         return new Foswiki::Search::InfoCache($session, '');
     }
@@ -37,15 +41,17 @@ sub query {
     my $isAdmin = $session->{users}->isAdmin( $session->{user} );
 
     my $searchAllFlag = ( $webNames =~ /(^|[\,\s])(all|on)([\,\s]|$)/i );
-    my @webs = Foswiki::Search::InfoCache::_getListOfWebs( $webNames, $recurse, $searchAllFlag );
+    my @webs = Foswiki::Search::InfoCache::_getListOfWebs(
+        $webNames, $recurse, $searchAllFlag );
 
     my @resultCacheList;
     foreach my $web (@webs) {
         # can't process what ain't thar
         next unless $session->webExists($web);
 
         my $webObject = Foswiki::Meta->new( $session, $web );
-        my $thisWebNoSearchAll = $webObject->getPreference('NOSEARCHALL') || '';
+        my $thisWebNoSearchAll = $webObject->getPreference('NOSEARCHALL')
+          || '';
 
         # make sure we can report this web on an 'all' search
         # DON'T filter out unless it's part of an 'all' search.
@@ -55,11 +61,14 @@ sub query {
             && ( $thisWebNoSearchAll =~ /on/i || $web =~ /^[\.\_]/ )
             && $web ne $session->{webName} );
 
-        my $infoCache = _webQuery($query, $web, $inputTopicSet, $session, $options);
+        my $infoCache = _webQuery(
+            $query, $web, $inputTopicSet, $session, $options);
         $infoCache->sortResults( $options );
         push(@resultCacheList, $infoCache);
     }
-    my $resultset = new Foswiki::Search::ResultSet(\@resultCacheList, $options->{groupby}, $options->{order}, Foswiki::isTrue( $options->{reverse} ));
+    my $resultset = new Foswiki::Search::ResultSet(
+        \@resultCacheList, $options->{groupby}, $options->{order},
+        Foswiki::isTrue( $options->{reverse} ));
     #TODO: $options should become redundant
     $resultset->sortResults( $options );
     return $resultset;
@@ -70,7 +79,7 @@ sub query {
 sub _webQuery {
     my ( $query, $web, $inputTopicSet, $session, $options ) = @_;
     ASSERT( scalar( @{ $query->{tokens} } ) > 0 ) if DEBUG;
-#print STDERR "ForkingSEARCH(".join(', ', @{ $query->{tokens} }).")\n";
+    #print STDERR "ForkingSEARCH(".join(', ', @{ $query->{tokens} }).")\n";
     # default scope is 'text'
     $options->{'scope'} = 'text'
       unless ( defined( $options->{'scope'} )
@@ -81,12 +90,14 @@ sub _webQuery {
         #then we start with the whole web
         #TODO: i'm sure that is a flawed assumption
         my $webObject = Foswiki::Meta->new( $session, $web );
-        $topicSet = Foswiki::Search::InfoCache::getTopicListIterator( $webObject, $options );
+        $topicSet = Foswiki::Search::InfoCache::getTopicListIterator(
+            $webObject, $options );
     }
     ASSERT( UNIVERSAL::isa( $topicSet, 'Foswiki::Iterator' ) ) if DEBUG;
 
-#print STDERR "######## Forking search ($web) tokens ".scalar(@{$query->{tokens}})." : ".join(',', @{$query->{tokens}})."\n";
-# AND search - search once for each token, ANDing result together
+    #print STDERR "######## Forking search ($web) tokens "
+    # .scalar(@{$query->{tokens}})." : ".join(',', @{$query->{tokens}})."\n";
+    # AND search - search once for each token, ANDing result together
     foreach my $token ( @{ $query->{tokens} } ) {
 
         my $tokenCopy = $token;
@@ -101,8 +112,9 @@ sub _webQuery {
         unless ( $options->{'scope'} eq 'text' ) {
             my $qtoken = $tokenCopy;
 
-# FIXME I18N
-# http://foswiki.org/Tasks/Item1646 this causes us to use/leak huge amounts of memory if called too often
+            # FIXME I18N
+            # http://foswiki.org/Tasks/Item1646 this causes us to use/leak
+            # huge amounts of memory if called too often
             $qtoken = quotemeta($qtoken) if ( $options->{'type'} ne 'regex' );
 
             my @topicList;
@@ -141,7 +153,8 @@ sub _webQuery {
             $topicSet->reset();
             while ( $topicSet->hasNext() ) {
                 my $webtopic = $topicSet->next();
-                my ($Iweb, $topic) = Foswiki::Func::normalizeWebTopicName($web, $webtopic);
+                my ($Iweb, $topic) = Foswiki::Func::normalizeWebTopicName(
+                    $web, $webtopic);
 
                 if ( $topicMatches{$topic} ) {
                 } else {
@@ -174,11 +187,12 @@ sub search {
     my $sDir;
 
     if (ref($web)) {
-        # 1.0.9 and earlier had ($searchString, $topics, $options, $sDir)
-        # remap                 ($searchString, $web,    $topics,  $session
+        # 1.0.9 and earlier had ($searchString, \@topics, $options, $sDir)
+        # remap                 ($searchString, $web,    \@topics,  $session
         $sDir = $session;
         $options = $topics;
-        $topics = $web;
+        # add dir and extension to topic names
+        $topics = [ map { "$sDir/$_.txt" } @$web ];
     } else {
         $sDir = "$Foswiki::cfg{DataDir}/$web";
         # Flatten the iterator

diff --git a/tools/native_search/FoswikiNativeSearch.xs b/tools/native_search/FoswikiNativeSearch.xs
@@ -9,16 +9,13 @@
 #include "XSUB.h"
 
 /*
- * Unpack perl args into an array of (read only) strings. The function name
+ * Unpack perl args into a null-terminated array of strings. The function name
  * is dictated by the mapping in the default typemap i.e.
  * (char** -> T_PACKEDARRAY -> XS_unpack_charPtrPtr
+ * The array of strings is allocated on the heap.
  */
 char ** XS_unpack_charPtrPtr(SV* rv) {
 	AV *av;
-	SV **ssv;
-	char **s;
-	int avlen;
-	int x;
 
 	if (SvROK(rv) && (SvTYPE(SvRV(rv)) == SVt_PVAV))
 		av = (AV*)SvRV(rv);
@@ -27,58 +24,49 @@ char ** XS_unpack_charPtrPtr(SV* rv) {
 		return ((char**)NULL);
 	}
 
-	/* is it empty? */
-	avlen = av_len(av);
-	if (avlen < 0){
-		warn("unpack_args: array was empty");
-		return ((char**)NULL);
-	}
+	int length = av_len(av) + 1; /* av_len is the last index */
 
-	/* av_len+2 == number of strings, plus 1 for an end-of-array sentinel.
-	 */
-	s = (char **)malloc(sizeof(char*) * (avlen + 2));
-	if (s == NULL){
-		warn("unpack_args: unable to malloc char**");
-		return ((char**)NULL);
-	}
-	for (x = 0; x <= avlen; ++x){
-        s[x] = (char*)NULL;
-		ssv = av_fetch(av, x, 0);
-		if (ssv != NULL){
-            s[x] = (char *)malloc( SvCUR(*ssv) + 1 );
-            // Test commented out; fails with some perl versions, for no
-            // good reason
-			//if (SvPOK(*ssv))
-				strcpy(s[x], SvPV(*ssv, PL_na));
-			//else
-			//	warn("unpack_args: array elem %d was not a string.", x);
-		}
-	}
-	s[x] = (char*)NULL; /* sentinel */
+    /* array is null-terminated */
+    char ** s = (char **)calloc(length + 1, sizeof(char*));
+    if (!s) {
+    	warn("unpack_args: unable to allocate char**");
+    	return ((char**)NULL);
+    }
+
+    int i;
+    for (i = 0; i < length; i++) {
+    	SV ** ssv = av_fetch(av, i, 0);
+    	if (ssv) {
+            s[i] = (char *)calloc( SvCUR(*ssv) + 1, sizeof(char) );
+            /* *Requires* that data be char strings */
+     		strcpy(s[i], SvPV(*ssv, PL_na));
+    	}
+    }
 	return s;
 }
 
 /*
- * Convert a C char** to a Perl AV*, freeing the char** and the strings
+ * Convert a char** to a Perl AV*, freeing the char** and the strings
  * stored in it.  The function name is dictated by the mapping in the
  * default typemap i.e.
  * (char** -> T_PACKEDARRAY -> XS_pack_charPtrPtr
  */
 void XS_pack_charPtrPtr(SV* st, char **s, int n) {
 	AV *av = newAV();
-	SV *sv;
-	char **c;
-    if (!s)
-        return;
-	for(c = s; *c; c++){
-		sv = newSVpv(*c, 0);
-		av_push(av, sv);
-        free(*c);
-	}
-	sv = newSVrv(st, NULL);	  /* upgrade stack SV to an RV */
+
+    if (s) {
+    	char **c;
+    	for (c = s; *c; c++) {
+    		SV * sv = newSVpv(*c, 0);
+    		av_push(av, sv);
+            free(*c);
+    	}
+        free(s);
+    }
+
+	SV * sv = newSVrv(st, NULL);	  /* upgrade stack SV to an RV */
 	SvREFCNT_dec(sv);         /* discard */
 	SvRV(st) = (SV*)av;       /* make stack RV point at our AV */
-    free(s);
 }
 
 MODULE = FoswikiNativeSearch     PACKAGE = FoswikiNativeSearch
@@ -87,4 +75,4 @@ char**
 cgrep(argv)
 	char ** argv
     PREINIT:
-        int count_charPtrPtr;
+        int count_charPtrPtr; /* Dummy, unused */
diff --git a/tools/native_search/cgrep.c b/tools/native_search/cgrep.c
@@ -160,6 +160,7 @@ char** cgrep(char** argv) {
             warn("Open '%s' failed %s\n", fname, strerror(errno));
         }
     }
+    free(pattern);
     free(linebuf);
     result = _backup(matchCacheSize, matchCache, result);
     cleanup(argv);