Item10657:

* don't store catchall, just index it * make all language specifc fields dynamic fields * removing sharedLibs from solr.xml as it defeats any other <lib dir="..." /> definitions in solrconfig.xml (wtf) * simplifying class path definition in solrconfig.xml * CONTENT_LANGUAGE now defaults to 'detect' instead of 'en' * added automatic language detection in case CONTENT_LANGUAGE is unset or equals 'detect' * added velocity template handler to browse content on the solr server directly * use text field for highlightning instead of catchall (must be a stored field, which catchall isn't anymore) * replaced contributor facet with author facet in default SolrSearch interface (makes more sense most of the time) git-svn-id: http://svn.foswiki.org/trunk/SolrPlugin@14470 0b4bb1d4-4e5a-0410-9cc4-b2b747904278
foswiki · Mar 26, 2012 · b94f670 · b94f670
1 parent 627025e
commit b94f670
Show file tree

Hide file tree

Showing 9 changed files with 248 additions and 204 deletions.
diff --git a/data/System/SolrPlugin.txt b/data/System/SolrPlugin.txt
@@ -505,7 +505,7 @@ useful for spacial search.
 | spell | text_spell | multivalued | | used for spellchecking |
 | id | string | | stored | unique identifier for each document; this is the _external_ id usable in applications; there's an internal solr document id not related to this field |
 | collection | string | | stored | identifies a set of documents comming from the same content collection; by default all content stored in Foswiki (topics and attachments) is gathered in the =wiki= collection set in =Foswiki::cfg{SolrPlugin}{DefaultCollection}= |
-| language | string | | stored | language of the current document; this is derived from  =Foswiki::cfg{Site}{Locale}= but may be overwritten using the =CONTENT_LANGUAGE= preference |
+| language | string | | stored | language of the current document; this may be specified explicitly using the =CONTENT_LANGUAGE= preference, or set to "detect" to let the solr update chain detect the language automatically |
 | url | string | | stored | url used to  access the document being indexed |
 | type | type | | stored | holds the type facet of the document; this is "image" for all kinds of images, "video" for all kinds of videos, "topic" for Foswiki topics and the verbatim file extension for everything else; note: plugins like Foswiki:Extensions/MetaCommentPlugin might use specific types as well (like "comment" in this case) |
 | web | string | | stored | name of the web this document is located in |

diff --git a/data/System/SolrSearchBaseTemplate.txt b/data/System/SolrSearchBaseTemplate.txt
@@ -65,10 +65,11 @@
   %MAKETEXT{"Topic type"}%=field_TopicType_lst,
   %MAKETEXT{"Collection"}%=collection,
   %MAKETEXT{"Language"}%=language,
-  %MAKETEXT{"Top contributors"}%=contributor,
+  %MAKETEXT{"Author"}%=author,
 %TMPL:END%
 %TMPL:DEF{"solr::extramapping"}%
   %MAKETEXT{"Contributor"}%=contributor,
+  %MAKETEXT{"Author"}%=Author,
   %MAKETEXT{"Timespan"}%=date,
   %MAKETEXT{"Keyword"}%=keyword,
   %MAKETEXT{"English"}%=en,
@@ -96,7 +97,7 @@
   %MAKETEXT{"1 year"}%=date:[NOW-1YEAR TO NOW],
   %MAKETEXT{"older"}%=date:[* TO NOW/YEAR]
 %TMPL:END%
-%TMPL:DEF{"solr::hidesinglefacets"}%web, type, collection, language, contributor, field_TopicType_lst, facetquery%TMPL:END%
+%TMPL:DEF{"solr::hidesinglefacets"}%web, type, collection, language, author, field_TopicType_lst, facetquery%TMPL:END%
 %TMPL:DEF{"solr::disjunctivefacets"}%%TMPL:END%
 %TMPL:DEF{"solr::combinedfacets"}%%TMPL:END%
 %TMPL:DEF{"solr::facet::include::web"}%%TMPL:END%
@@ -107,7 +108,9 @@
   then="ApplicationLicense|TopicStub|TopicType|ClassifiedTopic|TaggedTopic|CategorizedTopic|DataForm|WikiTopic|WebTool|ApplicationTopic|DocuTopic|TopicView|TopicTemplate|TopicFunction|WikiApplication"
 }%%TMPL:END%
 %TMPL:DEF{"solr::facet::include::contributor"}%%TMPL:END%
-%TMPL:DEF{"solr::facet::exclude::contributor"}%UnknownUser|WikiGuest|(TWiki|Project)Contributor|AdminGroup|RegistrationAgent|AdminUser|TestUser%TMPL:END%
+%TMPL:DEF{"solr::facet::exclude::contributor"}%UnknownUser|WikiGuest|(TWiki|Project|Foswiki)Contributor|AdminGroup|RegistrationAgent|AdminUser|TestUser%TMPL:END%
+%TMPL:DEF{"solr::facet::include::author"}%%TMPL:P{"solr::facet::include::contributor"}%%TMPL:END%
+%TMPL:DEF{"solr::facet::exclude::author"}%%TMPL:P{"solr::facet::exclude::contributor"}%%%TMPL:END%
 %TMPL:DEF{"solr::facet::include::type"}%%TMPL:END%
 %TMPL:DEF{"solr::facet::exclude::type"}%%TMPL:END%
 %TMPL:DEF{"solr::facet::include::collection"}%%TMPL:END%
@@ -191,7 +194,6 @@ jQuery(function($) {
   id="solr1"
   jump="%URLPARAM{"jump"}%"
   lucky="%URLPARAM{"lucky"}%"
-  contributor="%URLPARAM{"contributor"}%"
   display="%URLPARAM{"display" default="%TMPL:P{"solr::defaultdisplay"}%"}%"
   origtopic="%URLPARAM{"origtopic" default="%BASEWEB%.%BASETOPIC%"}%"
   type="%URLPARAM{"type" default="%TMPL:P{"solr::defaultquerytype"}%"}%"
@@ -210,7 +212,7 @@ jQuery(function($) {
   facetdateend="%TMPL:P{"solr::facetdateend"}%"
   facetdategap="%TMPL:P{"solr::facetdategap"}%"
   facetdateother="%TMPL:P{"solr::facetdategother"}%"
-  facetlimit="50,tag=100,contributor=10,category=50"
+  facetlimit="50,tag=100,author=10,category=50"
   facetsort="%TMPL:P{"solr::facetsort"}%"
   filter="%URLPARAM{"filter" multiple="on" separator=", "}%"
   web="%URLPARAM{"web" default="%TMPL:P{"solr::defaultweb"}%"}%" 
@@ -293,13 +295,13 @@ jQuery(function($) {
     pageropts=""
   }%
   %TMPL:P{"solr::facet"
-    id="contributor"
+    id="author"
     format="$percntSPACEOUT{$key}$percnt&nbsp;($count)"
     title="$percntSPACEOUT{$key}$percnt"
     value="$id:$key"
     input="checkbox"
-    include="%TMPL:P{"solr::facet::include::contributor"}%"
-    exclude="%TMPL:P{"solr::facet::exclude::contributor"}%"
+    include="%TMPL:P{"solr::facet::include::author"}%"
+    exclude="%TMPL:P{"solr::facet::exclude::author"}%"
     extraclass=""
     pageropts=""
   }%
@@ -395,7 +397,7 @@ jQuery(function($) {
    called in "solr::sidebar" macro. 
 
    Parameters:
-      * id - the identifier of the current facet, e.g. category, contributor
+      * id - the identifier of the current facet, e.g. category, author
              this must be included in the "solr::facets" list
       * format - format of a single facet value 
       * value - when a facet value is selected this value is added to the next query

diff --git a/lib/Foswiki/Plugins/SolrPlugin.pm b/lib/Foswiki/Plugins/SolrPlugin.pm
@@ -16,6 +16,7 @@ package Foswiki::Plugins::SolrPlugin;
 use strict;
 use Foswiki::Func ();
 use Foswiki::Plugins ();
+use Error qw(:try);
 
 our $VERSION = '$Rev: 20091124 (2009-11-24) $';
 our $RELEASE = '1.10';
@@ -34,15 +35,83 @@ sub initPlugin {
   %searcher = ();
   %indexer = ();
 
-  Foswiki::Func::registerTagHandler('SOLRSEARCH', \&SOLRSEARCH);
-  Foswiki::Func::registerTagHandler('SOLRFORMAT', \&SOLRFORMAT);
-  Foswiki::Func::registerTagHandler('SOLRSIMILAR', \&SOLRSIMILAR);
-  Foswiki::Func::registerTagHandler('SOLRSCRIPTURL', \&SOLRSCRIPTURL);
-  Foswiki::Func::registerRESTHandler('search', \&restSOLRSEARCH);
-  Foswiki::Func::registerRESTHandler('terms', \&restSOLRTERMS);
-  Foswiki::Func::registerRESTHandler('similar', \&restSOLRSIMILAR);
-  Foswiki::Func::registerRESTHandler('autocomplete', \&restSOLRAUTOCOMPLETE);
-  Foswiki::Func::registerRESTHandler('optimize', \&restOPTIMIZE);
+  Foswiki::Func::registerTagHandler('SOLRSEARCH', sub {
+    my ($session, $params, $theTopic, $theWeb) = @_;
+
+    return getSearcher($session)->handleSOLRSEARCH($params, $theWeb, $theTopic);
+  });
+
+
+  Foswiki::Func::registerTagHandler('SOLRFORMAT', sub {
+    my ($session, $params, $theTopic, $theWeb) = @_;
+
+    return getSearcher($session)->handleSOLRFORMAT($params, $theWeb, $theTopic);
+  });
+
+
+  Foswiki::Func::registerTagHandler('SOLRSIMILAR', sub {
+    my ($session, $params, $theTopic, $theWeb) = @_;
+
+    return getSearcher($session)->handleSOLRSIMILAR($params, $theWeb, $theTopic);
+  });
+
+  Foswiki::Func::registerTagHandler('SOLRSCRIPTURL', sub {
+    my ($session, $params, $theTopic, $theWeb) = @_;
+
+    return getSearcher($session)->handleSOLRSCRIPTURL($params, $theWeb, $theTopic);
+  });
+
+
+  Foswiki::Func::registerRESTHandler('search', sub {
+    my $session = shift;
+
+    my $web = $session->{webName};
+    my $topic = $session->{topicName};
+    return getSearcher($session)->restSOLRSEARCH($web, $topic);
+  });
+
+
+  Foswiki::Func::registerRESTHandler('terms', sub {
+    my $session = shift;
+
+    my $web = $session->{webName};
+    my $topic = $session->{topicName};
+    return getSearcher($session)->restSOLRTERMS($web, $topic);
+  });
+
+
+  Foswiki::Func::registerRESTHandler('similar', sub {
+    my $session = shift;
+
+    my $web = $session->{webName};
+    my $topic = $session->{topicName};
+    return getSearcher($session)->restSOLRSIMILAR($web, $topic);
+  });
+
+  Foswiki::Func::registerRESTHandler('autocomplete', sub {
+    my $session = shift;
+
+    my $web = $session->{webName};
+    my $topic = $session->{topicName};
+    return getSearcher($session)->restSOLRAUTOCOMPLETE($web, $topic);
+  });
+
+
+  Foswiki::Func::registerRESTHandler('optimize', sub {
+    my $session = shift;
+    return getIndexer($session)->optimize();
+  });
+
+  Foswiki::Func::registerRESTHandler('crawl', sub {
+    my $session = shift;
+
+    my $query = Foswiki::Func::getCgiQuery();
+    my $name = $query->param("name");
+    my $path = $query->param("path");
+    my $depth = $query->param("depth");
+
+    return getCrawler($session, $name)->crawl($path, $depth);
+  });
 
   return 1;
 }
@@ -77,66 +146,22 @@ sub getIndexer {
   return $indexer;
 }
 
-sub SOLRSEARCH {
-  my ($session, $params, $theTopic, $theWeb) = @_;
-
-  return getSearcher($session)->handleSOLRSEARCH($params, $theWeb, $theTopic);
-}
-
-sub SOLRFORMAT {
-  my ($session, $params, $theTopic, $theWeb) = @_;
-
-  return getSearcher($session)->handleSOLRFORMAT($params, $theWeb, $theTopic);
-}
-
-sub SOLRSIMILAR {
-  my ($session, $params, $theTopic, $theWeb) = @_;
-
-  return getSearcher($session)->handleSOLRSIMILAR($params, $theWeb, $theTopic);
-}
-
-sub SOLRSCRIPTURL {
-  my ($session, $params, $theTopic, $theWeb) = @_;
-
-  return getSearcher($session)->handleSOLRSCRIPTURL($params, $theWeb, $theTopic);
-}
-
-sub restOPTIMIZE {
-  my $session = shift;
-
-  getIndexer($session)->optimize();
-}
-
-sub restSOLRSEARCH {
-  my $session = shift;
+sub getCrawler {
+  my ($session , $name) = @_;
 
-  my $web = $session->{webName};
-  my $topic = $session->{topicName};
-  return getSearcher($session)->restSOLRSEARCH($web, $topic);
-}
+  throw Error::Simple("no crawler name") unless defined $name;
+
+  my $params = $Foswiki::cfg{SolrPlugin}{Crawler}{$name};
 
-sub restSOLRTERMS {
-  my $session = shift;
-
-  my $web = $session->{webName};
-  my $topic = $session->{topicName};
-  return getSearcher($session)->restSOLRTERMS($web, $topic);
-}
+  throw Error::Simple("unknown crawler $name") unless defined $params;
 
-sub restSOLRSIMILAR {
-  my $session = shift;
-
-  my $web = $session->{webName};
-  my $topic = $session->{topicName};
-  return getSearcher($session)->restSOLRSIMILAR($web, $topic);
-}
-
-sub restSOLRAUTOCOMPLETE {
-  my $session = shift;
+  my $module = $params->{module};
+  eval "use $module";
+  if ($@) {
+    throw Error::Simple($@);
+  }
 
-  my $web = $session->{webName};
-  my $topic = $session->{topicName};
-  return getSearcher($session)->restSOLRAUTOCOMPLETE($web, $topic);
+  return $module->new($session, %$params);
 }
 
 sub indexCgi {

diff --git a/lib/Foswiki/Plugins/SolrPlugin/Base.pm b/lib/Foswiki/Plugins/SolrPlugin/Base.pm
@@ -1,6 +1,6 @@
 # Plugin for Foswiki - The Free and Open Source Wiki, http://foswiki.org/
 #
-# Copyright (C) 2009-2011 Michael Daum http://michaeldaumconsulting.com
+# Copyright (C) 2009-2012 Michael Daum http://michaeldaumconsulting.com
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -14,6 +14,8 @@
 package Foswiki::Plugins::SolrPlugin::Base;
 
 use strict;
+use warnings;
+
 use Foswiki::Func ();
 use Foswiki::Plugins ();
 use WebService::Solr ();

diff --git a/lib/Foswiki/Plugins/SolrPlugin/Index.pm b/lib/Foswiki/Plugins/SolrPlugin/Index.pm
@@ -13,6 +13,7 @@
 
 package Foswiki::Plugins::SolrPlugin::Index;
 use strict;
+use warnings;
 
 use Foswiki::Plugins::SolrPlugin::Base ();
 our @ISA = qw( Foswiki::Plugins::SolrPlugin::Base );
@@ -368,7 +369,7 @@ sub indexTopic {
 
   # tag and analyze language
   my $contentLanguage = $this->getContentLanguage($web, $topic);
-  if (defined $contentLanguage) {
+  if (defined $contentLanguage && $contentLanguage ne 'detect') {
     $doc->add_fields(
       language => $contentLanguage,
       'text_'.$contentLanguage => $text,
@@ -568,12 +569,9 @@ sub getContentLanguage {
   }
 
   my $prefsLanguage = Foswiki::Func::getPreferencesValue('CONTENT_LANGUAGE') || '';
-  my $siteLanguage = $Foswiki::cfg{Site}{Locale} || 'en';
-  $siteLanguage =~ s/_.*$//; # the prefix: e.g. de, en
+  my $contentLanguage = $Foswiki::cfg{SolrPlugin}{SupportedLanguages}{$prefsLanguage} || 'detect';
 
-  my $contentLanguage = $Foswiki::cfg{SolrPlugin}{SupportedLanguages}{$prefsLanguage || $siteLanguage};
-
-  #$this->log("contentLanguage=$contentLanguage");
+  $this->log("contentLanguage=$contentLanguage");
 
   Foswiki::Func::popTopicContext() if $donePush;
 
@@ -716,7 +714,7 @@ sub indexAttachment {
   # tag and analyze language
   # SMELL: silently assumes all attachments to a topic are the same langauge
   my $contentLanguage = $this->getContentLanguage($web, $topic);
-  if (defined $contentLanguage) {
+  if (defined $contentLanguage && $contentLanguage ne 'detect') {
     $doc->add_fields(
       language => $contentLanguage,
       'text_'.$contentLanguage => $attText,
@@ -1134,6 +1132,7 @@ sub getGrantedUsers {
   # set {knownUsers} and {nrKnownUsers}
   $this->getListOfUsers();
 
+  $text = $meta->text() unless defined $text;
   $text ||= '';
 
   my @grantedUsers = ();
@@ -1186,14 +1185,12 @@ sub getGrantedUsers {
 
 ################################################################################
 sub getAclFields {
-  my ($this, $web, $topic, $meta, $text) = @_;
-
-  $text = $meta->text() unless defined $text;
+  my $this = shift;
 
   my @aclFields = ();
 
   # permissions
-  my @grantedUsers = $this->getGrantedUsers($web, $topic, $meta, $text);
+  my @grantedUsers = $this->getGrantedUsers(@_);
   foreach my $wikiName (@grantedUsers) {
     push @aclFields, 'access_granted' => $wikiName;
   }