diff --git a/data/System/DBIStoreContrib.txt b/data/System/DBIStoreContrib.txt new file mode 100644 index 0000000..1e29e53 --- /dev/null +++ b/data/System/DBIStoreContrib.txt @@ -0,0 +1,42 @@ +---+!! !DBIStoreContrib + +%SHORTDESCRIPTION% + +%TOC% + +*This extension will *not* work with Foswiki 1.1.x or earlier.* + +This extension (currently) implements search operations (query and text search) using [[http://www.sqlite.org/][SQLite]], the popular lightweight implementation of an SQL relational database. It uses the special cache hooks recently added to the Foswiki RCS store to cache topics in the database, and executes %SEARCH= using SQL. We get over the fact that SQL does not implement all the query search features of Foswiki by _hoisting_ SQL expressions out of the Foswiki search statements, leaving behind only those parts of the expression that SQL can't handle. + +*If you are experienced with SQLite and / or other SQL DMBS, you are invited with open arms to contribute to the further development of this extension.* + +The extension is currently classed as experimental because it has a number of problems: + 1 The SQL schema (and the query generator) are (probably) sub-optimal, and require extensive tuning. + 1 SQLite is fine for small data, but does not scale well. Large data requires use of an industrial strength DB (such as MySQL) instead. + +The longer term goal is to implement a full back-end store using an SQL RDBMS, rather than just a simple cache as at present. + +Mapping to another DB *should* be as simple as setting up a different +DSN, but life is never that simple. Not all DBs imlpement the REGEXP +operator, for example. Please go ahead and try, though. + +---++ Installation Instructions + +%$INSTALL_INSTRUCTIONS% + * Go to =configure= and set a destination + +---++ Info + +| Author(s): | Crawford Currie http://c-dot.co.uk | +| Copyright: | © | +| License: | [[http://www.gnu.org/licenses/gpl.html][GPL (Gnu General Public License)]] | +| Release: | %$RELEASE% | +| Version: | %$VERSION% | +| Change History: |   | +| Dependencies: | %$DEPENDENCIES% | +| Home page: | http://foswiki.org/bin/view/Extensions/DBIStoreContrib | +| Support: | http://foswiki.org/bin/view/Support/DBIStoreContrib | + + diff --git a/lib/Foswiki/Contrib/DBIStoreContrib.pm b/lib/Foswiki/Contrib/DBIStoreContrib.pm new file mode 100644 index 0000000..53401a7 --- /dev/null +++ b/lib/Foswiki/Contrib/DBIStoreContrib.pm @@ -0,0 +1,11 @@ +# See bottom of file for license and copyright information. +package Foswiki::Contrib::DBIStoreContrib; + +use strict; + +our $VERSION = '$Rev$'; # version of *this file*. + +our $RELEASE = '1.0'; + +our $SHORTDESCRIPTION = '(Experimental) use of DBI to implement an SQL query search'; + diff --git a/lib/Foswiki/Contrib/DBIStoreContrib/Config.spec b/lib/Foswiki/Contrib/DBIStoreContrib/Config.spec new file mode 100644 index 0000000..1bba18a --- /dev/null +++ b/lib/Foswiki/Contrib/DBIStoreContrib/Config.spec @@ -0,0 +1,11 @@ +#--+ Extensions +#--++ DBIStoreContrib +# **STRING 120** +# DBI DSN to use to connect to the database. +$Foswiki::cfg{Extensions}{DBIStoreContrib}{DSN} = 'dbi:SQLite:dbname=$Foswiki::cfg{WorkingDir}/dbcache'; +# **STRING 80** +# Username to use to connect to the database. +$Foswiki::cfg{Extensions}{DBIStoreContrib}{Username} = ''; +# **STRING 80** +# Password to use to connect to the database. +$Foswiki::cfg{Extensions}{DBIStoreContrib}{Password} = ''; diff --git a/lib/Foswiki/Contrib/DBIStoreContrib/DEPENDENCIES b/lib/Foswiki/Contrib/DBIStoreContrib/DEPENDENCIES new file mode 100644 index 0000000..6a7bab6 --- /dev/null +++ b/lib/Foswiki/Contrib/DBIStoreContrib/DEPENDENCIES @@ -0,0 +1,5 @@ +# Dependencies for DBIStoreContrib +# Example: +# Time::ParseDate,>=2003.0211,cpan,Required. +# Foswiki::Plugins,>=1.2,perl,Requires version 1.2 of handler API. + diff --git a/lib/Foswiki/Contrib/DBIStoreContrib/HoistSQL.pm b/lib/Foswiki/Contrib/DBIStoreContrib/HoistSQL.pm new file mode 100644 index 0000000..c2fe082 --- /dev/null +++ b/lib/Foswiki/Contrib/DBIStoreContrib/HoistSQL.pm @@ -0,0 +1,331 @@ +# See bottom of file for copyright and license details + +=begin TML + +---+ package Foswiki::Contrib::DBIStoreContrib::HoistSQL + +Static functions to extract SQL expressions from queries. The SQL can +be used to pre-filter topics cached in an SQL DB for more efficient +query matching. + +=cut + +package Foswiki::Contrib::DBIStoreContrib::HoistSQL; + +use strict; + +use Foswiki::Infix::Node (); +use Foswiki::Query::Node (); + +# Try to optimise a query by hoisting SQL searches +# out of the query. +# +# patterns we need to look for: +# +# top level is defined by a sequence of AND and OR conjunctions +# second level, operator that can be mapped to SQL +# second level LHS is a field access +# second level RHS is a static string or number +# So, say I have: +# number=99 AND string='String' AND (moved.by='AlbertCamus' +# OR moved.by ~ '*bert*') +# This can be fully hoisted, to +# SELECT topic.name FROM topic,FIELD,MOVED +# WHERE +# EXISTS( +# SELECT tid FROM FIELD +# WHERE FIELD.tid=topic.tid AND FIELD.name='number' AND FIELD.value=99) +# AND +# EXISTS( +# SELECT tid FROM FIELD +# WHERE FIELD.tid=topic.tid AND FIELD.name='string' +# AND FIELD.string='String') +# AND ( +# EXISTS( +# SELECT tid FROM FIELD +# WHERE MOVED.tid=topic.tid AND MOVED.by='AlbertCamus') +# OR +# EXISTS( +# SELECT tid FROM FIELD +# WHERE MOVED.tid=topic.tid AND MOVED.by LIKE '%bert%') +# ) + +use constant MONITOR => 0; + +=begin TML + +---++ ObjectMethod hoist($query) -> $sql_statement + +Hoisting consists of assembly of a WHERE clause. There may be a +point where the expression can't be converted to SQL, because some operator +(for example, a date operator) can't be done in SQL. But in most cases +the hoisting allows us to extract a set of criteria that can be AND and +ORed together in an SQL statement sufficient to narrow down and isolate +that subset of topics that might match the query. + +The result is a string SQL query, and the $query is modified to replace +the hoisted expressions with constants. + +=cut + +sub hoist { + my ($node, $indent) = @_; + + return undef unless ref( $node->{op} ); + + $indent ||= ''; + + if ( $node->{op}->{name} eq '(' ) { + return hoist( $node->{params}[0], "$indent(" ); + } + + print STDERR "${indent}hoist ", $node->stringify(), "\n" if MONITOR; + if ( $node->{op}->{name} eq 'and' ) { + my $lhs = hoist( $node->{params}[0], "${indent}l" ); + my $rhs = _hoistB( $node->{params}[1], "${indent}r" ); + if ( $lhs && $rhs ) { + $node->makeConstant(Foswiki::Infix::Node::NUMBER, 1); + print STDERR "${indent}L&R\n" if MONITOR; + return "($lhs) AND ($rhs)"; + } + elsif ( $lhs ) { + $node->{params}[0]->makeConstant(Foswiki::Infix::Node::NUMBER, 1); + print STDERR "${indent}L\n" if MONITOR; + return $lhs; + } + elsif ($rhs) { + $node->{params}[1]->makeConstant(Foswiki::Infix::Node::NUMBER, 1); + print STDERR "${indent}R\n" if MONITOR; + return $rhs; + } + } + else { + my $or = _hoistB($node, "${indent}|"); + if ($or) { + $node->makeConstant(Foswiki::Infix::Node::NUMBER, 1); + return $or; + } + } + + print STDERR "\tFAILED\n" if MONITOR; + return undef; +} + +sub _hoistB { + my ($node, $indent) = @_; + + return unless ref( $node->{op} ); + + if ( $node->{op}->{name} eq '(' ) { + return _hoistB( $node->{params}[0], "${indent}(" ); + } + + print STDERR "${indent}OR ", $node->stringify(), "\n" if MONITOR; + + if ( $node->{op}->{name} eq 'or' ) { + my $lhs = _hoistB( $node->{params}[0], "${indent}l" ); + my $rhs = _hoistC( $node->{params}[1], "${indent}r", 0 ); + if ( $lhs && $rhs ) { + print STDERR "${indent}L&R\n" if MONITOR; + return "($lhs) OR ($rhs)"; + } + } + else { + return _hoistC($node, "${indent}|", 0); + } + + return undef; +} + +sub _hoistC { + my ($node, $indent, $negated) = @_; + + return undef unless ref( $node->{op} ); + + my $op = $node->{op}->{name}; + if ( $op eq '(' ) { + return _hoistC( $node->{params}[0], "${indent}(", $negated ); + } + + print STDERR "${indent}EQ ", $node->stringify(), "\n" if MONITOR; + my ($lhs, $rhs, $table, $test); + + if ( $op eq 'not' ) { + return _hoistC( $node->{params}[0], "${indent}(", !$negated ); + } + elsif ( $op eq '=' || $op eq '!=' ) { + ($lhs, $table) = _hoistValue( $node->{params}[0], "${indent}l" ); + $rhs = _hoistConstant( $node->{params}[1] ); + if ( !$lhs || !$rhs ) { + # = and != are symmetric, so try the other order + ($lhs, $table) = _hoistValue( $node->{params}[1], "${indent}r" ); + $rhs = _hoistConstant( $node->{params}[0] ); + } + if ( $lhs && $rhs ) { + print STDERR "${indent}R=L\n" if MONITOR; + $test = "$lhs$op'$rhs'"; + $test = "NOT($test)" if $negated; + } + } + elsif ( $op eq '~' ) { + ($lhs, $table) = _hoistValue( $node->{params}[0], "${indent}l" ); + $rhs = _hoistConstant( $node->{params}[1] ); + if ( $lhs && $rhs ) { + my $escape = ''; + $rhs = quotemeta($rhs); + if ($rhs =~ /'/) { + $rhs =~ s/([s'])/s$1/g; + $escape = " ESCAPE 's'"; + } + $rhs =~ s/\\\?/./g; + $rhs =~ s/\\\*/.*/g; + print STDERR "${indent}L~R\n" if MONITOR; + $test = "$lhs REGEXP '^(?s:$rhs)\$'$escape"; + $test = "NOT($test)" if $negated; + } + } + elsif ( $op eq '=~' ) { + ($lhs, $table) = _hoistValue( $node->{params}[0], "${indent}l" ); + $rhs = _hoistConstant( $node->{params}[1] ); + if ( $lhs && $rhs ) { + my $escape = ''; + if ($rhs =~ /'/) { + $rhs =~ s/([s'])/s$1/g; + $escape = " ESCAPE 's'"; + } + print STDERR "${indent}L=~R\n" if MONITOR; + $test = "$lhs REGEXP '$rhs'$escape"; + $test = "NOT($test)" if $negated; + } + } + if ($table && $test) { + if ($table ne 'topic') { + # Have to use an EXISTS if the sub-test refers to another table + return < +# or +# . +# may be aliased +# Returns a partial SQL statement that can be followed by a condition for +# testing the value. +# A limited set of functions - UPPER, LOWER, +sub _hoistValue { + my ($node, $indent) = @_; + my $op = ref( $node->{op}) ? $node->{op}->{name} : ''; + + print STDERR "${indent}V ", $node->stringify(), "\n" if MONITOR; + + if ( $op eq '(' ) { + return _hoistValue( $node->{params}[0] ); + } + + if ( $op eq 'lc' ) { + my ($lhs, $table) = _hoistValue( $node->{params}[0], "${indent}$op" ); + return ("LOWER($lhs)", $table) if $lhs; + } elsif ( $op eq 'uc' ) { + my ($lhs, $table) = _hoistValue( $node->{params}[0], "${indent}$op" ); + return ("UPPER($lhs)", $table) if $lhs; + } elsif ( $op eq 'length' ) { + # This is slightly risky, because 'length' also works on array + # values, but SQL LEN only works on text values. + my ($lhs, $table) = _hoistValue( $node->{params}[0], "${indent}$op" ); + return ("LENGTH($lhs)", $table) if $lhs; + } elsif ( $op eq '.' ) { + my $lhs = $node->{params}[0]; + my $rhs = $node->{params}[1]; + if ( !ref( $lhs->{op} ) + && !ref( $rhs->{op} ) + && $lhs->{op} eq Foswiki::Infix::Node::NAME + && $rhs->{op} eq Foswiki::Infix::Node::NAME ) + { + $lhs = $lhs->{params}[0]; + $rhs = $rhs->{params}[0]; + if ( $Foswiki::Query::Node::aliases{$lhs} ) { + $lhs = $Foswiki::Query::Node::aliases{$lhs}; + } + if ( $lhs =~ /^META:(\w+)/ ) { + + return ("$1.$rhs", $1); + } + + if ( $rhs eq 'text' ) { + # Special case for the text body + return ('topic.text', 'topic'); + } + + if ( $rhs eq 'raw' ) { + # Special case for the text body + return ('topic.raw', 'topic'); + } + + # Otherwise assume the term before the dot is the form name + return ("EXISTS(SELECT * FROM FORM WHERE FORM.tid=topic.tid AND FORM.name='$lhs') AND FIELD.name='$rhs' AND FIELD.value", + "FIELD") + } + } + elsif ( !ref( $node->{op} ) + && $node->{op} eq Foswiki::Infix::Node::NAME ) { + # A simple name + if ( $node->{params}[0] =~ /^(name|web|text|raw)$/ ) { + + # Special case for the topic name, web or text body + return ("topic.$1", 'topic'); + } + else { + return ("FIELD.name='$node->{params}[0]' AND FIELD.value", + 'FIELD'); + } + } + + print STDERR "\tFAILED\n" if MONITOR; + return (undef, undef); +} + +# Expecting a constant +sub _hoistConstant { + my $node = shift; + + if ( + !ref( $node->{op} ) + && ( $node->{op} eq Foswiki::Infix::Node::STRING + || $node->{op} eq Foswiki::Infix::Node::NUMBER ) + ) + { + return $node->{params}[0]; + } + return; +} + +1; +__DATA__ + +Module of Foswiki - The Free and Open Source Wiki, http://foswiki.org/, http://Foswiki.org/ + +Copyright (C) 2010 Foswiki Contributors. All Rights Reserved. +Foswiki Contributors are listed in the AUTHORS file in the root +of this distribution. NOTE: Please extend that file, not this notice. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. For +more details read LICENSE in the root of this distribution. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +As per the GPL, removal of this notice is prohibited. + +Author: Crawford Currie http://c-dot.co.uk diff --git a/lib/Foswiki/Contrib/DBIStoreContrib/Listener.pm b/lib/Foswiki/Contrib/DBIStoreContrib/Listener.pm new file mode 100644 index 0000000..31ae4ff --- /dev/null +++ b/lib/Foswiki/Contrib/DBIStoreContrib/Listener.pm @@ -0,0 +1,269 @@ +# See bottom of file for license and copyright information +package Foswiki::Contrib::DBIStoreContrib::Listener; + +=begin TML + +---+ package Foswiki::Contrib::DBIStoreContrib::Listener; +Implements Foswiki::Store::Listener. + +Object that listens to low level store events, and maintains an SQL +database (on the other side of DBI). + +=cut + +use strict; +use warnings; + +use DBI; +use Foswiki::Meta; +use Error ':try'; +use Assert; + +use constant MONITOR => 0; + +our $db; +our @TABLES = keys(%Foswiki::Meta::VALIDATE); # META: types +our $ATTRS; + +# @ISA not required (base class is empty) +#our @ISA = ('Foswiki::Store::Listener'); + +# Construct object, just used as a handle +sub new { + my $class = shift; + + unless ($ATTRS) { + $ATTRS = {}; + foreach my $type (@TABLES) { + my @keys; + foreach my $g qw(require allow other) { + if (defined $Foswiki::Meta::VALIDATE{$type}->{$g}) { + push(@keys, @{$Foswiki::Meta::VALIDATE{$type}->{$g}}); + } + } + foreach my $key (@keys) { + $ATTRS->{$type}->{$key} = 1; + } + } + } + + unless ($db) { + $db = bless({}, $class); + } + return $db; +} + +# Connect on demand +sub _connect { + my ($this, $session) = @_; + + return 1 if $this->{handle}; + + print STDERR "CONNECT $Foswiki::cfg{Extensions}{DBIStoreContrib}{DSN}..." + if MONITOR; + $this->{handle} = DBI->connect( + $Foswiki::cfg{Extensions}{DBIStoreContrib}{DSN}, + $Foswiki::cfg{Extensions}{DBIStoreContrib}{Username}, + $Foswiki::cfg{Extensions}{DBIStoreContrib}{Password}, + { RaiseError => 1 }); + + # Check if the DB is initialised with a quick sniff of the metatypes + eval { + $this->{handle}->selectrow_array('SELECT * from metatypes'); + }; + if( $@ ) { + $this->_createTables(); + $this->_preload($session); + } + + print STDERR "connected $this->{handle}\n" if MONITOR; + return 1; +} + +# Does the table exist in the DB? +sub _tableExists { + my ($this, $type) = @_; + return 1 if $Foswiki::Meta::VALIDATE{$type}->{_default} + || grep { /^$type$/ } @TABLES; + my $check = $this->{handle}->selectcol_arrayref(<{$t}}); + $this->{handle}->do(<{handle}->do(<{_default}; +} + +# Create all the base tables in the DB (including all default META: tables) +sub _createTables { + my $this = shift; + + # Create the topic table. This links the web name, topic name, + # topic text and raw text of the topic. + $this->{handle}->do(<{handle}->do(<_createTableForMETA($t); + } +} + +# Load all existing webs and topics into the cache DB (expensive) +sub _preload { + my ($this, $session) = @_; + my $root = Foswiki::Meta->new($session); + my $wit = $root->eachWeb(); + while ($wit->hasNext()) { + my $w = $wit->next(); + print STDERR "PRELOAD $w\n" if MONITOR; + my $web = Foswiki::Meta->new($session , $w ); + my $tit = $web->eachTopic(); + while ($tit->hasNext()) { + my $t = $tit->next(); + my $topic = Foswiki::Meta->load( $session, $w, $t ); + $this->insert($topic); + } + } +} + +sub _makeTID { + my $tob = shift; + return $tob->web().'/'.$tob->topic(); +} + +# Implements Foswiki::Store::Listener +sub insert { + my ($this, $mo) = @_; + + if (defined $mo->topic()) { + my $tid = _makeTID($mo); + #print STDERR "\tInsert $tid\n" if MONITOR; + $this->_connect($mo->session()); + $this->{handle}->do( + 'INSERT INTO topic (tid,web,name,text,raw) VALUES (?,?,?,?,?);', + {}, $tid, $mo->web(), $mo->topic(), + $mo->text(), $mo->getEmbeddedStoreForm()); + + foreach my $type (keys %$mo) { + + # Make sure it's registered. + next unless (defined $Foswiki::Meta::VALIDATE{$type}); + + # If it's not default, we may have to create the table + $this->_createTableForMETA($type) + unless $this->_tableExists($type); + + # Insert this row + my $data = $mo->{$type}; + foreach my $item (@$data) { + # Filter attrs by those legal in the schema + my @kn = grep { $ATTRS->{$type}->{$_} } keys(%$item); + my @kl = ('tid', @kn); + my $sql = "INSERT INTO $type (" . join(',', map { "'$_'" } @kl) + . ") VALUES (".join(',', map { '?' } @kl).");"; + $this->{handle}->do($sql, {}, $tid, + map { $item->{$_} } @kn); + } + } + } +} + +# Implements Foswiki::Store::Listener +sub update { + my ($this, $old, $new) = @_; + # SMELL: there's got to be a better way + $this->remove($old); + $this->insert($new || $old); +} + +# Implements Foswiki::Store::Listener +sub remove { + my ($this, $mo) = @_; + + my $tids; + $this->_connect($mo->session()); + if (defined $mo->topic()) { + push(@$tids, _makeTID($mo)); + } else { + $tids = $this->{handle}->selectcol_arrayref(<web()); +SELECT tid FROM topic WHERE web=?; +SQL + } + my $ph = join(',', map { '?' } @$tids); + #print STDERR "\tRemove ".join(',',@$tids)."\n" if MONITOR; + + foreach my $table ('topic', @TABLES) { + $this->{handle}->do(<_connect($session); + print STDERR "$sql\n" if MONITOR; + my $names = $db->{handle}->selectcol_arrayref($sql); + print STDERR "HITS: ".scalar(@$names)."\n" if MONITOR; + return $names; +} + +1; +__DATA__ + +Author: Crawford Currie http://c-dot.co.uk + +Module of Foswiki - The Free and Open Source Wiki, http://foswiki.org/, http://Foswiki.org/ + +Copyright (C) 2010 Foswiki Contributors. All Rights Reserved. +Foswiki Contributors are listed in the AUTHORS file in the root +of this distribution. NOTE: Please extend that file, not this notice. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. For +more details read LICENSE in the root of this distribution. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +As per the GPL, removal of this notice is prohibited. + diff --git a/lib/Foswiki/Contrib/DBIStoreContrib/MANIFEST b/lib/Foswiki/Contrib/DBIStoreContrib/MANIFEST new file mode 100644 index 0000000..017a800 --- /dev/null +++ b/lib/Foswiki/Contrib/DBIStoreContrib/MANIFEST @@ -0,0 +1,9 @@ +# Release manifest for DBIStoreContrib +!noci +data/System/DBIStoreContrib.txt 0644 Documentation +lib/Foswiki/Contrib/DBIStoreContrib.pm 0644 Perl module +lib/Foswiki/Contrib/DBIStoreContrib/Config.spec 0644 Configuration +lib/Foswiki/Contrib/DBIStoreContrib/HoistSQL.pm +lib/Foswiki/Contrib/DBIStoreContrib/Listener.pm +lib/Foswiki/Store/QueryAlgorithms/DBIStoreContrib.pm 0444 +lib/Foswiki/Store/SearchAlgorithms/DBIStoreContrib.pm 0444 diff --git a/lib/Foswiki/Contrib/DBIStoreContrib/build.pl b/lib/Foswiki/Contrib/DBIStoreContrib/build.pl new file mode 100755 index 0000000..59a2be1 --- /dev/null +++ b/lib/Foswiki/Contrib/DBIStoreContrib/build.pl @@ -0,0 +1,24 @@ +#!/usr/bin/perl -w +BEGIN { unshift @INC, split( /:/, $ENV{FOSWIKI_LIBS} ); } +use Foswiki::Contrib::Build; + +# Create the build object +$build = new Foswiki::Contrib::Build('DBIStoreContrib'); + +# (Optional) Set the details of the repository for uploads. +# This can be any web on any accessible Foswiki installation. +# These defaults will be used when expanding tokens in .txt +# files, but be warned, they can be overridden at upload time! + +# name of web to upload to +$build->{UPLOADTARGETWEB} = 'Extensions'; +# Full URL of pub directory +$build->{UPLOADTARGETPUB} = 'http://foswiki.org/pub'; +# Full URL of bin directory +$build->{UPLOADTARGETSCRIPT} = 'http://foswiki.org/bin'; +# Script extension +$build->{UPLOADTARGETSUFFIX} = ''; + +# Build the target on the command line, or the default target +$build->build($build->{target}); + diff --git a/lib/Foswiki/Store/QueryAlgorithms/DBIStoreContrib.pm b/lib/Foswiki/Store/QueryAlgorithms/DBIStoreContrib.pm new file mode 100644 index 0000000..e2479ff --- /dev/null +++ b/lib/Foswiki/Store/QueryAlgorithms/DBIStoreContrib.pm @@ -0,0 +1,316 @@ +# See bottom of file for license and copyright information + +=begin TML + +---+ package Foswiki::Store::QueryAlgorithms::DBIStoreContrib + +=cut + +package Foswiki::Store::QueryAlgorithms::DBIStoreContrib; + +use strict; +use warnings; + +#@ISA = ( 'Foswiki::Query::QueryAlgorithms' ); # interface + +use Foswiki::Search::Node (); +use Foswiki::Meta (); +use Foswiki::Search::InfoCache (); +use Foswiki::Search::ResultSet (); +use Foswiki::MetaCache (); +use Foswiki::Query::Node (); +use Foswiki::Contrib::DBIStoreContrib::Listener (); + +# Debug prints +use constant MONITOR => 1; + +# See Foswiki::Query::QueryAlgorithms.pm for details +sub query { + my ( $query, $interestingTopics, $session, $options ) = @_; + + print STDERR "Initial query: ".$query->stringify()."\n" if MONITOR; + # Fold constants + my $context = Foswiki::Meta->new( $session, $session->{webName} ); + $query->simplify( tom => $context, data => $context ); + print STDERR "Simplified to: ". $query->stringify() . "\n" if MONITOR; + + my $isAdmin = $session->{users}->isAdmin( $session->{user} ); + + # First make a list of interesting webs + my $webNames = $options->{web} || ''; + my $recurse = $options->{'recurse'} || ''; + my $searchAllFlag = ( $webNames =~ /(^|[\,\s])(all|on)([\,\s]|$)/i ); + my @webs = Foswiki::Search::InfoCache::_getListOfWebs( $webNames, $recurse, + $searchAllFlag ); + + my @interestingWebs; + foreach my $web (@webs) { + + # can't process what ain't thar + next unless $session->webExists($web); + + my $webObject = Foswiki::Meta->new( $session, $web ); + my $thisWebNoSearchAll = + Foswiki::isTrue( $webObject->getPreference('NOSEARCHALL') ); + + # make sure we can report this web on an 'all' search + # DON'T filter out unless it's part of an 'all' search. + unless ( $searchAllFlag + && !$isAdmin + && ( $thisWebNoSearchAll || $web =~ /^[\.\_]/ ) + && $web ne $session->{webName} ) + { + push( @interestingWebs, $web ); + } + } + + # Got to be something worth searching for + return [] unless scalar(@interestingWebs); + + # Try and hoist regular expressions out of the query that we + # can use to refine the topic set + + require Foswiki::Contrib::DBIStoreContrib::HoistSQL; + my $hoistedSQL = Foswiki::Contrib::DBIStoreContrib::HoistSQL::hoist($query) || 1; + + if ($hoistedSQL) { + print STDERR "Hoisted '$hoistedSQL', remaining query: " . $query->stringify . "\n" + if MONITOR; + + # Did hoisting eliminate the dynamic query? + if ($query->evaluatesToConstant()) { + print STDERR "\t...eliminated static query\n" if MONITOR; + } + } + + my $sql = + 'SELECT tid FROM topic WHERE ' + . ( $hoistedSQL ? "$hoistedSQL AND " : '' ) + . "topic.web IN (" + . join( ',', map { "'$_'" } @interestingWebs ) . ')'; + + if ( $interestingTopics && scalar(@$interestingTopics) ) { + $sql .= " AND topic.name IN (" + . join( ',', map { "'$_'" } @$interestingTopics ) . ')'; + } # otherwise there is no topic name filter + + $sql .= ' ORDER BY web,name'; + + my $topicSet = Foswiki::Contrib::DBIStoreContrib::Listener::query( + $session, $sql ); + my $filter = Foswiki::Search::InfoCache::getOptionFilter($options); + + # Collate results into one-per-web result sets to mimic the old + # per-web search behaviour. + my %results; + foreach my $webtopic (@$topicSet) { + my ( $Iweb, $topic ) = + $Foswiki::Plugins::SESSION->normalizeWebTopicName( + undef, $webtopic ); + + my $cache = + $Foswiki::Plugins::SESSION->search->metacache->get( $Iweb, $topic ); + my $meta = $cache->{tom}; + + # Note that we filter the non-web topic name + next unless &$filter($topic); + + $results{$Iweb} ||= + new Foswiki::Search::InfoCache($Foswiki::Plugins::SESSION); + + if ($query) { + print STDERR "Processing " . $meta->getPath() . "\n" if MONITOR; + + # this 'lazy load' will become useful when @$topics becomes + # an infoCache + $meta = $meta->load() unless ( $meta->latestIsLoaded() ); + my $match = $query->evaluate( tom => $meta, data => $meta ); + if ($match) { + $results{$Iweb}->addTopic($meta); + } + else { + print STDERR "NO MATCH for " . $query->stringify . "\n" + if MONITOR; + } + } + else { + $results{$Iweb}->addTopic($meta); + } + } + + my $resultset = + new Foswiki::Search::ResultSet( [values %results], $options->{groupby}, + $options->{order}, Foswiki::isTrue( $options->{reverse} ) ); + + #TODO: $options should become redundant + $resultset->sortResults($options); + return $resultset; +} + +# The getField function is here to allow for Store specific optimisations +# such as direct database lookups. +sub getField { + my ( $this, $node, $data, $field ) = @_; + + my $result; + if ( UNIVERSAL::isa( $data, 'Foswiki::Meta' ) ) { + + # The object being indexed is a Foswiki::Meta object, so + # we have to use a different approach to treating it + # as an associative array. The first thing to do is to + # apply our "alias" shortcuts. + my $realField = $field; + if ( $Foswiki::Query::Node::aliases{$field} ) { + $realField = $Foswiki::Query::Node::aliases{$field}; + } + + if ( $realField eq 'META:TOPICINFO' ) { + + # Ensure the revision info is populated from the store + $data->getRevisionInfo(); + } + if ( $realField =~ s/^META:// ) { + if ( $Foswiki::Query::Node::isArrayType{$realField} ) { + + # Array type, have to use find + my @e = $data->find($realField); + $result = \@e; + } + else { + $result = $data->get($realField); + } + } + elsif ( $realField eq 'name' ) { + + # Special accessor to compensate for lack of a topic + # name anywhere in the saved fields of meta + return $data->topic(); + } + elsif ( $realField eq 'text' ) { + + # Special accessor to compensate for lack of the topic text + # name anywhere in the saved fields of meta + return $data->text(); + } + elsif ( $realField eq 'raw' ) { + + return $data->getEmbeddedStoreForm(); + } + elsif ( $realField eq 'web' ) { + + # Special accessor to compensate for lack of a web + # name anywhere in the saved fields of meta + return $data->web(); + } + elsif ( $data->topic() ) { + + # The field name isn't an alias, check to see if it's + # the form name + my $form = $data->get('FORM'); + if ( $form && $field eq $form->{name} ) { + + # SHORTCUT;it's the form name, so give me the fields + # as if the 'field' keyword had been used. + # TODO: This is where multiple form support needs to reside. + # Return the array of FIELD for further indexing. + my @e = $data->find('FIELD'); + return \@e; + } + else { + + # SHORTCUT; not a predefined name; assume it's a field + # 'name' instead. + # SMELL: Needs to error out if there are multiple forms - + # or perhaps have a heuristic that gives access to the + # uniquely named field. + $result = $data->get( 'FIELD', $field ); + $result = $result->{value} if $result; + } + } + } + elsif ( ref($data) eq 'ARRAY' ) { + + # Array objects are returned during evaluation, e.g. when + # a subset of an array is matched for further processing. + + # Indexing an array object. The index will be one of: + # 1. An integer, which is an implicit index='x' query + # 2. A name, which is an implicit name='x' query + if ( $field =~ /^\d+$/ ) { + + # Integer index + $result = $data->[$field]; + } + else { + + # String index + my @res; + + # Get all array entries that match the field + foreach my $f (@$data) { + my $val = getField( undef, $node, $f, $field ); + push( @res, $val ) if defined($val); + } + if ( scalar(@res) ) { + $result = \@res; + } + else { + + # The field name wasn't explicitly seen in any of the records. + # Try again, this time matching 'name' and returning 'value' + foreach my $f (@$data) { + next unless ref($f) eq 'HASH'; + if ( $f->{name} + && $f->{name} eq $field + && defined $f->{value} ) + { + push( @res, $f->{value} ); + } + } + if ( scalar(@res) ) { + $result = \@res; + } + } + } + } + elsif ( ref($data) eq 'HASH' ) { + + # A hash object may be returned when a sub-object of a Foswiki::Meta + # object has been matched. + $result = $data->{ $node->{params}[0] }; + } + else { + $result = $node->{params}[0]; + } + return $result; +} + +# Get a referenced topic +# See Foswiki::Store::QueryAlgorithms.pm for details +sub getRefTopic { + my ( $this, $relativeTo, $w, $t ) = @_; + return Foswiki::Meta->load( $relativeTo->session, $w, $t ); +} + +1; +__END__ + +Foswiki - The Free and Open Source Wiki, http://foswiki.org/ + +Copyright (C) 2010 Foswiki Contributors. Foswiki Contributors +are listed in the AUTHORS file in the root of this distribution. +NOTE: Please extend that file, not this notice. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. For +more details read LICENSE in the root of this distribution. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +As per the GPL, removal of this notice is prohibited. + +Author: Crawford Currie http://c-dot.co.uk diff --git a/lib/Foswiki/Store/SearchAlgorithms/DBIStoreContrib.pm b/lib/Foswiki/Store/SearchAlgorithms/DBIStoreContrib.pm new file mode 100644 index 0000000..1469b30 --- /dev/null +++ b/lib/Foswiki/Store/SearchAlgorithms/DBIStoreContrib.pm @@ -0,0 +1,81 @@ +# See bottom of file for license and copyright information +package Foswiki::Store::SearchAlgorithms::DBIStoreContrib; + +use strict; +use Assert; +use Foswiki::Search::InfoCache (); +use Foswiki::Query::Parser (); + +# Analyse the requirements of the search, and redirect to the query +# algorithm. This is kinda like the reverse of hoisting regexes :-) +sub query { + my ( $query, $inputTopicSet, $session, $options ) = @_; + + if (( @{ $query->{tokens} } ) == 0) { + return new Foswiki::Search::InfoCache($session, ''); + } + + # Convert the search to a query + # AND search - search once for each token, ANDing result together + my @ands; + foreach my $token ( @{ $query->{tokens} } ) { + + my $tokenCopy = $token; + + # flag for AND NOT search + my $invert = ( $tokenCopy =~ s/^\!//o ) ? 'NOT ' : ''; + + # scope can be 'topic' (default), 'text' or "all" + # scope='topic', e.g. Perl search on topic name: + my %topicMatches; + my @ors; + if ( $options->{'scope'} =~ /^(topic|all)$/ ) { + my $expr = $tokenCopy; + + $expr = quotemeta($expr) unless ( $options->{'type'} eq 'regex' ); + $expr = "(?i:$expr)" unless $options->{'casesensitive'}; + push(@ors, "${invert}name =~ '$expr'"); + } + + # scope='text', e.g. grep search on topic text: + if ( $options->{'scope'} =~ /^(text|all)$/ ) { + my $expr = $tokenCopy; + + $expr = quotemeta($expr) unless ( $options->{'type'} eq 'regex' ); + $expr = "(?i:$expr)" unless $options->{'casesensitive'}; + + push(@ors, "${invert}raw =~ '$expr'"); + } + push(@ands, '(' . join(' OR ', @ors) . ')'); + + } + my $queryParser = Foswiki::Query::Parser->new(); + $query = $queryParser->parse(join(' AND ', @ands)); + + eval "require $Foswiki::cfg{Store}{QueryAlgorithm}"; + die $@ if $@; + my $fn = $Foswiki::cfg{Store}{QueryAlgorithm}.'::query'; + no strict 'refs'; + return &$fn($query, $inputTopicSet, $session, $options); + use strict 'refs'; +} + +1; +__END__ +Author: Crawford Currie http://c-dot.co.uk + +Copyright (C) 2010 Foswiki Contributors. All Rights Reserved. +Foswiki Contributors are listed in the AUTHORS file in the root +of this distribution. NOTE: Please extend that file, not this notice. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. For +more details read LICENSE in the root of this distribution. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +As per the GPL, removal of this notice is prohibited.