Item10596: extract webs into seperate mongodb databases named foswiki…

…__Web__Subweb git-svn-id: http://svn.foswiki.org/trunk/MongoDBPlugin@11301 0b4bb1d4-4e5a-0410-9cc4-b2b747904278
foswiki · Apr 5, 2011 · 0134bc3 · 0134bc3
1 parent d8285c0
commit 0134bc3
Show file tree

Hide file tree

Showing 8 changed files with 96 additions and 52 deletions.
diff --git a/lib/Foswiki/Plugins/MongoDBPlugin.pm b/lib/Foswiki/Plugins/MongoDBPlugin.pm
@@ -141,7 +141,7 @@ sub getMongoDB {
                 port => $Foswiki::cfg{MongoDBPlugin}{port} || '27017',
                 username => $Foswiki::cfg{MongoDBPlugin}{username},
                 password => $Foswiki::cfg{MongoDBPlugin}{password},
-                database => $Foswiki::cfg{MongoDBPlugin}{database} || 'foswiki',
+#                database => $Foswiki::cfg{MongoDBPlugin}{database} || 'foswiki',
             }
         );
     }
@@ -153,10 +153,6 @@ sub _update {
     my $session = shift;
     my $query   = Foswiki::Func::getCgiQuery();
 
-    #lets make sure we have the javascript we'll rely on later
-    _updateDatabase($session, $query);
-
-
     my $webParam     = $query->param('updateweb') || 'Sandbox';
     my $recurse   = Foswiki::Func::isTrue($query->param('recurse'), ($webParam eq 'all'));
 
@@ -175,6 +171,9 @@ sub _update {
 
     my $result = "\n importing: \n";
     foreach my $web (@webNames) {
+    #lets make sure we have the javascript we'll rely on later
+    _updateDatabase($session, $web, $query);
+
         my @topicList = Foswiki::Func::getTopicList($web);
         print STDERR "start web: $web ($#topicList)\n";
 
@@ -228,7 +227,7 @@ sub _removeTopic {
 
     #    $query->{'_attachment'} = $topic if (defined($attachment));
 
-    my $ret = getMongoDB()->remove( 'current', $query );
+    my $ret = getMongoDB()->remove($web, 'current', $query );
 
 }
 
@@ -269,8 +268,8 @@ sub _updateTopic {
 #even then, we have a hard limit of 40 indexes, so we're going to have to get more creative.
 #mind you, we don't really need indexes for speed, just to cope with query() resultsets that contain more than 1Meg of documents - so maybe we can delay creation until that happens?
                     getMongoDB()->ensureIndex(
-                        'current',
-                        { $key . '.' . $elem->{name} => 1 },
+                        getMongoDB()->_getCollection($web, 'current'),
+                        { $key . '.' . $elem->{name}.'.value' => 1 },
                         { name => $key . '.' . $elem->{name} }
                     );
                 }
@@ -319,38 +318,53 @@ sub _updateTopic {
 
     $meta->{_raw_text} = $raw_text || $savedMeta->getEmbeddedStoreForm();
 
-    my $ret = getMongoDB()->update( 'current', "$web.$topic", $meta );
+    my $ret = getMongoDB()->update($web, 'current', "$web.$topic", $meta );
 }
 
 #restHandler used to update the javascript saved in MongoDB
 sub _updateDatabase {
     my $session = shift;
     my $query   = Foswiki::Func::getCgiQuery();
+    #TODO: actually, should do all webs if not specified..
+    my $web = shift;
+    if (not defined($web) or ($web eq 'MongoDBPlugin')) {
+        my $count = 0;
+        #do all webs..
+        my @webNames = Foswiki::Func::getListOfWebs( '' , undef );
+        foreach $web (@webNames) {
+            $count += _updateDatabase($session, $web);
+        }
+        return $count;
+    }
 
-    #load from topic..
-    my $meta = Foswiki::Func::readTopic('Sandbox', 'TestParseTime');
+    print STDERR "loading js into $web\n";
 
     Foswiki::Func::loadTemplate('mongodb_js');
     my $foswiki_d2n_js = Foswiki::Func::expandTemplate('foswiki_d2n_js');
-    getMongoDB()->updateSystemJS('foswiki_d2n', $foswiki_d2n_js);
+    getMongoDB()->updateSystemJS($web, 'foswiki_d2n', $foswiki_d2n_js);
 
     my $foswiki_getRef_js = Foswiki::Func::expandTemplate('foswiki_getRef_js');
-    getMongoDB()->updateSystemJS('foswiki_getRef', $foswiki_getRef_js);
+    getMongoDB()->updateSystemJS($web, 'foswiki_getRef', $foswiki_getRef_js);
 
     my $foswiki_getField_js = Foswiki::Func::expandTemplate('foswiki_getField_js');
-    getMongoDB()->updateSystemJS('foswiki_getField', $foswiki_getField_js);
+    getMongoDB()->updateSystemJS($web, 'foswiki_getField', $foswiki_getField_js);
 
     my $foswiki_toLowerCase_js = Foswiki::Func::expandTemplate('foswiki_toLowerCase_js');
-    getMongoDB()->updateSystemJS('foswiki_toLowerCase', $foswiki_toLowerCase_js);
+    getMongoDB()->updateSystemJS($web, 'foswiki_toLowerCase', $foswiki_toLowerCase_js);
 
     my $foswiki_toUpperCase_js = Foswiki::Func::expandTemplate('foswiki_toUpperCase_js');
-    getMongoDB()->updateSystemJS('foswiki_toUpperCase', $foswiki_toUpperCase_js);
+    getMongoDB()->updateSystemJS($web, 'foswiki_toUpperCase', $foswiki_toUpperCase_js);
 
     my $foswiki_length_js = Foswiki::Func::expandTemplate('foswiki_length_js');
-    getMongoDB()->updateSystemJS('foswiki_length', $foswiki_length_js);
+    getMongoDB()->updateSystemJS($web, 'foswiki_length', $foswiki_length_js);
 
     my $foswiki_normaliseTopic_js = Foswiki::Func::expandTemplate('foswiki_normaliseTopic_js');
-    getMongoDB()->updateSystemJS('foswiki_normaliseTopic', $foswiki_normaliseTopic_js);
+    getMongoDB()->updateSystemJS($web, 'foswiki_normaliseTopic', $foswiki_normaliseTopic_js);
+
+    my $foswiki_getDatabaseName_js = Foswiki::Func::expandTemplate('foswiki_getDatabaseName_js');
+    getMongoDB()->updateSystemJS($web, 'foswiki_getDatabaseName', $foswiki_getDatabaseName_js);
+
+    return 1;
 }
 
 

diff --git a/lib/Foswiki/Plugins/MongoDBPlugin/DB.pm b/lib/Foswiki/Plugins/MongoDBPlugin/DB.pm
@@ -49,28 +49,28 @@ sub new {
 
 sub query {
     my $self           = shift;
+    my $database       = shift;
     my $collectionName = shift;
     my $ixhQuery       = shift;
     my $queryAttrs     = shift || {};
 
     my $startTime = [Time::HiRes::gettimeofday];
 
-    my $collection = $self->_getCollection('current');
+    my $collection = $self->_getCollection($database, 'current');
     print STDERR "searching mongo : "
       . Dumper($ixhQuery) . " , "
       . Dumper($queryAttrs) . "\n"
       if MONITOR;
 
 #debugging for upstream
 print STDERR "----------------------------------------------------------------------------------\n" if DEBUG;
-my $connection = $self->_connect();
-my $database   = $connection->get_database( $self->{database} );
-#$database->run_command({"profile" => 2});
+my $db   = $self->_getDatabase( $database );
+#$db->run_command({"profile" => 2});
 
 #use Devel::Peek;
-#Dump($database->run_command({"count" => 'current', "query" => $ixhQuery}));
+#Dump($db->run_command({"count" => 'current', "query" => $ixhQuery}));
 #print STDERR "----------------------------------------------------------------------------------\n";
-    my $long_count = $database->run_command({"count" => 'current', "query" => $ixhQuery});
+    my $long_count = $db->run_command({"count" => 'current', "query" => $ixhQuery});
     my $cursor = $collection->query( $ixhQuery, $queryAttrs );
     #TODO: this is to make sure we're getting the cursor->count before anyone uses the cursor.
     my $count = $long_count;
@@ -95,8 +95,8 @@ use Data::Dumper;
       . ( $cursor->has_next() ? 'true' : 'false' ) . "\n" if DEBUG;
 
 #more debugging
-#print STDERR "get_collection(system.profile)".Dumper($database->get_collection("system.profile")->find->all)."\n";
-#p$database->run_command({"profile" => 0});
+#print STDERR "get_collection(system.profile)".Dumper($db->get_collection("system.profile")->find->all)."\n";
+#$db->run_command({"profile" => 0});
 print STDERR "----------------------------------------------------------------------------------\n" if DEBUG;
 
     #end timer
@@ -110,22 +110,23 @@ print STDERR "------------------------------------------------------------------
 
 sub update {
     my $self           = shift;
+    my $database       = shift;
     my $collectionName = shift;
     my $address        = shift;
     my $hash           = shift;
 
     #    use Data::Dumper;
     print STDERR "+++++ mongo update $address == ".Dumper($hash)."\n" if MONITOR;
 
-    my $collection = $self->_getCollection($collectionName);
+    my $collection = $self->_getCollection($database, $collectionName);
 
 #TODO: not the most efficient place to create and index, but I want to be sure, to be sure.
     $self->ensureIndex( $collection, { _topic => 1 }, { name => '_topic' } );
-    $self->ensureIndex(
-        $collection,
-        { _topic => 1,             _web   => 1 },
-        { name   => '_topic:_web', unique => 1 }
-    );
+#    $self->ensureIndex(
+#        $collection,
+#        { _topic => 1,             _web   => 1 },
+#        { name   => '_topic:_web', unique => 1 }
+#    );
     $self->ensureIndex(
         $collection,
         { 'TOPICINFO.author' => 1 },
@@ -176,16 +177,16 @@ sub update {
 #MongoDB's ensure_index causes the server to re0index, even if that index already exists, so we need to wrap it.
 sub ensureIndex {
     my $self       = shift;
-    my $collection = shift;    #either a collection object of a name
+    my $collection = shift;#must be a collection obj
     my $indexRef   = shift;    #can be a hashref or an ixHash
     my $options    = shift;
 
     ASSERT( defined( $options->{name} ) ) if DEBUG;
 
     if ( ref($collection) eq '' ) {
-
+die 'must convert $collection param to be a collection obj';
         #convert name of collection to collection obj
-        $collection = $self->_getCollection($collection);
+        #$collection = $self->_getCollection($database, $collection);
     }
 
     #cache the indexes we know about
@@ -218,20 +219,22 @@ sub ensureIndex {
 
 sub remove {
     my $self           = shift;
+    my $database       = shift;
     my $collectionName = shift;
     my $mongoDbQuery           = shift;
 
-    my $collection = $self->_getCollection($collectionName);
+    my $collection = $self->_getCollection($database, $collectionName);
 
     $collection->remove($mongoDbQuery);
 }
 
 sub updateSystemJS {
     my $self = shift;
+    my $database       = shift;
     my $functionname = shift;
     my $sourcecode = shift;
 
-    my $collection = $self->_getCollection('system.js');
+    my $collection = $self->_getCollection($database, 'system.js');
 
 use MongoDB::Code;
     my $code = MongoDB::Code->new('code' => $sourcecode);
@@ -246,15 +249,35 @@ use MongoDB::Code;
 
 
 #######################################################
-#Webname?
-sub _getCollection {
+sub getDatabaseName {
     my $self           = shift;
-    my $collectionName = shift;
+    my $web       = shift;
+
+    #using webname as database name, so we need to sanitise
+    #replace / with __ and pre-pend foswiki__ ?
+    $web =~ s/\//__/g;
+    return 'foswiki__'.$web;
+}
+sub _getDatabase {
+    my $self           = shift;
+    my $database       = shift;
+
+    #using webname as database name, so we need to sanitise
+    #replace / with __ and pre-pend foswiki__ ?
+    $database =~ s/\//__/g;
+    $database = 'foswiki__'.$database;
 
     my $connection = $self->_connect();
-    my $database   = $connection->get_database( $self->{database} );
+    return $connection->get_database( $database );
+}
+sub _getCollection {
+    my $self           = shift;
+    my $database       = shift;
+    my $collectionName = shift;
+
+    my $db   = $self->_getDatabase($database);
 
-    return $database->get_collection($collectionName);
+    return $db->get_collection($collectionName);
 }
 
 sub _connect {
@@ -276,9 +299,10 @@ sub _MONGODB {
     my $params = shift;
 
     my $web   = $params->{web};
+    my $database = $web;
     my $topic = $params->{topic};
 
-    my $collection = $self->_getCollection('current');
+    my $collection = $self->_getCollection($database, 'current');
     my $data = $collection->find_one( { _web => $web, _topic => $topic } );
 
     use Foswiki::Plugins::MongoDBPlugin::Meta;

diff --git a/lib/Foswiki/Plugins/MongoDBPlugin/HoistMongoDB.pm b/lib/Foswiki/Plugins/MongoDBPlugin/HoistMongoDB.pm
@@ -445,7 +445,7 @@ sub convertFunction {
         #TODO: like all accessses, this needs alot of undef protection.
         my $addr = '('.convertStringToJS( $$value[1] ).')';
 ##%TMPL:DEF{foswiki_getRef_js}%function(host, collection, currentqueryweb, topic) {
-        my $ref = 'foswiki_getRef(\'localhost\', \'foswiki.current\', this._web, ' . convertStringToJS( $$value[0] ) . ')';
+        my $ref = 'foswiki_getRef(\'localhost\', foswiki_getDatabaseName(this._web)+\'.current\', this._web, ' . convertStringToJS( $$value[0] ) . ')';
         $addr =~ s/this/$ref/;
         return $addr;
     }

diff --git a/lib/Foswiki/Plugins/MongoDBPlugin/Listener.pm b/lib/Foswiki/Plugins/MongoDBPlugin/Listener.pm
@@ -43,6 +43,9 @@ sub insert {
     my %args = @_;
 
     return if ( defined( $args{newattachment} ) );
+
+    #creating a new web... so we need to add the js we use to get foswiki style functionality
+    Foswiki::Plugins::MongoDBPlugin::_updateDatabase($args{newmeta}->{_session}, $args{newmeta}->web) if ($args{newmeta}->topic eq 'WebPreferences');
 
     Foswiki::Plugins::MongoDBPlugin::_updateTopic( $args{newmeta}->web,
         $args{newmeta}->topic, $args{newmeta} );
@@ -119,7 +122,7 @@ sub loadTopic {
 #    my $_[1]    = shift;
 #    my $_[2] = shift;
 
-    my $session = $_[1]->{_session};    #TODO: naughty, but we see to get called before Foswiki::Func::SESSION is set up :(
+    my $session = $_[1]->{_session};    #TODO: naughty, but we seem to get called before Foswiki::Func::SESSION is set up :(
 
 $_[0]->{count} = {} unless (defined($_[0]->{count}));
 $_[0]->{count}{$_[1]->web} = {} unless (defined($_[0]->{count}{$_[1]->web}));

diff --git a/lib/Foswiki/Plugins/MongoDBPlugin/Meta.pm b/lib/Foswiki/Plugins/MongoDBPlugin/Meta.pm
@@ -81,7 +81,7 @@ sub reload {
     $this->{FILEATTACHMENT} = [];
 
     my $collection =
-      Foswiki::Plugins::MongoDBPlugin::getMongoDB->_getCollection('current');
+      Foswiki::Plugins::MongoDBPlugin::getMongoDB->_getCollection($this->{_web}, 'current');
     my $data = $collection->find_one(
         { _web => $this->{_web}, _topic => $this->{_topic} } );
     $this->loadFromBSONData($data);

diff --git a/lib/Foswiki/Store/QueryAlgorithms/MongoDB.pm b/lib/Foswiki/Store/QueryAlgorithms/MongoDB.pm
@@ -315,10 +315,10 @@ sub doMongoSearch {
     #  . Dumper($queryAttrs) . "\n";
 
 #    my $collection =
-#      Foswiki::Plugins::MongoDBPlugin::getMongoDB()->_getCollection('current');
+#      Foswiki::Plugins::MongoDBPlugin::getMongoDB()->_getCollection($web, 'current');
 #    my $cursor = $collection->query($ixhQuery, $queryAttrs);
     my $cursor = Foswiki::Plugins::MongoDBPlugin::getMongoDB()
-      ->query( 'current', $ixhQuery, $queryAttrs );
+      ->query($web, 'current', $ixhQuery, $queryAttrs );
 
     return $cursor;
 }

diff --git a/lib/Foswiki/Store/SearchAlgorithms/MongoDB.pm b/lib/Foswiki/Store/SearchAlgorithms/MongoDB.pm
@@ -84,7 +84,7 @@ sub search {
     }
 
     my $cursor = Foswiki::Plugins::MongoDBPlugin::getMongoDB()
-      ->query( 'current', \%elements );
+      ->query($web, 'current', \%elements );
     return new Foswiki::Search::MongoDBInfoCache( $Foswiki::Plugins::SESSION,
         $web, $options, $cursor );
 }
@@ -362,7 +362,7 @@ sub _webQuery {
     }
 
     my $cursor = Foswiki::Plugins::MongoDBPlugin::getMongoDB()
-      ->query( 'current', $ixhQuery, $queryAttrs );
+      ->query($web, 'current', $ixhQuery, $queryAttrs );
 
     return new Foswiki::Search::MongoDBInfoCache( $Foswiki::Plugins::SESSION,
         $web, $options, $cursor );

diff --git a/templates/mongodb_js.tmpl b/templates/mongodb_js.tmpl
@@ -43,11 +43,14 @@
 
     return [web, topic];
 }%TMPL:END%
-
+%TMPL:DEF{foswiki_getDatabaseName_js}%function(web) {
+    web = web.replace(/\//, '__');
+    return 'foswiki__'+web;
+}%TMPL:END%
 
 %TMPL:DEF{foswiki_getRef_js}%function(host, collection, web, topic) {
     conn = new Mongo(host);
-    current = conn.getCollection(collection);
+    current = conn.getCollection(collection); //collection is actually database.collection
 
     if (topic ==undefined) {
         return topic;