Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

SERVER-211 - simple ttl collections where we can expire based on age

  • Loading branch information...
commit 25bdc679a0e559d64ec7f22b0468cf5b1671c4e7 1 parent 844026a
Eliot authored May 10, 2012
22  jstests/slowNightly/ttl1.js
... ...
@@ -0,0 +1,22 @@
  1
+
  2
+t = db.ttl1;
  3
+t.drop();
  4
+
  5
+now = (new Date()).getTime();
  6
+
  7
+for ( i=0; i<24; i++ )
  8
+    t.insert( { x : new Date( now - ( 3600 * 1000 * i ) ) } );
  9
+db.getLastError();
  10
+
  11
+assert.eq( 24 , t.count() );
  12
+
  13
+t.ensureIndex( { x : 1 } , { expireAfterSeconds : 20000 } );
  14
+
  15
+assert.soon( 
  16
+    function() {
  17
+        return t.count() < 24;
  18
+    }, "never deleted" , 120 * 1000
  19
+);
  20
+
  21
+assert.eq( 0 , t.find( { x : { $lt : new Date( now - 20000000 ) } } ).count() );
  22
+assert.eq( 6 , t.count() );
1  src/mongo/SConscript
@@ -196,6 +196,7 @@ serverOnlyFiles = [ "db/curop.cpp",
196 196
                     "db/d_globals.cpp",
197 197
                     "db/pagefault.cpp",
198 198
                     "util/compress.cpp",
  199
+                    "db/ttl.cpp",
199 200
                     "db/d_concurrency.cpp",
200 201
                     "db/lockstat.cpp",
201 202
                     "db/lockstate.cpp",
4  src/mongo/db/db.cpp
@@ -49,6 +49,7 @@
49 49
 #include "d_concurrency.h"
50 50
 #include "../s/d_writeback.h"
51 51
 #include "d_globals.h"
  52
+#include "ttl.h"
52 53
 
53 54
 #if defined(_WIN32)
54 55
 # include "../util/ntservice.h"
@@ -530,7 +531,8 @@ namespace mongo {
530 531
         snapshotThread.go();
531 532
         d.clientCursorMonitor.go();
532 533
         PeriodicTask::theRunner->go();
533  
-        
  534
+        startTTLBackgroundJob();
  535
+
534 536
 #ifndef _WIN32
535 537
         CmdLine::launchOk();
536 538
 #endif
110  src/mongo/db/ttl.cpp
... ...
@@ -0,0 +1,110 @@
  1
+// ttl.cpp
  2
+
  3
+/**
  4
+*    Copyright (C) 2008 10gen Inc.
  5
+*
  6
+*    This program is free software: you can redistribute it and/or  modify
  7
+*    it under the terms of the GNU Affero General Public License, version 3,
  8
+*    as published by the Free Software Foundation.
  9
+*
  10
+*    This program is distributed in the hope that it will be useful,
  11
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13
+*    GNU Affero General Public License for more details.
  14
+*
  15
+*    You should have received a copy of the GNU Affero General Public License
  16
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
+*/
  18
+
  19
+#include "pch.h"
  20
+
  21
+#include "mongo/util/background.h"
  22
+#include "mongo/db/instance.h"
  23
+#include "mongo/db/databaseholder.h"
  24
+#include "mongo/db/ops/delete.h"
  25
+
  26
+namespace mongo {
  27
+
  28
+    class TTLMonitor : public BackgroundJob {
  29
+    public:
  30
+        TTLMonitor(){}
  31
+        virtual ~TTLMonitor(){}
  32
+
  33
+        virtual string name() const { return "TTLMonitor"; }
  34
+        
  35
+        static string secondsExpireField;
  36
+        
  37
+        void doTTLForDB( const string& dbName ) {
  38
+            vector<BSONObj> indexes;
  39
+            {
  40
+                auto_ptr<DBClientCursor> cursor = db.query( dbName + ".system.indexes" , 
  41
+                                                            BSON( secondsExpireField << BSON( "$exists" << true ) ) );
  42
+                if ( cursor.get() ) {
  43
+                    while ( cursor->more() ) {
  44
+                        indexes.push_back( cursor->next().getOwned() );
  45
+                    }
  46
+                }
  47
+            }
  48
+            
  49
+            for ( unsigned i=0; i<indexes.size(); i++ ) {
  50
+                BSONObj idx = indexes[i];
  51
+                
  52
+
  53
+                BSONObj key = idx["key"].Obj();
  54
+                uassert( 16187 , "key for ttl index can only have 1 field" , key.nFields() == 1 );
  55
+
  56
+                BSONObj query;
  57
+                {
  58
+                    BSONObjBuilder b;
  59
+                    b.appendDate( "$lt" , curTimeMillis64() - ( 1000 * idx[secondsExpireField].numberLong() ) );
  60
+                    query = BSON( key.firstElement().fieldName() << b.obj() );
  61
+                }
  62
+                
  63
+                LOG(1) << "TTL: " << key << " \t " << query << endl;
  64
+                
  65
+                long long n = 0;
  66
+                {
  67
+                    string ns = idx["ns"].String();
  68
+                    Client::WriteContext ctx( ns );
  69
+                    nsdetails( ns.c_str() )->setUserFlag( NamespaceDetails::Flag_UsePowerOf2Sizes );
  70
+                    n = deleteObjects( ns.c_str() , query , false , true );
  71
+                }
  72
+
  73
+                LOG(1) << "\tTTL deleted: " << n << endl;
  74
+            }
  75
+            
  76
+            
  77
+        }
  78
+
  79
+        virtual void run() {
  80
+            Client::initThread( name().c_str() );
  81
+
  82
+            while ( ! inShutdown() ) {
  83
+                sleepsecs( 60 );
  84
+
  85
+                LOG(3) << "TTLMonitor thread awake" << endl;
  86
+                
  87
+                set<string> dbs;
  88
+                {
  89
+                    Lock::DBRead lk( "local" );
  90
+                    dbHolder().getAllShortNames( dbs );
  91
+                }
  92
+                
  93
+                for ( set<string>::const_iterator i=dbs.begin(); i!=dbs.end(); ++i ) {
  94
+                    string db = *i;
  95
+                    doTTLForDB( db );
  96
+                }
  97
+
  98
+            }
  99
+        }
  100
+
  101
+        DBDirectClient db;
  102
+    };
  103
+
  104
+    void startTTLBackgroundJob() {
  105
+        TTLMonitor* ttl = new TTLMonitor();
  106
+        ttl->go();
  107
+    }    
  108
+    
  109
+    string TTLMonitor::secondsExpireField = "expireAfterSeconds";
  110
+}
23  src/mongo/db/ttl.h
... ...
@@ -0,0 +1,23 @@
  1
+// ttl.h
  2
+
  3
+/**
  4
+*    Copyright (C) 2008 10gen Inc.
  5
+*
  6
+*    This program is free software: you can redistribute it and/or  modify
  7
+*    it under the terms of the GNU Affero General Public License, version 3,
  8
+*    as published by the Free Software Foundation.
  9
+*
  10
+*    This program is distributed in the hope that it will be useful,
  11
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13
+*    GNU Affero General Public License for more details.
  14
+*
  15
+*    You should have received a copy of the GNU Affero General Public License
  16
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
+*/
  18
+
  19
+#pragma once
  20
+
  21
+namespace mongo {
  22
+    void startTTLBackgroundJob();
  23
+}

7 notes on commit 25bdc67

Sid Upadhyay

@erh What do you think of the redis implementation of TTL? http://redis.io/commands/expire

Eliot
Owner
erh commented on 25bdc67 May 10, 2012
Sid Upadhyay
Eliot
Owner
erh commented on 25bdc67 May 11, 2012
Sid Upadhyay
Eliot
Owner
erh commented on 25bdc67 May 11, 2012
Sid Upadhyay
Please sign in to comment.
Something went wrong with that request. Please try again.