Merge pull request graphite-project#13 from slackhappy/master

add fallocate support (optional) to carbon

conf/carbon.conf.example

@@ -107,6 +107,15 @@ WHISPER_AUTOFLUSH = False
 # depending on the underlying storage configuration.
 # WHISPER_SPARSE_CREATE = False
 
+# By default new Whisper files are created pre-allocated with the data region
+# filled with zeros to prevent fragmentation and speed up contiguous reads and
+# writes (which are common). Enabling this option will cause Whisper to create
+# fallocate the file instead.  Only beneficial on a filesystem that
+# supports fallocate.  It maintains the benefits of contiguous reads/writes.
+# but with a much faster creation speed. Enabling this option may
+# allow a large increase of MAX_CREATES_PER_MINUTE.
+# WHISPER_FALLOCATE_CREATE = False
+#
 # Enabling this option will cause Whisper to lock each Whisper file it writes
 # to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when
 # multiple carbon-cache daemons are writing to the same files

lib/carbon/conf.py

@@ -45,6 +45,7 @@
   LOG_CACHE_HITS = True,
   WHISPER_AUTOFLUSH=False,
   WHISPER_SPARSE_CREATE=False,
+  WHISPER_FALLOCATE_CREATE=False,
   WHISPER_LOCK_WRITES=False,
   MAX_DATAPOINTS_PER_MESSAGE=500,
   MAX_AGGREGATION_INTERVALS=5,
@@ -209,6 +210,12 @@ def postOptions(self):
             log.msg("Enabling Whisper autoflush")
             whisper.AUTOFLUSH = True
 
+        if settings.WHISPER_FALLOCATE_CREATE:
+            if whisper.CAN_FALLOCATE:
+                log.msg("Enabling Whisper fallocate support")
+            else:
+                log.err("WHISPER_FALLOCATE_CREATE is enabled but linking failed.")
+
         if settings.WHISPER_LOCK_WRITES:
             if whisper.CAN_LOCK:
                 log.msg("Enabling Whisper file locking")

lib/carbon/writer.py

@@ -115,7 +115,7 @@ def writeCachedDataPoints():
 
         log.creates("creating database file %s (archive=%s xff=%s agg=%s)" % 
                     (dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
-        whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE)
+        whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE)
         os.chmod(dbFilePath, 0755)
         instrumentation.increment('creates')