More fixes and improved graphs

new/Makefile.am

@@ -20,6 +20,7 @@ test_SOURCES    = berkeleydb.h \
 				  generator_runtime.cc \
 				  getopts.h \
 				  getopts.c \
+				  graph.h \
 				  hamsterdb.h \
 				  hamsterdb.cc \
 				  main.cc

new/berkeleydb.cc

@@ -76,7 +76,7 @@ compare_db64(DB *db, const DBT *dbt1, const DBT *dbt2, size_t *)
 }
 
 void
-BerkeleyDatabase::get_metrics(Metrics *metrics)
+BerkeleyDatabase::get_metrics(Metrics *metrics, bool live)
 {
 }
 

new/berkeleydb.h

@@ -41,7 +41,7 @@ class BerkeleyDatabase : public Database
     }
 
     // Fills |metrics| with additional metrics
-    virtual void get_metrics(Metrics *metrics);
+    virtual void get_metrics(Metrics *metrics, bool live = false);
 
   protected:
     // the actual implementation(s)

new/database.h

@@ -89,7 +89,7 @@ class Database
     ham_status_t cursor_close(Cursor *cursor);
 
     // Fills |metrics| with additional metrics
-    virtual void get_metrics(Metrics *metrics) = 0;
+    virtual void get_metrics(Metrics *metrics, bool live = false) = 0;
 
   protected:
     // the actual implementation(s)

new/generator.h

@@ -12,8 +12,8 @@
 #ifndef GENERATOR_H__
 #define GENERATOR_H__
 
-#include "metrics.h"
 #include "database.h"
+#include "graph.h"
 
 class Configuration;
 
@@ -25,8 +25,8 @@ class Generator
   public:
     enum {
       kCommandInsert = 0,
-      kCommandErase = 1,
-      kCommandFind = 2,
+      kCommandFind = 1,
+      kCommandErase = 2,
       kCommandCommitTransaction = 3,
       kCommandCreate,
       kCommandOpen,
@@ -40,50 +40,23 @@ class Generator
 
     // constructor
     Generator(int id, Configuration *conf, Database *db)
-      : m_id(id), m_config(conf), m_db(db), m_last_status(0),
-        m_opspersec_graph(0) {
+      : m_id(id), m_config(conf), m_db(db), m_last_status(0), m_graph(0) {
       memset(&m_record, 0, sizeof(m_record));
-
       memset(&m_opspersec, 0, sizeof(m_opspersec));
-      memset(&m_latency_graphs, 0, sizeof(m_latency_graphs));
 
       // only create graph output for the first hamsterdb thread!
       if (conf->metrics >= Configuration::kMetricsPng
-           && !strcmp(db->get_name(), "hamsterdb")) {
-        if (id == 0) {
-          for (int i = 0; i < 4; i++) {
-            char filename[128];
-            sprintf(filename, "%s-lat%d.dat", m_db->get_name(), i);
-            m_latency_graphs[i] = fopen(filename, "w");
-            if (!m_latency_graphs[i]) {
-              printf("error writing to file: %s\n", strerror(errno));
-              exit(-1);
-            }
-            setvbuf(m_latency_graphs[i], NULL, _IOFBF, 10 * 1024 * 1024);
-          }
-          char filename[128];
-          sprintf(filename, "%s-ops.dat", m_db->get_name());
-          m_opspersec_graph = fopen(filename, "w");
-          if (!m_opspersec_graph) {
-            printf("error writing to file: %s\n", strerror(errno));
-            exit(-1);
-          }
-          setvbuf(m_opspersec_graph, NULL, _IOFBF, 2 * 1024 * 1024);
-        }
+           && !strcmp(db->get_name(), "hamsterdb")
+           && id == 0) {
+        m_graph = new Graph("hamsterdb");
       }
     }
 
     // destructor
     virtual ~Generator() {
-      for (int i = 0; i < 4; i++) {
-        if (m_latency_graphs[i]) {
-          fclose(m_latency_graphs[i]);
-          m_latency_graphs[i] = 0;
-        }
-      }
-      if (m_opspersec_graph) {
-        fclose(m_opspersec_graph);
-        m_opspersec_graph = 0;
+      if (m_graph) {
+        delete m_graph;
+        m_graph = 0;
       }
     }
 
@@ -119,17 +92,6 @@ class Generator
     }
 
   protected:
-    // adds information to the latency graph
-    void add_latency_graph(int command, double time, double latency) {
-      fprintf(m_latency_graphs[command], "%f %f\n", time, latency);
-    }
-
-    // add information to the "operations per second" graph
-    void add_opspersec_graph(uint64_t elapsed_seconds) {
-      fprintf(m_opspersec_graph, "%lu %lu %lu %lu %lu\n", elapsed_seconds,
-              m_opspersec[0], m_opspersec[1], m_opspersec[2], m_opspersec[3]);
-    }
-
     // unique ID - used to create the database
     int m_id;
 
@@ -147,15 +109,11 @@ class Generator
     // different databases
     ham_record_t m_record;
 
-    // file handles for the latency output; index is the command id
-    // (kCommandInsert, -Erase, -Find, -CommitTransaction)
-    FILE *m_latency_graphs[4];
-
-    // file handles for the operation-per-second
-    FILE *m_opspersec_graph;
+    // the performance graphs
+    Graph *m_graph;
 
     // accumulating operations-per-seconds for the graphs
-    uint64_t m_opspersec[4];
+    uint32_t m_opspersec[4];
 };
 
 #endif /* GENERATOR_H__ */

new/generator_parser.h

@@ -19,6 +19,7 @@
 #include <boost/progress.hpp>
 
 #include "timer.h"
+#include "metrics.h"
 #include "generator.h"
 #include "database.h"
 

new/generator_runtime.cc

@@ -147,6 +147,11 @@ RuntimeGenerator::execute()
   if (m_state == kStateStopped)
     return (false);
 
+  double insert_latency = 0.0;
+  double erase_latency = 0.0;
+  double find_latency = 0.0;
+  double commit_latency = 0.0;
+
   int cmd = get_next_command();
   switch (cmd) {
     case Generator::kCommandCreate:
@@ -159,13 +164,13 @@ RuntimeGenerator::execute()
       close();
       break;
     case Generator::kCommandInsert:
-      insert();
+      insert_latency = insert();
       break;
     case Generator::kCommandErase:
-      erase();
+      erase_latency = erase();
       break;
     case Generator::kCommandFind:
-      find();
+      find_latency = find();
       break;
     case Generator::kCommandBeginTransaction:
       txn_begin();
@@ -174,7 +179,7 @@ RuntimeGenerator::execute()
       txn_abort();
       break;
     case Generator::kCommandCommitTransaction:
-      txn_commit();
+      commit_latency = txn_commit();
       break;
     default:
       assert(!"shouldn't be here");
@@ -185,6 +190,35 @@ RuntimeGenerator::execute()
   if (m_progress && m_config->limit_ops)
     (*m_progress) += 1;
 
+  // write page fetch/flush graphs?
+  if (m_graph) {
+    Metrics m;
+    m_db->get_metrics(&m);
+
+    double elapsed = m_start.seconds();
+
+    uint32_t flushes = 0;
+    uint32_t fetches = 0;
+    if (m.hamster_metrics.page_count_flushed
+            > m_metrics.hamster_metrics.page_count_flushed) {
+      flushes = m.hamster_metrics.page_count_flushed
+              - m_metrics.hamster_metrics.page_count_flushed;
+      m_metrics.hamster_metrics.page_count_flushed
+              = m.hamster_metrics.page_count_flushed;
+    }
+    if (m.hamster_metrics.page_count_fetched
+            > m_metrics.hamster_metrics.page_count_fetched) {
+      fetches = m.hamster_metrics.page_count_fetched
+              - m_metrics.hamster_metrics.page_count_fetched;
+      m_metrics.hamster_metrics.page_count_fetched
+              = m.hamster_metrics.page_count_fetched;
+    }
+
+    m_graph->add_latency_metrics(elapsed, insert_latency, find_latency,
+                erase_latency, commit_latency, fetches, flushes);
+  }
+
+
   return (true);
 }
 
@@ -239,7 +273,7 @@ RuntimeGenerator::close()
   m_metrics.elapsed_wallclock_seconds = m_start.seconds();
 }
 
-void
+double
 RuntimeGenerator::insert()
 {
   ham_key_t key = generate_key();
@@ -257,7 +291,6 @@ RuntimeGenerator::insert()
   double elapsed = t.seconds();
 
   m_opspersec[kCommandInsert]++;
-  add_latency_graph(kCommandInsert, m_start.seconds(), elapsed);
 
   if (m_metrics.insert_latency_min > elapsed)
     m_metrics.insert_latency_min = elapsed;
@@ -275,9 +308,11 @@ RuntimeGenerator::insert()
   }
 
   m_metrics.insert_ops++;
+
+  return (elapsed);
 }
 
-void
+double
 RuntimeGenerator::erase()
 {
   ham_key_t key = generate_key();
@@ -294,7 +329,6 @@ RuntimeGenerator::erase()
   double elapsed = t.seconds();
 
   m_opspersec[kCommandErase]++;
-  add_latency_graph(kCommandErase, m_start.seconds(), elapsed);
 
   if (m_metrics.erase_latency_min > elapsed)
     m_metrics.erase_latency_min = elapsed;
@@ -306,9 +340,11 @@ RuntimeGenerator::erase()
     m_success = false;
 
   m_metrics.erase_ops++;
+
+  return (elapsed);
 }
 
-void
+double
 RuntimeGenerator::find()
 {
   ham_key_t key = generate_key();
@@ -327,7 +363,6 @@ RuntimeGenerator::find()
   double elapsed = t.seconds();
 
   m_opspersec[kCommandFind]++;
-  add_latency_graph(kCommandFind, m_start.seconds(), elapsed);
 
   if (m_metrics.find_latency_min > elapsed)
     m_metrics.find_latency_min = elapsed;
@@ -340,6 +375,8 @@ RuntimeGenerator::find()
 
   m_metrics.find_bytes += m_record.size;
   m_metrics.find_ops++;
+
+  return (elapsed);
 }
 
 void
@@ -381,7 +418,7 @@ RuntimeGenerator::txn_abort()
   m_metrics.other_ops++;
 }
 
-void
+double
 RuntimeGenerator::txn_commit()
 {
   tee("TXN_COMMIT");
@@ -400,7 +437,6 @@ RuntimeGenerator::txn_commit()
   double elapsed = t.seconds();
 
   m_opspersec[kCommandCommitTransaction]++;
-  add_latency_graph(kCommandCommitTransaction, m_start.seconds(), elapsed);
 
   if (m_metrics.txn_commit_latency_min > elapsed)
     m_metrics.txn_commit_latency_min = elapsed;
@@ -412,6 +448,7 @@ RuntimeGenerator::txn_commit()
     m_success = false;
 
   m_metrics.txn_commit_ops++;
+  return (elapsed);
 }
 
 ham_key_t
@@ -496,15 +533,16 @@ RuntimeGenerator::limit_reached()
       return (true);
   }
 
-  // reached time limit?
-  if (m_config->limit_seconds
-      || m_config->metrics >= Configuration::kMetricsPng) {
+  // reached time limit and/or update latency graphs?
+  if (m_config->limit_seconds || m_graph) {
     double new_elapsed = m_start.seconds();
     if (new_elapsed - m_elapsed_seconds >= 1.) {
       if (m_progress)
         (*m_progress) += (unsigned)(new_elapsed - m_elapsed_seconds);
       m_elapsed_seconds = new_elapsed;
-      add_opspersec_graph(m_elapsed_seconds);
+      if (m_graph)
+        m_graph->add_opspersec_graph(m_elapsed_seconds, m_opspersec[0],
+                        m_opspersec[1], m_opspersec[2], m_opspersec[3]);
       memset(&m_opspersec, 0, sizeof(m_opspersec));
     }
     if (m_config->limit_seconds && new_elapsed > m_config->limit_seconds) {

new/generator_runtime.h

@@ -18,6 +18,7 @@
 #include <boost/random/uniform_01.hpp>
 #include <boost/progress.hpp>
 
+#include "metrics.h"
 #include "timer.h"
 #include "generator.h"
 #include "datasource.h"
@@ -73,19 +74,19 @@ class RuntimeGenerator : public Generator
     void create();
 
     // inserts a key/value pair
-    void insert();
+    double insert();
 
     // erases a key/value pair
-    void erase();
+    double erase();
 
     // lookup of a key/value pair
-    void find();
+    double find();
 
     // begins a new transaction
     void txn_begin();
 
     // commits a transaction
-    void txn_commit();
+    double txn_commit();
 
     // aborts a transaction
     void txn_abort();

new/gnuplot-lat

@@ -3,7 +3,4 @@ set terminal png
 set xlabel "time"
 set ylabel "latency (thread #1)"
 set style data linespoint
-plot "hamsterdb-lat0.dat" using 1:2 title "insert", \
-"hamsterdb-lat1.dat" using 1:2 title "erase", \
-"hamsterdb-lat2.dat" using 1:2 title "find", \
-"hamsterdb-lat3.dat" using 1:2 title "txn-commit"
+plot "hamsterdb-lat.dat" using 1:2 title "insert"

new/gnuplot-ops

@@ -3,7 +3,4 @@ set terminal png
 set xlabel "time"
 set ylabel "operations (all threads)"
 set style data linespoint
-plot "hamsterdb-ops.dat" using 1:2 title "insert", \
-"" using 1:3 title "erase", \
-"" using 1:4 title "find", \
-"" using 1:5 title "txn-commit"
+plot "hamsterdb-ops.dat" using 1:2 title "insert"