Permalink
Browse files

CBD-771: Stats archiver uses ETS to replace Mnesia

Stats are stored in memory resident ETS tables and are dumped to
backup files periodically. Cascade and truncate happen in memory
now. Stats backup files are saved in data/<node>/stats directory
which is parallel to data/<node>/mnesia.

Using ETS to replace mnesia certainly saves CPU usage. On idle
2-node cluster with 3~5 buckets, I see the CPU% drops at least
10%.

Based on work of Liang Guo.

Change-Id: If57932e0caae6e28e9b57b6317ba87e0b870b422
Reviewed-on: http://review.couchbase.org/22545
Tested-by: Aliaksey Kandratsenka <alkondratenko@gmail.com>
Reviewed-by: Aliaksey Kandratsenka <alkondratenko@gmail.com>
  • Loading branch information...
aartamonau authored and alk committed Jan 25, 2013
1 parent 4bbe56b commit 9e0a980e538ee1414327c1b0f0cfc5fbb123a5a6
Showing with 250 additions and 730 deletions.
  1. +5 −0 CHANGES
  2. +1 −2 Makefile
  3. +0 −4 cluster_run
  4. +1 −4 doc/ns-server-hierarchy.txt
  5. +0 −580 src/mb_mnesia.erl
  6. +0 −44 src/mb_mnesia_sup.erl
  7. +30 −23 src/misc.erl
  8. +36 −2 src/ns_cluster.erl
  9. +0 −2 src/ns_server_cluster_sup.erl
  10. +170 −47 src/stats_archiver.erl
  11. +7 −22 src/stats_reader.erl
View
@@ -14,6 +14,11 @@ Between versions 2.0.1 and 2.*
grab bucket stats now is /pools/default/buckets/<bucket name>/stats
REST call.
+* (CBD-771) Stats archives are not stored in mnesia anymore.
+
+ Instead they are collected in ETS tables and saved to plain files
+ from time to time.
+
-----------------------------------------
Between versions 2.0.0 and 2.0.1
-----------------------------------------
View
@@ -139,7 +139,6 @@ do-install:
cp cbbrowse_logs $(DESTDIR)$(PREFIX)/bin/cbbrowse_logs
cp cbcollect_info $(DESTDIR)$(PREFIX)/bin/cbcollect_info
chmod +x $(DESTDIR)$(PREFIX)/bin/couchbase-server $(DESTDIR)$(PREFIX)/bin/cbbrowse_logs $(DESTDIR)$(PREFIX)/bin/cbcollect_info
- mkdir -p $(DESTDIR)$(PREFIX)/var/lib/couchbase/mnesia
mkdir -p $(DESTDIR)$(PREFIX)/var/lib/couchbase/logs
cp ebucketmigrator $(DESTDIR)$(PREFIX)/bin/ebucketmigrator
chmod +x $(DESTDIR)$(PREFIX)/bin/ebucketmigrator
@@ -186,7 +185,7 @@ $(COUCHBASE_PLT): | all
do_build_plt:
dialyzer --output_plt $(COUCHBASE_PLT) --build_plt \
- --apps compiler crypto erts inets kernel mnesia os_mon sasl ssl stdlib xmerl \
+ --apps compiler crypto erts inets kernel os_mon sasl ssl stdlib xmerl \
$(COUCH_PATH)/src/mochiweb \
$(COUCH_PATH)/src/snappy $(COUCH_PATH)/src/etap $(realpath $(COUCH_PATH)/src/ibrowse) \
$(realpath $(COUCH_PATH)/src/lhttpc) \
View
@@ -76,10 +76,6 @@ def start_cluster(num_nodes, start_index, host, extra_args, args_prefix):
os.makedirs(logdir)
except:
pass
- try:
- os.stat("data/n_{0}/mnesia".format(i))
- except:
- os.makedirs("data/n_{0}/mnesia".format(i))
args = args_prefix + ["erl", "+MMmcs" "30",
"+A", "16", "+sbtu",
"+P", "327680", "-pa"] + ebin_path + [
@@ -16,9 +16,6 @@
cookie changes as well")
(gen-server 'ns_cluster
"performs node join/leave requests")
- (supervisor 'mb_mnesia_sup :mode :one_for_one
- (gen-event 'mb_mnesia_events)
- (gen-server 'mb_mnesia))
(supervisor 'ns_config_sup :mode :rest_for_one
(gen-event 'ns_config_events
"fired when any ns_config variable is changed")
@@ -201,7 +198,7 @@
ns_stats_event"
(pubsub-link nil :to 'ns_stats_event))
(gen-server '(stats_reader "default")
- "used to read mnesia stats archive")
+ "used to read stats archive")
(event-handler '(failover_safeness_level "default") :to 'ns_stats_event
"estimates bucket's replication up-to-date-ness
'from this node' based on stats it sees"))))
Oops, something went wrong.

0 comments on commit 9e0a980

Please sign in to comment.