Improve mem3

src/mem3/src/mem3_shards.erl

@@ -526,11 +526,17 @@ cache_clear(St) ->
     St#st{cur_size = 0}.
 
 maybe_spawn_shard_writer(DbName, Shards, IdleTimeout) ->
-    case ets:member(?OPENERS, DbName) of
+    try ets:member(?OPENERS, DbName) of
         true ->
             ignore;
         false ->
             spawn_shard_writer(DbName, Shards, IdleTimeout)
+    catch
+        error:badarg ->
+            % We might have been called before mem3 finished initializing
+            % from the error:badarg clause in for_db/2, for instance, so
+            % we shouldn't expect ?OPENERS to exist yet
+            ignore
     end.
 
 spawn_shard_writer(DbName, Shards, IdleTimeout) ->

src/mem3/src/mem3_sup.erl

@@ -18,19 +18,39 @@ start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init(_Args) ->
+    % Some startup order constraints based on call dependencies:
+    %
+    % * mem3_events gen_event should be started before all the others
+    %
+    % * mem3_nodes gen_server is needed so everyone can call mem3:nodes()
+    %
+    % * mem3_sync_nodes needs to run before mem3_sync and
+    %   mem3_sync_event_listener, so they can both can call
+    %   mem3_sync_nodes:add/1
+    %
+    % * mem3_distribution force connects nodes from mem3:nodes(), so start it
+    %   before mem3_sync since mem3_sync:initial_sync/0 expects the connected
+    %   nodes to be there when calling mem3_sync_nodes:add(nodes())
+    %
+    % * mem3_sync_event_listener has to start after mem3_sync, so it can call
+    %   mem3_sync:push/2
+    %
+    % * mem3_seeds and mem3_reshard_sup can wait till the end, as they will
+    %   spawn background work that can go on for a while: seeding system dbs
+    %   from other nodes running resharding jobs
+    %
     Children = [
         child(mem3_events),
         child(mem3_nodes),
-        child(mem3_distribution),
-        child(mem3_seeds),
-        % Order important?
+        child(mem3_shards),
         child(mem3_sync_nodes),
+        child(mem3_distribution),
         child(mem3_sync),
-        child(mem3_shards),
         child(mem3_sync_event_listener),
+        child(mem3_seeds),
         child(mem3_reshard_sup)
     ],
-    {ok, {{one_for_one, 10, 1}, couch_epi:register_service(mem3_epi, Children)}}.
+    {ok, {{rest_for_one, 10, 1}, couch_epi:register_service(mem3_epi, Children)}}.
 
 child(mem3_events) ->
     MFA = {gen_event, start_link, [{local, mem3_events}]},

src/rexi/src/rexi_server_mon.erl

@@ -57,7 +57,6 @@ aggregate_queue_len(ChildMod) ->
 % Mem3 cluster callbacks
 
 cluster_unstable(Server) ->
-    couch_log:notice("~s : cluster unstable", [?MODULE]),
     gen_server:cast(Server, cluster_unstable),
     Server.
 
@@ -75,7 +74,7 @@ init(ChildMod) ->
         ?CLUSTER_STABILITY_PERIOD_SEC
     ),
     start_servers(ChildMod),
-    couch_log:notice("~s : started servers", [ChildMod]),
+    couch_log:info("~s : started servers", [ChildMod]),
     {ok, ChildMod}.
 
 handle_call(status, _From, ChildMod) ->
@@ -93,13 +92,13 @@ handle_call(Msg, _From, St) ->
 % can be started, but do not immediately stop nodes, defer that till cluster
 % stabilized.
 handle_cast(cluster_unstable, ChildMod) ->
-    couch_log:notice("~s : cluster unstable", [ChildMod]),
+    couch_log:info("~s : cluster unstable", [ChildMod]),
     start_servers(ChildMod),
     {noreply, ChildMod};
 % When cluster is stable, start any servers for new nodes and stop servers for
 % the ones that disconnected.
 handle_cast(cluster_stable, ChildMod) ->
-    couch_log:notice("~s : cluster stable", [ChildMod]),
+    couch_log:info("~s : cluster stable", [ChildMod]),
     start_servers(ChildMod),
     stop_servers(ChildMod),
     {noreply, ChildMod};