Permalink
Browse files

Problem: Leader competition is always enabled

Solution: Make leader election an opt-in feature. A peer can decide
whether it likes to compete in leader election or be just passive
but still pass around the election messages because otherwise no
consent would be reached.
  • Loading branch information...
sappo committed May 13, 2018
1 parent 525ab4e commit d15d114b2e5123c4b43b72a728b1d655b2a5b0b0
Showing with 194 additions and 30 deletions.
  1. +8 −0 api/python_cffi.slurp
  2. +9 −0 api/zyre.api
  3. +9 −0 include/zyre.h
  4. +17 −0 src/zyre.c
  5. +9 −0 src/zyre_classes.h
  6. +64 −4 src/zyre_election.c
  7. +24 −0 src/zyre_group.c
  8. +10 −0 src/zyre_group.h
  9. +44 −26 src/zyre_node.c
View
@@ -94,6 +94,14 @@ void
int
zyre_set_endpoint (zyre_t *self, const char *format, ...);
// This options enables a peer to actively contest for leadership in the
// given group. If this option is not set the peer will still participate in
// elections but never gets elected. This ensures that a consent for a leader
// is reached within a group even though not every peer is contesting for
// leadership.
void
zyre_set_contest_in_group (zyre_t *self, const char *group);
// Set an alternative endpoint value when using GOSSIP ONLY. This is useful
// if you're advertising an endpoint behind a NAT.
void
View
@@ -102,6 +102,15 @@
<return type = "integer" />
</method>
<method name = "set contest in group" state = "draft">
This options enables a peer to actively contest for leadership in the
given group. If this option is not set the peer will still participate in
elections but never gets elected. This ensures that a consent for a leader
is reached within a group even though not every peer is contesting for
leadership.
<argument name = "group" type = "string" />
</method>
<method name = "set advertised endpoint" state = "draft">
Set an alternative endpoint value when using GOSSIP ONLY. This is useful
if you're advertising an endpoint behind a NAT.
View
@@ -216,6 +216,15 @@ ZYRE_EXPORT void
zyre_test (bool verbose);
#ifdef ZYRE_BUILD_DRAFT_API
// *** Draft method, for development use, may change without warning ***
// This options enables a peer to actively contest for leadership in the
// given group. If this option is not set the peer will still participate in
// elections but never gets elected. This ensures that a consent for a leader
// is reached within a group even though not every peer is contesting for
// leadership.
ZYRE_EXPORT void
zyre_set_contest_in_group (zyre_t *self, const char *group);
// *** Draft method, for development use, may change without warning ***
// Set an alternative endpoint value when using GOSSIP ONLY. This is useful
// if you're advertising an endpoint behind a NAT.
View
@@ -277,6 +277,23 @@ zyre_set_endpoint (zyre_t *self, const char *format, ...)
return zsock_wait (self->actor) == 0? 0: -1;
}
#ifdef ZYRE_BUILD_DRAFT_API
// DRAFT-API: Election
// --------------------------------------------------------------------------
// This options enables a peer to actively contest for leadership in the
// given group. If this option is not set the peer will still participate in
// elections but never gets elected. This ensures that a consent for a leader
// is reached within a group even though not every peer is contesting for
// leadership.
void
zyre_set_contest_in_group (zyre_t *self, const char *group) {
assert (self);
assert (group);
zstr_sendx (self->actor, "SET CONTEST" , group, NULL);
}
#endif
#ifdef ZYRE_BUILD_DRAFT_API
void
zyre_set_advertised_endpoint (zyre_t *self, const char *endpoint)
View
@@ -60,6 +60,15 @@ typedef struct _zyre_node_t zyre_node_t;
// *** To avoid double-definitions, only define if building without draft ***
#ifndef ZYRE_BUILD_DRAFT_API
// *** Draft method, defined for internal use only ***
// This options enables a peer to actively contest for leadership in the
// given group. If this option is not set the peer will still participate in
// elections but never gets elected. This ensures that a consent for a leader
// is reached within a group even though not every peer is contesting for
// leadership.
ZYRE_PRIVATE void
zyre_set_contest_in_group (zyre_t *self, const char *group);
// *** Draft method, defined for internal use only ***
// Set an alternative endpoint value when using GOSSIP ONLY. This is useful
// if you're advertising an endpoint behind a NAT.
View
@@ -318,14 +318,74 @@ zyre_election_test (bool verbose)
zclock_sleep (500);
// Join topology
zyre_join (node1, "GLOBAL");
zyre_join (node2, "GLOBAL");
zyre_join (node1, "GROUP_1");
zyre_set_contest_in_group (node1, "GROUP_1");
zyre_join (node2, "GROUP_1");
zyre_set_contest_in_group (node2, "GROUP_1");
zyre_join (node1, "GLOBAL1");
zyre_join (node2, "GLOBAL1");
zyre_join (node1, "GROUP_2");
zyre_join (node2, "GROUP_2");
zyre_set_contest_in_group (node2, "GROUP_2");
zyre_join (node1, "GROUP_3");
zyre_join (node2, "GROUP_3");
// Give peers time to perform elections
zclock_sleep (1500);
// Check election results
int num_of_global_leaders = 0;
int num_of_global1_leaders = 0;
int num_of_leader_messages = 0;
zyre_event_t *event;
do {
// Recv from node1
event = zyre_event_new (node1);
if (streq (zyre_event_type (event), "LEADER")) {
if (streq (zyre_event_group (event), "GROUP_1")) {
num_of_leader_messages++;
if (streq (zyre_uuid (node1), zyre_event_peer_uuid (event)))
num_of_global_leaders++;
}
else
if (streq (zyre_event_group (event), "GROUP_2")) {
num_of_leader_messages++;
if (streq (zyre_uuid (node1), zyre_event_peer_uuid (event)))
num_of_global1_leaders++;
}
else
if (streq (zyre_event_group (event), "GROUP_3"))
assert (false);
}
zyre_event_destroy (&event);
// Recv from node2
event = zyre_event_new (node2);
if (streq (zyre_event_type (event), "LEADER")) {
if (streq (zyre_event_group (event), "GROUP_1")) {
num_of_leader_messages++;
if (streq (zyre_uuid (node2), zyre_event_peer_uuid (event)))
num_of_global_leaders++;
}
else
if (streq (zyre_event_group (event), "GROUP_2")) {
num_of_leader_messages++;
if (streq (zyre_uuid (node2), zyre_event_peer_uuid (event)))
num_of_global1_leaders++;
}
else
if (streq (zyre_event_group (event), "GROUP_3"))
assert (false);
}
zyre_event_destroy (&event);
} while (num_of_leader_messages < 4);
assert (num_of_global_leaders == 1);
assert (num_of_global1_leaders == 1);
// @TODO: Test leaving leader
zyre_stop (node1);
zyre_stop (node2);
View
@@ -23,6 +23,7 @@ struct _zyre_group_t {
zhash_t *peers; // Peers in group
#ifdef ZYRE_BUILD_DRAFT_API
// DRAFT-API: Election
bool contest; // Wheather the peer actively contest for leadership of this group
zyre_peer_t *leader; // Peer that has been elected as leader for this group
zyre_election_t *election; // Election handler, is NULL if there's no active election
#endif
@@ -48,6 +49,10 @@ zyre_group_new (const char *name, zhash_t *container)
zyre_group_t *self = (zyre_group_t *) zmalloc (sizeof (zyre_group_t));
self->name = strdup (name);
self->peers = zhash_new ();
#ifdef ZYRE_BUILD_DRAFT_API
// DRAFT-API: Election
self->contest = false;
#endif
// Insert into container if requested
if (container) {
@@ -157,6 +162,24 @@ zyre_group_require_election (zyre_group_t *self)
return self->election;
}
// --------------------------------------------------------------------------
// Enables peer to actively contest for leadership in this group.
void
zyre_group_set_contest (zyre_group_t *self) {
assert (self);
self->contest = true;
}
// --------------------------------------------------------------------------
// Returns true if this peer actively contests for leadership, otherwise
// false.
bool
zyre_group_contest (zyre_group_t *self) {
assert (self);
return self->contest;
}
// --------------------------------------------------------------------------
// Return the election handler for this group.
@@ -232,6 +255,7 @@ zyre_group_test (bool verbose)
assert (rc == 0);
if (verbose)
zre_msg_print (msg);
zre_msg_destroy (&msg);
zuuid_destroy (&me);
View
@@ -45,10 +45,19 @@ ZYRE_PRIVATE void
ZYRE_PRIVATE zlist_t *
zyre_group_peers (zyre_group_t *self);
#ifdef ZYRE_BUILD_DRAFT_API
// DRAFT-API: Election
// Find or create an election for a group
zyre_election_t *
zyre_group_require_election (zyre_group_t *self);
// Enables peer to actively contest for leadership in this group.
void
zyre_group_set_contest (zyre_group_t *self);
bool
zyre_group_contest (zyre_group_t *self);
// Return the election handler for this group.
zyre_election_t *
zyre_group_election (zyre_group_t *self);
@@ -64,6 +73,7 @@ zyre_peer_t *
// Sets the peer that has been elected leader of this group.
void
zyre_group_set_leader (zyre_group_t *self, zyre_peer_t *leader);
#endif
// Self test of this class
ZYRE_PRIVATE void
View
@@ -89,6 +89,9 @@ typedef struct {
// --------------------------------------------------------------------------
// Local helper
static zyre_group_t *
zyre_node_require_peer_group (zyre_node_t *self, const char *name);
static int
s_string_compare (void *item1, void *item2)
{
@@ -494,6 +497,16 @@ zyre_node_recv_api (zyre_node_t *self)
zstr_free (&value);
}
else
#ifdef ZYRE_BUILD_DRAFT_API
// DRAFT-API: Election
if (streq (command, "SET CONTEST")) {
char *groupname = zmsg_popstr (request);
zyre_group_t *group = zyre_node_require_peer_group (self, groupname);
zyre_group_set_contest (group);
zstr_free (&groupname);
}
else
#endif
#ifdef ZYRE_BUILD_DRAFT_API
if (streq (command, "SET ADVERTISED ENDPOINT")) {
self->advertised_endpoint = zmsg_popstr (request);
@@ -894,6 +907,7 @@ zyre_node_require_peer_group (zyre_node_t *self, const char *name)
zyre_group_t *group = (zyre_group_t *) zhash_lookup (self->peer_groups, name);
if (!group)
group = zyre_group_new (name, self->peer_groups);
return group;
}
@@ -1094,24 +1108,26 @@ zyre_node_recv_peer (zyre_node_t *self)
#ifdef ZYRE_BUILD_DRAFT_API
// DRAFT-API: Election
if (zlist_exists (self->own_groups, (char *) zre_msg_group (msg))) {
// Start election if there's an active election abort it
zyre_election_t *election = zyre_group_election (group);
if (election) {
// Discard a running election because the number of peers change
zyre_election_destroy (&election);
}
election = zyre_election_new ();
zyre_group_set_election (group, election);
if (zyre_group_contest (zyre_node_require_peer_group (self, zre_msg_group (msg)))) {
// Start election if there's an active election abort it
zyre_election_t *election = zyre_group_election (group);
if (election) {
// Discard a running election because the number of peers change
zyre_election_destroy (&election);
}
election = zyre_election_new ();
zyre_group_set_election (group, election);
// Start challenge for leadership
zyre_election_set_caw (election, strdup (zuuid_str (self->uuid)));
zre_msg_t *election_msg = zyre_election_build_elect_msg (election);
zre_msg_set_group (election_msg, zre_msg_group (msg));
if (self->verbose)
zsys_info ("(%s) [%s] send ELECT message - %s",
self->name, zre_msg_group (msg), zuuid_str (self->uuid));
// Start challenge for leadership
zyre_election_set_caw (election, strdup (zuuid_str (self->uuid)));
zre_msg_t *election_msg = zyre_election_build_elect_msg (election);
zre_msg_set_group (election_msg, zre_msg_group (msg));
if (self->verbose)
zsys_info ("(%s) [%s] send ELECT message - %s",
self->name, zre_msg_group (msg), zuuid_str (self->uuid));
zyre_group_send (group, &election_msg);
zyre_group_send (group, &election_msg);
}
}
#endif
}
@@ -1131,14 +1147,16 @@ zyre_node_recv_peer (zyre_node_t *self)
// Discard a running election because the number of peers change
zyre_election_destroy (&election);
}
election = zyre_election_new ();
zyre_group_set_election (group, election);
// Start challenge for leadership
zyre_election_set_caw (election, strdup (zuuid_str (self->uuid)));
zre_msg_t *election_msg = zyre_election_build_elect_msg (election);
zre_msg_set_group (election_msg, zre_msg_group (msg));
zyre_group_send (group, &election_msg);
if (zyre_group_contest (zyre_node_require_peer_group (self, zre_msg_group (msg)))) {
election = zyre_election_new ();
zyre_group_set_election (group, election);
// Start challenge for leadership
zyre_election_set_caw (election, strdup (zuuid_str (self->uuid)));
zre_msg_t *election_msg = zyre_election_build_elect_msg (election);
zre_msg_set_group (election_msg, zre_msg_group (msg));
zyre_group_send (group, &election_msg);
}
}
}
}
@@ -1430,9 +1448,9 @@ zyre_node_ping_peer (const char *key, void *item, void *argument)
zyre_peer_send (peer, &msg);
zre_msg_destroy (&msg);
// Inform the calling application this peer is being evasive
zstr_sendm (self->outbox, "EVASIVE");
zstr_sendm (self->outbox, "EVASIVE");
zstr_sendm (self->outbox, zyre_peer_identity (peer));
zstr_send (self->outbox, zyre_peer_name (peer));
zstr_send (self->outbox, zyre_peer_name (peer));
}
return 0;
}

0 comments on commit d15d114

Please sign in to comment.