Permalink
Browse files

PE: Remote: Allow remote nodes that start containers with pacemaker r…

…emote inside
  • Loading branch information...
beekhof committed Apr 12, 2017
1 parent 2124e04 commit 26c59fb7d128f83d0b4d35ae9b9d088359103d31
@@ -291,6 +291,6 @@ node_t *pe_create_node(const char *id, const char *uname, const char *type,
bool remote_id_conflict(const char *remote_name, pe_working_set_t *data);
void common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *node, long options, void *print_data);
resource_t *find_container_child(const char *stem, resource_t * rsc, node_t *node);
bool fix_remote_addr(resource_t * rsc);
#endif
@@ -142,6 +142,7 @@ struct node_shared_s {
gboolean shutdown;
gboolean expected_up;
gboolean is_dc;
gboolean unpacked;

This comment has been minimized.

Show comment
Hide comment
@kgaillot

kgaillot May 1, 2017

Collaborator

I'm guessing we don't have to worry about adding to the end of the struct b/c node_shared_s is internal only? How would someone know that? Neither node_shared_s nor node_s have doxygen blocks (none of the pengine API is documented, actually), but presumably node_s is meant to be public.

@kgaillot

kgaillot May 1, 2017

Collaborator

I'm guessing we don't have to worry about adding to the end of the struct b/c node_shared_s is internal only? How would someone know that? Neither node_shared_s nor node_s have doxygen blocks (none of the pengine API is documented, actually), but presumably node_s is meant to be public.

This comment has been minimized.

Show comment
Hide comment
@beekhof

beekhof May 1, 2017

Member

I can't think of any occasion where there would be a reason to create a node_shared struct that didn't exist.
In general a blanket "only ever use provided allocators, if there isn't one, its private" statement might be sufficient.

@beekhof

beekhof May 1, 2017

Member

I can't think of any occasion where there would be a reason to create a node_shared struct that didn't exist.
In general a blanket "only ever use provided allocators, if there isn't one, its private" statement might be sufficient.

int num_resources;
GListPtr running_rsc; /* resource_t* */
View
@@ -368,7 +368,8 @@ create_remote_resource(
if(tuple->ipaddr) {
create_nvp(xml_obj, "addr", tuple->ipaddr);
} else {
create_nvp(xml_obj, "addr", "localhost");
// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
create_nvp(xml_obj, "addr", "#uname");

This comment has been minimized.

Show comment
Hide comment
@kgaillot

kgaillot May 1, 2017

Collaborator

So "#uname" means "the host that the connection is running on"? And therefore any remote with "#uname" must use a unique port? And in the bundle context, this is handled by setting --net=host and control-port?

@kgaillot

kgaillot May 1, 2017

Collaborator

So "#uname" means "the host that the connection is running on"? And therefore any remote with "#uname" must use a unique port? And in the bundle context, this is handled by setting --net=host and control-port?

This comment has been minimized.

Show comment
Hide comment
@beekhof

beekhof May 1, 2017

Member

Correct to all

@beekhof

beekhof May 1, 2017

Member

Correct to all

}
if(data->control_port) {
View
@@ -1017,6 +1017,133 @@ get_ticket_state_legacy(gpointer key, gpointer value, gpointer user_data)
}
}
static void
unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * data_set)
{
const char *resource_discovery_enabled = NULL;
xmlNode *attrs = NULL;
resource_t *rsc = NULL;
const char *shutdown = NULL;
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
return;
}
if ((this_node == NULL) || (is_remote_node(this_node) == FALSE)) {
return;
}
crm_trace("Processing remote node id=%s, uname=%s", this_node->details->id, this_node->details->uname);
this_node->details->remote_maintenance =
crm_atoi(crm_element_value(state, XML_NODE_IS_MAINTENANCE), "0");
rsc = this_node->details->remote_rsc;
if (this_node->details->remote_requires_reset == FALSE) {
this_node->details->unclean = FALSE;
this_node->details->unseen = FALSE;
}
attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
add_node_attrs(attrs, this_node, TRUE, data_set);
shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);
if (shutdown != NULL && safe_str_neq("0", shutdown)) {
crm_info("Node %s is shutting down", this_node->details->uname);
this_node->details->shutdown = TRUE;
if (rsc) {
rsc->next_role = RSC_ROLE_STOPPED;
}
}
if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
crm_info("Node %s is in standby-mode", this_node->details->uname);
this_node->details->standby = TRUE;
}
if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance")) ||
(rsc && !is_set(rsc->flags, pe_rsc_managed))) {
crm_info("Node %s is in maintenance-mode", this_node->details->uname);
this_node->details->maintenance = TRUE;
}
resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.",
XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
} else {
/* if we're here, this is either a baremetal node and fencing is enabled,
* or this is a container node which we don't care if fencing is enabled
* or not on. container nodes are 'fenced' by recovering the container resource
* regardless of whether fencing is enabled. */
crm_info("Node %s has resource discovery disabled", this_node->details->uname);
this_node->details->rsc_discovery_enabled = FALSE;
}
}
}
static bool
unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set)
{
bool changed = false;
xmlNode *lrm_rsc = NULL;
for (xmlNode *state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
const char *id = NULL;
const char *uname = NULL;
node_t *this_node = NULL;
bool process = FALSE;
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
continue;
}
id = crm_element_value(state, XML_ATTR_ID);
uname = crm_element_value(state, XML_ATTR_UNAME);
this_node = pe_find_node_any(data_set->nodes, id, uname);
if (this_node == NULL) {
crm_info("Node %s is unknown", id);
continue;
} else if (this_node->details->unpacked) {
crm_info("Node %s is already processed", id);
continue;
} else if (is_remote_node(this_node) == FALSE && is_set(data_set->flags, pe_flag_stonith_enabled)) {
// A redundant test, but preserves the order for regression tests
process = TRUE;
} else if (is_remote_node(this_node)) {
resource_t *rsc = this_node->details->remote_rsc;
if (fence || (rsc && rsc->role == RSC_ROLE_STARTED)) {
determine_remote_online_status(data_set, this_node);
unpack_handle_remote_attrs(this_node, state, data_set);
process = TRUE;
}
} else if (this_node->details->online) {
process = TRUE;
} else if (fence) {
process = TRUE;
}
if(process) {
crm_trace("Processing lrm resource entries on %shealthy%s node: %s",
fence?"un":"", is_remote_node(this_node)?" remote":"",
this_node->details->uname);
changed = TRUE;
this_node->details->unpacked = TRUE;
lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
unpack_lrm_resources(this_node, lrm_rsc, data_set);
}
}
return changed;
}
/* remove nodes that are down, stopping */
/* create +ve rsc_to_node constraints between resources and the nodes they are running on */
/* anything else? */
@@ -1027,7 +1154,6 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set)
const char *uname = NULL;
xmlNode *state = NULL;
xmlNode *lrm_rsc = NULL;
node_t *this_node = NULL;
crm_trace("Beginning unpack");
@@ -1125,152 +1251,25 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set)
}
}
/* Now that we know all node states, we can safely handle migration ops */
for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
continue;
}
id = crm_element_value(state, XML_ATTR_ID);
uname = crm_element_value(state, XML_ATTR_UNAME);
this_node = pe_find_node_any(data_set->nodes, id, uname);
if (this_node == NULL) {
crm_info("Node %s is unknown", id);
continue;
} else if (is_remote_node(this_node)) {
/* online status of remote node can not be determined until all other
* resource status is unpacked. */
continue;
} else if (this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
crm_trace("Processing lrm resource entries on healthy node: %s",
this_node->details->uname);
lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
unpack_lrm_resources(this_node, lrm_rsc, data_set);
}
}
/* now that the rest of the cluster's status is determined
* calculate remote-nodes */
unpack_remote_status(status, data_set);
return TRUE;
}
gboolean
unpack_remote_status(xmlNode * status, pe_working_set_t * data_set)
{
const char *id = NULL;
const char *uname = NULL;
const char *shutdown = NULL;
resource_t *rsc = NULL;
GListPtr gIter = NULL;
xmlNode *state = NULL;
xmlNode *lrm_rsc = NULL;
node_t *this_node = NULL;
if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
crm_trace("no remote nodes to unpack");
return TRUE;
while(unpack_node_loop(status, FALSE, data_set)) {
crm_trace("Start another loop");
}
/* get online status */
for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
this_node = gIter->data;
// Now catch any nodes we didnt see
unpack_node_loop(status, is_set(data_set->flags, pe_flag_stonith_enabled), data_set);
if ((this_node == NULL) || (is_remote_node(this_node) == FALSE)) {
continue;
}
determine_remote_online_status(data_set, this_node);
}
for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
node_t *this_node = gIter->data;
/* process attributes */
for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
const char *resource_discovery_enabled = NULL;
xmlNode *attrs = NULL;
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
continue;
}
id = crm_element_value(state, XML_ATTR_ID);
uname = crm_element_value(state, XML_ATTR_UNAME);
this_node = pe_find_node_any(data_set->nodes, id, uname);
if ((this_node == NULL) || (is_remote_node(this_node) == FALSE)) {
if (this_node == NULL) {
continue;
}
crm_trace("Processing remote node id=%s, uname=%s", id, uname);
this_node->details->remote_maintenance =
crm_atoi(crm_element_value(state, XML_NODE_IS_MAINTENANCE), "0");
rsc = this_node->details->remote_rsc;
if (this_node->details->remote_requires_reset == FALSE) {
this_node->details->unclean = FALSE;
this_node->details->unseen = FALSE;
}
attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
add_node_attrs(attrs, this_node, TRUE, data_set);
shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);
if (shutdown != NULL && safe_str_neq("0", shutdown)) {
crm_info("Node %s is shutting down", this_node->details->uname);
this_node->details->shutdown = TRUE;
if (rsc) {
rsc->next_role = RSC_ROLE_STOPPED;
}
}
if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
crm_info("Node %s is in standby-mode", this_node->details->uname);
this_node->details->standby = TRUE;
}
if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance")) ||
(rsc && !is_set(rsc->flags, pe_rsc_managed))) {
crm_info("Node %s is in maintenance-mode", this_node->details->uname);
this_node->details->maintenance = TRUE;
}
resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.",
XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
} else {
/* if we're here, this is either a baremetal node and fencing is enabled,
* or this is a container node which we don't care if fencing is enabled
* or not on. container nodes are 'fenced' by recovering the container resource
* regardless of whether fencing is enabled. */
crm_info("Node %s has resource discovery disabled", this_node->details->uname);
this_node->details->rsc_discovery_enabled = FALSE;
}
}
}
/* process node rsc status */
for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
} else if(is_remote_node(this_node) == FALSE) {
continue;
}
id = crm_element_value(state, XML_ATTR_ID);
uname = crm_element_value(state, XML_ATTR_UNAME);
this_node = pe_find_node_any(data_set->nodes, id, uname);
if ((this_node == NULL) || (is_remote_node(this_node) == FALSE)) {
} else if(this_node->details->unpacked) {
continue;
}
crm_trace("Processing lrm resource entries on healthy remote node: %s",
this_node->details->uname);
lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
unpack_lrm_resources(this_node, lrm_rsc, data_set);
determine_remote_online_status(data_set, this_node);
}
return TRUE;
View
@@ -1675,6 +1675,38 @@ filter_parameters(xmlNode * param_set, const char *param_string, bool need_prese
}
}
bool fix_remote_addr(resource_t * rsc)
{
const char *name;
const char *value;
const char *attr_list[] = {
XML_ATTR_TYPE,
XML_AGENT_ATTR_CLASS,
XML_AGENT_ATTR_PROVIDER
};
const char *value_list[] = {
"remote",
"ocf",
"pacemaker"
};
name = "addr";
value = g_hash_table_lookup(rsc->parameters, name);
if (safe_str_eq(value, "#uname") == FALSE) {
return FALSE;
}
for (int lpc = 0; rsc && lpc < DIMOF(attr_list); lpc++) {
name = attr_list[lpc];
value = crm_element_value(rsc->xml, attr_list[lpc]);
if (safe_str_eq(value, value_list[lpc]) == FALSE) {
return FALSE;
}
}
return TRUE;
}
op_digest_cache_t *
rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
pe_working_set_t * data_set)
@@ -1724,6 +1756,13 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
local_versioned_params = create_xml_node(NULL, XML_TAG_VER_ATTRS);
pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set);
data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);
if(fix_remote_addr(rsc) && node) {
// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
crm_xml_add(data->params_all, "addr", node->details->uname);
crm_trace("Fixing addr for %s on %s", rsc->id, node->details->uname);
}
g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
g_hash_table_foreach(action->extra, hash2field, data->params_all);
g_hash_table_foreach(rsc->parameters, hash2field, data->params_all);
View
@@ -263,7 +263,15 @@ container_expand(resource_t * rsc, pe_working_set_t * data_set)
for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
container_grouping_t *tuple = (container_grouping_t *)gIter->data;
CRM_ASSERT(tuple);
if(fix_remote_addr(tuple->remote) && tuple->docker->allocated_to) {
// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
xmlNode *nvpair = get_xpath_object("//nvpair[@name='addr']", tuple->remote->xml, LOG_ERR);
g_hash_table_replace(tuple->remote->parameters, strdup("addr"), strdup(tuple->docker->allocated_to->details->uname));
crm_xml_add(nvpair, "value", tuple->docker->allocated_to->details->uname);
}

This comment has been minimized.

Show comment
Hide comment
@kgaillot

kgaillot May 1, 2017

Collaborator

How do you get around pengine/native.c:1490?

/* If this rsc is a remote connection resource associated
 * with a container ( which will most likely be a virtual guest )
 * do not allow the container to live on any remote-nodes.
 * remote-nodes managing nested remote-nodes should not be allowed. */
if (rsc->is_remote_node && rsc->container) {
    rsc_avoids_remote_nodes(rsc->container);
}
@kgaillot

kgaillot May 1, 2017

Collaborator

How do you get around pengine/native.c:1490?

/* If this rsc is a remote connection resource associated
 * with a container ( which will most likely be a virtual guest )
 * do not allow the container to live on any remote-nodes.
 * remote-nodes managing nested remote-nodes should not be allowed. */
if (rsc->is_remote_node && rsc->container) {
    rsc_avoids_remote_nodes(rsc->container);
}

This comment has been minimized.

Show comment
Hide comment
@beekhof

beekhof May 1, 2017

Member

We don't, all connection resources still live on the real cluster.
The difference now is that the other end can be inside a container that is itself on a remote node.

@beekhof

beekhof May 1, 2017

Member

We don't, all connection resources still live on the real cluster.
The difference now is that the other end can be inside a container that is itself on a remote node.

if(tuple->ip) {
tuple->ip->cmds->expand(tuple->ip, data_set);
}
Oops, something went wrong.

0 comments on commit 26c59fb

Please sign in to comment.