From 3a5b9676272949afd31ea3df10af48dc6f5d9633 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Tue, 10 Jan 2023 08:44:45 -0600 Subject: [PATCH 1/2] Changes for NNF Fencing Signed-off-by: Nate Thornton --- docs/guides/ha-cluster/fence_nnf.py | 217 ++++++++++++++++++++++++++++ docs/guides/ha-cluster/notes.md | 5 + docs/guides/ha-cluster/readme.md | 44 +++--- 3 files changed, 248 insertions(+), 18 deletions(-) create mode 100644 docs/guides/ha-cluster/fence_nnf.py create mode 100644 docs/guides/ha-cluster/notes.md diff --git a/docs/guides/ha-cluster/fence_nnf.py b/docs/guides/ha-cluster/fence_nnf.py new file mode 100644 index 00000000..0ba0c706 --- /dev/null +++ b/docs/guides/ha-cluster/fence_nnf.py @@ -0,0 +1,217 @@ +#!@PYTHON@ + +# Fence agent for Near Node Flash devices + +import sys +import logging +import atexit + +sys.path.append("@FENCEAGENTSLIBDIR@") + +from fencing import * +from fencing import fail, fail_usage, run_delay + +try: + from kubernetes import client +except ImportError: + logging.error("Couldn't import kubernetes.client - not found or not accessible") + +def get_power_status(api_client, options): + # Power status is the status of the Kubernetes Node resource if it is down, or + # the Fencing status of the NNF Node resource if the Kubernetes Node is up. + + name = options.get("--nnf-node-name") + logging.debug("Reading node resource %s", name) + + try: + node = client.CoreV1Api(api_client).read_node(name) + + except client.exceptions.ApiException as e: + logging.debug("Exception when reading node: %s %s", e, type(e)) + if e.status == 401: + fail(EC_LOGIN_DENIED) + elif e.status == 404: + fail(EC_STATUS) + + logging.error("Exception when reading node: %s %s", e, type(e)) + + for condition in node.status.conditions: + if condition.type == "Ready" and condition.status == "True": + return _get_nnf_node_power_status(api_client, options) + + logging.info("Node %s is not ready", name) + return "off" + +def _get_nnf_node_power_status(api_client, options): + + version = options.get("--api-version") + namespace = options.get("--nnf-node-name") + logging.debug("Reading NNF node resource %s/%s", version, namespace) + + try: + node = client.CustomObjectsApi(api_client).get_namespaced_custom_object( + "nnf.cray.hpe.com", + version, + namespace, + "nnfnodes", + "nnf-nlc" + ) + + except client.exceptions.ApiException as e: + logging.debug("Exception when reading NNF node: %s %s", e, type(e)) + if e.status == 404: + fail(EC_STATUS) + + logging.error("Exception when reading NNF node: %s %s", e, type(e)) + + fenced = node["status"].get("fenced", False) + logging.info("NNF Node fenced status: %s", fenced) + + return "off" if fenced else "on" + +def set_power_status(api_client, options): + + if options.get("--action") == "on": + # NNF Fencing Agent is not permitted to enable the power status; that is done via + # adminsitrator actions. + return + + api_object = client.CustomObjectsApi(api_client) + + version = options.get("--api-version") + namespace = options.get("--nnf-node-name") + + node = api_object.get_namespaced_custom_object( + "nnf.cray.hpe.com", + version, + namespace, + "nnfnodes", + "nnf-nlc" + ) + + if not node["status"].get("fenced", False): + node["status"]["fenced"] = True + api_object.patch_namespaced_custom_object_status( + "nnf.cray.hpe.com", + version, + namespace, + "nnfnodes", + "nnf-nlc", + node + ) + + +def define_new_opts(): + all_opt["kubernetes-service-host"] = { + "getopt" : ":", + "longopt" : "kubernetes-service-host", + "help" : "--kubernetes-service-host=[ADDRESS] The IP Address of the kubeapi server", + "shortdesc" : "Kubernetes API service IP address or hostname", + "required" : "0", + "order" : 1 + } + + all_opt["kubernetes-service-port"] = { + "getopt" : ":", + "longopt" : "kubernetes-service-port", + "help" : "--kubernetes-service-port=[PORT] The listen port of the kubeapi server", + "shortdesc" : "Kubernetes API service port", + "required" : "0", + "order" : 1 + } + + all_opt["service-token-file"] = { + "getopt" : ":", + "longopt" : "service-token-file", + "help" : "--service-token-file=[PATH] The path to the service token file", + "shortdesc" : "Path to the service token file", + "required" : "1", + "order" : 2 + } + + all_opt["service-cert-file"] = { + "getopt" : ":", + "longopt" : "service-cert-file", + "help" : "--service-cert-file=[PATH] The path to the service certificate file", + "shortdesc" : "Path to the service certificate file", + "required" : "1", + "order" : 2 + } + + all_opt["nnf-node-name"] = { + "getopt" : ":", + "longopt" : "nnf-node-name", + "help" : "--nnf-node-name=[PATH] The name of the NNF node", + "shortdesc" : "The name of the NNF node as listed in the system configuration", + "required" : "1", + "order" : 3 + } + + all_opt["api-version"] = { + "getopt" : ":", + "longopt" : "api-version", + "help" : "--api-version=[VERSION] Version of the NNF Node API", + "shortdesc" : "The API Version of the NNF node resource", + "required" : "0", + "default" : "v1alpha1", + "order" : 4 + } + + all_opt["localconfig"] = { + "getopt" : "", + "longopt" : "localconfig", + "help" : "--localconfig Use the local kubernetes config", + "required" : "0", + "order" : 4 + } + +def main(): + atexit.register(atexit_handler) + + device_opt = [ + "kubernetes-service-host", "kubernetes-service-port", + "service-token-file", "service-cert-file", + "nnf-node-name", + "api-version", + "localconfig", + "no_password" # signals to fencing.py to disable password requirement + ] + + define_new_opts() + + opt = process_input(device_opt) + options = check_input(device_opt, opt) + + docs = {} + docs["shortdesc"] = "Fencing agent for Near Node Flash" + docs["longdesc"] = "fence_nnf is a fencing agent for Near Node Flash storage nodes." + docs["vendorurl"] = "https://nearnodeflash.github.io/" + show_docs(options, docs) + + run_delay(options) + + if "--localconfig" in options: + from kubernetes import config + configuration = config.load_kube_config() + else: + configuration = client.Configuration( + host="https://%s:%s" % ( + options.get("--kubernetes-service-host"), + options.get("--kubernetes-service-port") + ) + ) + + with open(options.get("--service-token-file"), "r") as f: + token = f.read() + + configuration.api_key = { "authorization" : token } + configuration.api_key_prefix = { "authorization" : "Bearer" } + configuration.ssl_ca_cert = options.get("--service-cert-file") + + with client.ApiClient(configuration) as api_client: + result = fence_action(api_client, options, set_power_status, get_power_status) + + sys.exit(result) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docs/guides/ha-cluster/notes.md b/docs/guides/ha-cluster/notes.md new file mode 100644 index 00000000..19ad1654 --- /dev/null +++ b/docs/guides/ha-cluster/notes.md @@ -0,0 +1,5 @@ +pcs stonith create stonith-rabbit-node-1 fence_nnf pcmk_host_list=rabbit-node-1 kubernetes-service-host=10.30.107.247 kubernetes-service-port=6443 service-token-file=/etc/nnf/service.token service-cert-file=/etc/nnf/service.cert nnf-node-name=rabbit-node-1 verbose=1 + +pcs stonith create stonith-rabbit-compute-2 fence_redfish pcmk_host_list="rabbit-compute-2" ip=10.30.105.237 port=80 systems-uri=/redfish/v1/Systems/1 username=root password=REDACTED ssl_insecure=true verbose=1 + +pcs stonith create stonith-rabbit-compute-3 fence_redfish pcmk_host_list="rabbit-compute-3" ip=10.30.105.253 port=80 systems-uri=/redfish/v1/Systems/1 username=root password=REDACTED ssl_insecure=true verbose=1 \ No newline at end of file diff --git a/docs/guides/ha-cluster/readme.md b/docs/guides/ha-cluster/readme.md index 46044712..4903686c 100644 --- a/docs/guides/ha-cluster/readme.md +++ b/docs/guides/ha-cluster/readme.md @@ -5,10 +5,10 @@ categories: setup # High Availability Cluster -Rabbit software supports provisioning of Red Hat GFS2 (Global File System 2) storage. Per Red Hat: +NNF software supports provisioning of Red Hat GFS2 (Global File System 2) storage. Per RedHat: > GFS2 allows multiple nodes to share storage at a block level as if the storage were connected locally to each cluster node. GFS2 cluster file system requires a cluster infrastructure. -Therefore, in order to use GFS2, the Rabbit and its associated compute nodes must form a high availability cluster. +Therefore, in order to use GFS2, the NNF node and its associated compute nodes must form a high availability cluster. ## Cluster Setup @@ -24,34 +24,42 @@ HPE hardware implements software known as the Hardware System Supervisor (HSS), ### Compute Fencing -!!! warning - Usage of the Redfish fencing agent is not yet verified - The [Redfish fencing agent](https://github.com/ClusterLabs/fence-agents/tree/main/agents/redfish) from [ClusterLabs](https://github.com/ClusterLabs/fence-agents) should be used for Compute nodes in the cluster. Configure the agent with the following parameters: | Argument | Definition | | -------- | ---------- | -| `--ip=[ADDRESS]` | The IP address or hostname of the compute node's HSS node controller | -|`--systems-uri=[URI]` | The URI of the Systems object. Must be `/redfish/v1/Systems/Node0` | -|`--ssl-insecure` | Instructs the use of an insecure SSL exchange | +| `ip=[ADDRESS]` | The IP address or hostname of the HSS controller | +| `port=[PORT]` | The Port of the HSS controller. Must be `80` | +| `systems-uri=[URI]` | The URI of the Systems object. Must be `/redfish/v1/Systems/Node0` | +| `ssl-insecure=[BOOL]` | Instructs the use of an insecure SSL exchange. Must be `true` | +| `username=[USER]` | The user name for connecting to the HSS controller | +| `password=[PASSWORD]` | the password for connecting to the HSS controller | -### Rabbit Fencing +### NNF Fencing !!! info - Rabbit fencing agent is in active development; the description below is subject to change. - -Since the Rabbit node is connected to 16 compute blades, careful coordination around fencing of a Rabbit node is required to minimize the impact of the outage. When a Rabbit node is fenced, the corresponding Kubernetes Storage resource (`storages.dws.cray.hpe.com`) is updated with a status of 'Fenced'. The workload manager must observe this change and handle the movement of resources off the Rabbit node and clear the 'Fenced' status before forcibly rebooting the node. + NNF fencing agent is in active development; the description below is subject to change. -Configure the Rabbit agent with the following parameters: +Configure the NNF agent with the following parameters: | Argument | Definition | | -------- | ---------- | -| `--kubernetes-service-host=[ADDRESS]` | The IP address of the kubeapi server | -| `--kubernetes-service-port=[PORT]` | The listening port of the kubeapi server | -| `--service-token-file=[PATH]` | The location of the service token file. The file must be present on all nodes within the cluster | -| `--service-cert-file=[PATH]` | The location of the service certificate file. The file must be present on all nodes within the cluster | -| `--nnf-node-name=[RABBIT-NODE-NAME]` | Name of the rabbit node | +| `kubernetes-service-host=[ADDRESS]` | The IP address of the kubeapi server | +| `kubernetes-service-port=[PORT]` | The listening port of the kubeapi server | +| `service-token-file=[PATH]` | The location of the service token file. The file must be present on all nodes within the cluster | +| `service-cert-file=[PATH]` | The location of the service certificate file. The file must be present on all nodes within the cluster | +| `nnf-node-name=[NNF-NODE-NAME]` | Name of the NNF node as it is appears in the System Configuration | +| `api-version=[VERSION]` | The API Version of the NNF Node resource. Defaults to "v1alpha1" | + +Since the NNF node is connected to 16 compute blades, careful coordination around fencing of a NNF node is required to minimize the impact of the outage. When a Rabbit node is fenced, the corresponding DWS Storage resource (`storages.dws.cray.hpe.com`) status changes. The workload manager must observe this change and follow the procedure below to recover from the fencing status. + +1. Observed the `storage.Status` changed and that `storage.Status.RequiresReboot == True` +2. Set the `storage.Spec.State := Disabled` +4. Wait for a change to the Storage status `storage.Status.State == Disabled` +5. Reboot the NNF node +6. Set the `storage.Spec.State := Enabled` +7. Wait for `storage.Status.State == Enabled` ### Dummy Fencing From 3cd37fd0ec8bc4672fb28b4221ec88ca89de473e Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Thu, 12 Jan 2023 08:56:41 -0600 Subject: [PATCH 2/2] address pr comments Signed-off-by: Nate Thornton --- docs/guides/ha-cluster/fence_nnf.py | 9 +++++---- docs/guides/ha-cluster/readme.md | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/guides/ha-cluster/fence_nnf.py b/docs/guides/ha-cluster/fence_nnf.py index 0ba0c706..087dade8 100644 --- a/docs/guides/ha-cluster/fence_nnf.py +++ b/docs/guides/ha-cluster/fence_nnf.py @@ -5,11 +5,12 @@ import sys import logging import atexit +import http sys.path.append("@FENCEAGENTSLIBDIR@") from fencing import * -from fencing import fail, fail_usage, run_delay +from fencing import fail, run_delay try: from kubernetes import client @@ -28,9 +29,9 @@ def get_power_status(api_client, options): except client.exceptions.ApiException as e: logging.debug("Exception when reading node: %s %s", e, type(e)) - if e.status == 401: + if e.status == http.HTTPStatus.UNAUTHORIZED: fail(EC_LOGIN_DENIED) - elif e.status == 404: + elif e.status == http.HTTPStatus.NOT_FOUND: fail(EC_STATUS) logging.error("Exception when reading node: %s %s", e, type(e)) @@ -59,7 +60,7 @@ def _get_nnf_node_power_status(api_client, options): except client.exceptions.ApiException as e: logging.debug("Exception when reading NNF node: %s %s", e, type(e)) - if e.status == 404: + if e.status == http.HTTPStatus.NOT_FOUND: fail(EC_STATUS) logging.error("Exception when reading NNF node: %s %s", e, type(e)) diff --git a/docs/guides/ha-cluster/readme.md b/docs/guides/ha-cluster/readme.md index 4903686c..dcf7d70d 100644 --- a/docs/guides/ha-cluster/readme.md +++ b/docs/guides/ha-cluster/readme.md @@ -29,9 +29,9 @@ The [Redfish fencing agent](https://github.com/ClusterLabs/fence-agents/tree/mai | Argument | Definition | | -------- | ---------- | | `ip=[ADDRESS]` | The IP address or hostname of the HSS controller | -| `port=[PORT]` | The Port of the HSS controller. Must be `80` | -| `systems-uri=[URI]` | The URI of the Systems object. Must be `/redfish/v1/Systems/Node0` | -| `ssl-insecure=[BOOL]` | Instructs the use of an insecure SSL exchange. Must be `true` | +| `port=80` | The Port of the HSS controller. Must be `80` | +| `systems-uri=/redfish/v1/Systems/Node0` | The URI of the Systems object. Must be `/redfish/v1/Systems/Node0` | +| `ssl-insecure=true` | Instructs the use of an insecure SSL exchange. Must be `true` | | `username=[USER]` | The user name for connecting to the HSS controller | | `password=[PASSWORD]` | the password for connecting to the HSS controller |