From 5bcfd3c824ac37bc1c5e836a87f285d938e639c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Tue, 19 Dec 2023 23:12:11 +0100 Subject: [PATCH 1/6] Brief documentation about how to configure HDFS for rack awareness. --- .../pages/usage-guide/rack-awareness.adoc | 120 ++++++++++++++++++ docs/modules/hdfs/partials/nav.adoc | 1 + 2 files changed, 121 insertions(+) create mode 100644 docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc diff --git a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc new file mode 100644 index 00000000..0894f04e --- /dev/null +++ b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc @@ -0,0 +1,120 @@ += HDFS Rack Awareness + +Apache Hadoop supports a feature called Rack Awareness, which allows defining a topology for the nodes making up a cluster. +Hadoop will then use that topology to spread out replicas of blocks in a fashion that maximizes fault tolerance. + +The default write path, for example, is to put replicas of a a newly created block first on a different node, but within the same rack, and the second copy on a node in a remote rack. +In order for this to work properly, Hadoop needs to have information about the underlying infrastructure it runs on available - in a Kubernetes environment, this means obtaining information from the pods or nodes of the cluster. + +In order to enable gathering this information the Hadoop images contain https://github.com/stackabletech/hdfs-topology-provider on the classpath, which can be configured to read labels from Kubernetes objects. + +In the current version of the SDP this is not exposed as fully integrated functionality in the operator, but rather needs to be configured via config overrides. + + +NOTE: +Until the operator code has been merged, users will need to manually deploy RBAC objects to allow the Hadoop pods access to the necessary Kubernetes objects. + +Specifically this is a ClusterRole that allows reading pods and nodes, which needs to be bound to the individual ServiceAccounts that are deployed per Hadoop cluster. + +The following listing shows the generic objects that need to be deployed: + +[source,yaml] +---- +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hdfs-clusterrole-nodes +rules: + - apiGroups: + - "" + resources: + - nodes + - pods + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. +kind: ClusterRoleBinding +metadata: + name: hdfs-clusterrolebinding-nodes +roleRef: + kind: ClusterRole + name: hdfs-clusterrole-nodes + apiGroup: rbac.authorization.k8s.io +---- + +In addition to this, the ClusterRoleBinding object needs to be patched with an entry for every Hadoop cluster in the `subjects` field: + +[source,yaml] +---- +subjects: + - kind: ServiceAccount + name: hdfs--serviceaccount + namespace: +---- + +So for an HDFS cluster named `simple-hdfs` in the `stackable` namespace, the full ClusterRoleBinding would look like this: +[source,yaml] +---- +--- +apiVersion: rbac.authorization.k8s.io/v1 +# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. +kind: ClusterRoleBinding +metadata: + name: hdfs-clusterrolebinding-nodes +subjects: + - kind: ServiceAccount + name: hdfs-simple-hdfs-serviceaccount + namespace: stackable +roleRef: + kind: ClusterRole + name: hdfs-clusterrole-nodes + apiGroup: rbac.authorization.k8s.io +---- + +To then configure the cluster for rack awareness, the following setting needs to be set via config override: + +[source,yaml] +---- +apiVersion: hdfs.stackable.tech/v1alpha1 +kind: HdfsCluster +metadata: + name: simple-hdfs +spec: + nameNodes: + config: + configOverrides: + core-site.xml: + net.topology.node.switch.mapping.impl: tech.stackable.hadoop.StackableTopologyProvider +---- + +This instructs the namenode to use the topology tool for looking up information from Kubernetes. + +Configuration of the tool is then done via the environment variable `TOPOLOGY_LABELS`. + +This variable can be set to a semicolon separated list (maximum of two levels are allowed by default) of the following format: [node|pod]: + + +So for example `node:topology.kubernetes.io/zone;pod:app.kubernetes.io/role-group` would resolve to //. + + +A full example of configuring this would look like this: + +[source,yaml] +---- +apiVersion: hdfs.stackable.tech/v1alpha1 +kind: HdfsCluster +metadata: + name: simple-hdfs +spec: + nameNodes: + config: + configOverrides: + core-site.xml: + net.topology.node.switch.mapping.impl: tech.stackable.hadoop.StackableTopologyProvider + envOverrides: + TOPOLOGY_LABELS: "node:topology.kubernetes.io/zone;pod:app.kubernetes.io/role-group" +---- diff --git a/docs/modules/hdfs/partials/nav.adoc b/docs/modules/hdfs/partials/nav.adoc index 8ba4f559..65f28a95 100644 --- a/docs/modules/hdfs/partials/nav.adoc +++ b/docs/modules/hdfs/partials/nav.adoc @@ -10,6 +10,7 @@ ** xref:hdfs:usage-guide/resources.adoc[] ** xref:hdfs:usage-guide/scaling.adoc[] ** xref:hdfs:usage-guide/fuse.adoc[] +** xref:hdfs:usage-guide/rack-awareness.adoc[] ** xref:hdfs:usage-guide/logging-log-aggregation.adoc[] ** xref:hdfs:usage-guide/monitoring.adoc[] ** xref:hdfs:usage-guide/configuration-environment-overrides.adoc[] From 371d0e6e90842e3d7a982d83c660539e4068133a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 20 Dec 2023 13:09:07 +0100 Subject: [PATCH 2/6] Some minor fixes and optimizations. --- docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc index 0894f04e..a941298f 100644 --- a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc +++ b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc @@ -67,7 +67,7 @@ metadata: name: hdfs-clusterrolebinding-nodes subjects: - kind: ServiceAccount - name: hdfs-simple-hdfs-serviceaccount + name: hdfs-serviceaccount namespace: stackable roleRef: kind: ClusterRole @@ -88,6 +88,7 @@ spec: config: configOverrides: core-site.xml: + HADOOP_CLASSPATH: "/stackable/hadoop/share/hadoop/tools/lib/topology-provider-0.1.0.jar" net.topology.node.switch.mapping.impl: tech.stackable.hadoop.StackableTopologyProvider ---- @@ -111,10 +112,10 @@ metadata: name: simple-hdfs spec: nameNodes: - config: configOverrides: core-site.xml: net.topology.node.switch.mapping.impl: tech.stackable.hadoop.StackableTopologyProvider envOverrides: + HADOOP_CLASSPATH: "/stackable/hadoop/share/hadoop/tools/lib/topology-provider-0.1.0.jar" TOPOLOGY_LABELS: "node:topology.kubernetes.io/zone;pod:app.kubernetes.io/role-group" ---- From c55284f7d275c3b7302f96588c52b4290c4e912f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 20 Dec 2023 13:19:39 +0100 Subject: [PATCH 3/6] typo --- docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc index a941298f..3a34158e 100644 --- a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc +++ b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc @@ -3,7 +3,7 @@ Apache Hadoop supports a feature called Rack Awareness, which allows defining a topology for the nodes making up a cluster. Hadoop will then use that topology to spread out replicas of blocks in a fashion that maximizes fault tolerance. -The default write path, for example, is to put replicas of a a newly created block first on a different node, but within the same rack, and the second copy on a node in a remote rack. +The default write path, for example, is to put replicas of a newly created block first on a different node, but within the same rack, and the second copy on a node in a remote rack. In order for this to work properly, Hadoop needs to have information about the underlying infrastructure it runs on available - in a Kubernetes environment, this means obtaining information from the pods or nodes of the cluster. In order to enable gathering this information the Hadoop images contain https://github.com/stackabletech/hdfs-topology-provider on the classpath, which can be configured to read labels from Kubernetes objects. From 42301066d346ed156a7588c4bf7c0671b5147778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 20 Dec 2023 16:46:14 +0100 Subject: [PATCH 4/6] Apply suggestions from code review Fixes by @siegfriedweber Co-authored-by: Siegfried Weber --- docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc index 3a34158e..fef338fc 100644 --- a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc +++ b/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc @@ -11,8 +11,7 @@ In order to enable gathering this information the Hadoop images contain https:// In the current version of the SDP this is not exposed as fully integrated functionality in the operator, but rather needs to be configured via config overrides. -NOTE: -Until the operator code has been merged, users will need to manually deploy RBAC objects to allow the Hadoop pods access to the necessary Kubernetes objects. +NOTE: Until the operator code has been merged, users will need to manually deploy RBAC objects to allow the Hadoop pods access to the necessary Kubernetes objects. Specifically this is a ClusterRole that allows reading pods and nodes, which needs to be bound to the individual ServiceAccounts that are deployed per Hadoop cluster. @@ -56,7 +55,7 @@ subjects: namespace: ---- -So for an HDFS cluster named `simple-hdfs` in the `stackable` namespace, the full ClusterRoleBinding would look like this: +So for an HDFS cluster using the ServiceAccount `hdfs-serviceaccount` in the `stackable` namespace, the full ClusterRoleBinding would look like this: [source,yaml] ---- --- @@ -85,11 +84,11 @@ metadata: name: simple-hdfs spec: nameNodes: - config: configOverrides: core-site.xml: - HADOOP_CLASSPATH: "/stackable/hadoop/share/hadoop/tools/lib/topology-provider-0.1.0.jar" net.topology.node.switch.mapping.impl: tech.stackable.hadoop.StackableTopologyProvider + envOverrides: + HADOOP_CLASSPATH: "/stackable/hadoop/share/hadoop/tools/lib/topology-provider-0.1.0.jar" ---- This instructs the namenode to use the topology tool for looking up information from Kubernetes. From 8d32428c6c878de610c5528549bcfcd12aab56db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 20 Dec 2023 16:52:58 +0100 Subject: [PATCH 5/6] Moved page on rack awareness to operations section. --- .../hdfs/pages/usage-guide/{ => operations}/rack-awareness.adoc | 0 docs/modules/hdfs/partials/nav.adoc | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/modules/hdfs/pages/usage-guide/{ => operations}/rack-awareness.adoc (100%) diff --git a/docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc b/docs/modules/hdfs/pages/usage-guide/operations/rack-awareness.adoc similarity index 100% rename from docs/modules/hdfs/pages/usage-guide/rack-awareness.adoc rename to docs/modules/hdfs/pages/usage-guide/operations/rack-awareness.adoc diff --git a/docs/modules/hdfs/partials/nav.adoc b/docs/modules/hdfs/partials/nav.adoc index 65f28a95..4c2ad4dd 100644 --- a/docs/modules/hdfs/partials/nav.adoc +++ b/docs/modules/hdfs/partials/nav.adoc @@ -10,12 +10,12 @@ ** xref:hdfs:usage-guide/resources.adoc[] ** xref:hdfs:usage-guide/scaling.adoc[] ** xref:hdfs:usage-guide/fuse.adoc[] -** xref:hdfs:usage-guide/rack-awareness.adoc[] ** xref:hdfs:usage-guide/logging-log-aggregation.adoc[] ** xref:hdfs:usage-guide/monitoring.adoc[] ** xref:hdfs:usage-guide/configuration-environment-overrides.adoc[] ** xref:hdfs:usage-guide/operations/index.adoc[] *** xref:hdfs:usage-guide/operations/cluster-operations.adoc[] *** xref:hdfs:usage-guide/operations/pod-placement.adoc[] +** xref:hdfs:usage-guide/operations/rack-awareness.adoc[] *** xref:hdfs:usage-guide/operations/pod-disruptions.adoc[] *** xref:hdfs:usage-guide/operations/graceful-shutdown.adoc[] From 4b2202ae24f65b5a753f3623c6582f2cb9a7859d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 20 Dec 2023 16:59:05 +0100 Subject: [PATCH 6/6] Fixed list indentation. --- docs/modules/hdfs/partials/nav.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/hdfs/partials/nav.adoc b/docs/modules/hdfs/partials/nav.adoc index 4c2ad4dd..f3eaddf5 100644 --- a/docs/modules/hdfs/partials/nav.adoc +++ b/docs/modules/hdfs/partials/nav.adoc @@ -16,6 +16,6 @@ ** xref:hdfs:usage-guide/operations/index.adoc[] *** xref:hdfs:usage-guide/operations/cluster-operations.adoc[] *** xref:hdfs:usage-guide/operations/pod-placement.adoc[] -** xref:hdfs:usage-guide/operations/rack-awareness.adoc[] +*** xref:hdfs:usage-guide/operations/rack-awareness.adoc[] *** xref:hdfs:usage-guide/operations/pod-disruptions.adoc[] *** xref:hdfs:usage-guide/operations/graceful-shutdown.adoc[]