Crozzy auto config

client.go

@@ -38,6 +38,14 @@ func New(address string) (*Client, error) {
 		return nil, err
 	}
 
+	if address == "" {
+		var nnErr error
+		address, nnErr = GetNamenodeFromConfig()
+
+		if nnErr != nil {
+			return nil, nnErr
+		}
+	}
 	return NewForUser(address, username)
 }
 

client_auto.go

@@ -0,0 +1,95 @@
+package hdfs
+
+import (
+	"encoding/xml"
+	"errors"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+)
+
+type Property struct {
+	Name  string `xml:"name"`
+	Value string `xml:"value"`
+}
+
+type Result struct {
+	Property []Property `xml:"property"`
+}
+
+type HadoopConf map[string]string
+
+var ErrUnresolvedNamenode = errors.New("Couldn't find a namenode address in any config.")
+
+// Get Hadoop Properties - try to open a conf file, marshal the results
+// into a Result object and return the Properties of that object.
+func LoadHadoopConfig(path string) (HadoopConf, error) {
+	result := Result{}
+	f, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	xmlErr := xml.Unmarshal(f, &result)
+	if xmlErr != nil {
+		return nil, xmlErr
+	}
+
+	hadoopConf := make(HadoopConf)
+	for _, prop := range result.Property {
+		hadoopConf[prop.Name] = prop.Value
+	}
+	return hadoopConf, nil
+}
+
+func (conf HadoopConf) Namenodes() []string {
+	var nns []string
+	for key, value := range conf {
+		if strings.Contains(key, "fs.defaultFS") {
+			nnUrl, _ := url.Parse(value)
+			nns = append(nns, nnUrl.Host)
+		}
+		if strings.HasPrefix(key, "dfs.namenode.rpc-address") {
+			nns = append(nns, value)
+		}
+	}
+	return nns
+}
+
+// Return first namenode address we find in the hadoop config files
+// else we try and return HADOOP_NAMENODE env var value else err
+func GetNamenodeFromConfig() (string, error) {
+	hadoopHome := os.Getenv("HADOOP_HOME")
+	hadoopConfDir := os.Getenv("HADOOP_CONF_DIR")
+	var tryPaths []string
+	if hadoopHome != "" {
+		hdfsPath := path.Join(hadoopHome, "conf", "hdfs-site.xml")
+		corePath := path.Join(hadoopHome, "conf", "core-site.xml")
+		tryPaths = append(tryPaths, []string{hdfsPath, corePath}...)
+	}
+	if hadoopConfDir != "" {
+		confHdfsPath := path.Join(hadoopConfDir, "hdfs-site.xml")
+		confCorePath := path.Join(hadoopConfDir, "core-site.xml")
+		tryPaths = append(tryPaths, []string{confHdfsPath, confCorePath}...)
+	}
+	var nameNodes []string
+	for _, tryPath := range tryPaths {
+		hadoopConf, err := LoadHadoopConfig(tryPath)
+		if err == nil {
+			nameNodes = append(nameNodes, hadoopConf.Namenodes()...)
+		}
+	}
+
+	var address string
+	if len(nameNodes) > 0 {
+		address = nameNodes[0]
+	} else if os.Getenv("HADOOP_NAMENODE") != "" {
+		address = os.Getenv("HADOOP_NAMENODE")
+	} else {
+		return "", ErrUnresolvedNamenode
+	}
+
+	return address, nil
+}

client_auto_test.go

@@ -0,0 +1,31 @@
+package hdfs
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAutoClientEnvVar(t *testing.T) {
+	_, err := GetNamenodeFromConfig()
+	assert.Nil(t, err)
+}
+
+func TestAutoClientHadoopHome(t *testing.T) {
+	pwd, _ := os.Getwd()
+	os.Setenv("HADOOP_HOME", strings.Join([]string{pwd, "test"}, "/"))
+	nn, _ := GetNamenodeFromConfig()
+	require.Contains(t, []string{"hadoop-namenode-01:8020", "hadoop-namenode-02:8020"}, nn)
+	os.Setenv("HADOOP_HOME", "")
+}
+
+func TestAutoClientHadoopConfDir(t *testing.T) {
+	pwd, _ := os.Getwd()
+	os.Setenv("HADOOP_CONF_DIR", strings.Join([]string{pwd, "test"}, "/"))
+	nn, _ := GetNamenodeFromConfig()
+	assert.EqualValues(t, "testnode:9000", nn)
+	os.Setenv("HADOOP_CONF_DIR", "")
+}

test/conf/hdfs-site.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" ?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration>
+    <property>
+        <name>dfs.nameservices</name>
+        <value>tests</value>
+    </property>
+    <property>
+       <name>dfs.ha.automatic-failover.enabled</name>
+       <value>true</value>
+    </property>
+    <property>
+        <name>dfs.ha.namenodes.tests</name>
+        <value>nn1,nn2</value>
+    </property>
+    <property>
+        <name>dfs.ha.namenodes.tests</name>
+        <value>nn1,nn2</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-address.tests.nn1</name>
+        <value>hadoop-namenode-01:8020</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-address.tests.nn2</name>
+        <value>hadoop-namenode-02:8020</value>
+    </property>
+    <property>
+        <name>dfs.namenode.http-address.tests.nn1</name>
+        <value>hadoop-namenode-01:50070</value>
+    </property>
+    <property>
+        <name>dfs.namenode.http-address.tests.nn2</name>
+        <value>hadoop-namenode-02:50070</value>
+    </property>
+</configuration>

test/core-site.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+      <name>fs.defaultFS</name>
+      <value>hdfs://testnode:9000</value>
+    </property>
+</configuration>